From c1fdd940529e547d969cbbc25b99029d1eaad699 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Fri, 23 Aug 2019 16:16:38 +0200 Subject: [PATCH 001/224] feat: Add GitHub Actions --- .../.github/workflows/ci.yml | 20 +++++++++ .../.github/workflows/linting.yml | 41 +++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml create mode 100644 nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml new file mode 100644 index 000000000..4dc1f3685 --- /dev/null +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml @@ -0,0 +1,20 @@ +name: {{ cookiecutter.name }} CI +# This workflow is triggered on pushes and PRs to the repository. +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + nxf_ver: ['19.04.0', ''] + steps: + - uses: actions/checkout@v1 + - name: Install Nextflow + run: | + export NXF_VER=${{ matrix.nxf_ver }} + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + - name: Run test + run: | + nextflow run ${GITHUB_WORKSPACE} -profile test,docker \ No newline at end of file diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml new file mode 100644 index 000000000..628f63a5e --- /dev/null +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml @@ -0,0 +1,41 @@ +name: {{ cookiecutter.name }} lint +# This workflow is triggered on pushes and PRs to the repository. +on: [push, pull_request] + +jobs: + Markdown: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-node@v1 + with: + node-version: '10' + - name: Install markdownlint + run: | + npm install -g markdownlint-cli + - name: Run Markdownlint + run: | + markdownlint ${GITHUB_WORKSPACE} -c ${GITHUB_WORKSPACE}/.github/markdownlint.yml + nf-core: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Install Nextflow + run: | + export NXF_VER=${{ matrix.nxf_ver }} + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + - uses: actions/setup-python@v1 + with: + python-version: '3.6' + architecture: 'x64' + - name: Install pip + run: | + sudo apt install python3-pip + pip install --upgrade pip + - name: Install nf-core tools + run: | + pip install nf-core + - name: Run nf-core lint + run: | + nf-core lint ${GITHUB_WORKSPACE} \ No newline at end of file From b3d9ff2fc9fab976d6220f9f2e8721fe267d8aa5 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 28 Aug 2019 16:07:00 +0200 Subject: [PATCH 002/224] update CHAMGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 807b6ef9a..459543c69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ an AWS S3 bucket as the `--outdir`. * Fix workflow.onComplete() message when finishing pipeline * Update URL for joining the nf-core slack to https://nf-co.re/join/slack +* Add GitHub Action for CI and Linting ### Other From fb82d8186077059d880f5b4dcd25b25bb0a0d36c Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 28 Aug 2019 16:43:05 +0200 Subject: [PATCH 003/224] add badges --- .../pipeline-template/{{cookiecutter.name_noslash}}/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md index c5bf9c452..3c3302517 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md @@ -3,6 +3,8 @@ **{{ cookiecutter.description }}**. [![Build Status](https://travis-ci.com/{{ cookiecutter.name }}.svg?branch=master)](https://travis-ci.com/{{ cookiecutter.name }}) +[![GitHub Actions CI Status](https://github.com/{{ cookiecutter.name }}/workflows/ci/badge.svg)](https://github.com/{{ cookiecutter.name }}/actions) +[![GitHub Actions Linting Status](https://github.com/{{ cookiecutter.name }}/workflows/linting/badge.svg)](https://github.com/{{ cookiecutter.name }}/actions) [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.32.0-brightgreen.svg)](https://www.nextflow.io/) [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](http://bioconda.github.io/) From daa0857043e65d47e1ffd1f8ecf39f1ba3d3d2be Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Fri, 30 Aug 2019 11:39:58 +0200 Subject: [PATCH 004/224] fix badges --- .../{{cookiecutter.name_noslash}}/.github/workflows/ci.yml | 2 +- .../.github/workflows/linting.yml | 3 +-- .../pipeline-template/{{cookiecutter.name_noslash}}/README.md | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml index 4dc1f3685..8fd361870 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: {{ cookiecutter.name }} CI +name: nf-core CI # This workflow is triggered on pushes and PRs to the repository. on: [push, pull_request] diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml index 628f63a5e..8c1e97c07 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml @@ -1,4 +1,4 @@ -name: {{ cookiecutter.name }} lint +name: nf-core linting # This workflow is triggered on pushes and PRs to the repository. on: [push, pull_request] @@ -22,7 +22,6 @@ jobs: - uses: actions/checkout@v1 - name: Install Nextflow run: | - export NXF_VER=${{ matrix.nxf_ver }} wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - uses: actions/setup-python@v1 diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md index 3c3302517..0f64176a8 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md @@ -3,8 +3,8 @@ **{{ cookiecutter.description }}**. [![Build Status](https://travis-ci.com/{{ cookiecutter.name }}.svg?branch=master)](https://travis-ci.com/{{ cookiecutter.name }}) -[![GitHub Actions CI Status](https://github.com/{{ cookiecutter.name }}/workflows/ci/badge.svg)](https://github.com/{{ cookiecutter.name }}/actions) -[![GitHub Actions Linting Status](https://github.com/{{ cookiecutter.name }}/workflows/linting/badge.svg)](https://github.com/{{ cookiecutter.name }}/actions) +[![GitHub Actions CI Status](https://github.com/{{ cookiecutter.name }}/workflows/nf-core%20CI/badge.svg)](https://github.com/{{ cookiecutter.name }}/actions) +[![GitHub Actions Linting Status](https://github.com/{{ cookiecutter.name }}/workflows/nf-core%20linting/badge.svg)](https://github.com/{{ cookiecutter.name }}/actions) [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.32.0-brightgreen.svg)](https://www.nextflow.io/) [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](http://bioconda.github.io/) From 2a959e24b530c551d52ce42c4c2b22a73e65a55e Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Fri, 30 Aug 2019 11:40:31 +0200 Subject: [PATCH 005/224] add branch protection --- .../.github/workflows/branch.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml new file mode 100644 index 000000000..05b94b0da --- /dev/null +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml @@ -0,0 +1,14 @@ +name: nf-core branch protection +# This workflow is triggered on PRs to master branch on the repository +on: + pull_request: + branches: + - master + +jobs: + test: + runs-on: ubuntu-latest + steps: + # PRs are only ok if coming from an nf-core dev branch + - name: Check PRs + run: [ ${GITHUB_ACTOR} = "nf-core" ] && [ ${GITHUB_HEAD_REF} = "dev" ] \ No newline at end of file From 219ffb587d3f740c8f3c7edeb2b3eace3b94cc9f Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 3 Sep 2019 16:52:27 +0200 Subject: [PATCH 006/224] fix typo --- .../{{cookiecutter.name_noslash}}/.github/workflows/branch.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml index 05b94b0da..e3c557ce1 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml @@ -11,4 +11,5 @@ jobs: steps: # PRs are only ok if coming from an nf-core dev branch - name: Check PRs - run: [ ${GITHUB_ACTOR} = "nf-core" ] && [ ${GITHUB_HEAD_REF} = "dev" ] \ No newline at end of file + run: | + [ ${GITHUB_ACTOR} = "nf-core" ] && [ ${GITHUB_HEAD_REF} = "dev" ] \ No newline at end of file From 4a0a4e2c88a76b69a086212c649f947d73f157e6 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 7 Oct 2019 16:08:30 +0200 Subject: [PATCH 007/224] Version bump to v1.8dev --- CHANGELOG.md | 4 ++++ setup.py | 2 +- tests/lint_examples/minimal_working_example/Dockerfile | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf38320a1..8e83f0ac7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # nf-core/tools: Changelog +## v1.8dev + +..nothing yet.. + ## v1.7 ### PyPI package description diff --git a/setup.py b/setup.py index f67bfab2a..585c40ffa 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages import sys -version = '1.7' +version = '1.8dev' with open('README.md') as f: readme = f.read() diff --git a/tests/lint_examples/minimal_working_example/Dockerfile b/tests/lint_examples/minimal_working_example/Dockerfile index bcb5d79cd..fc562dc1d 100644 --- a/tests/lint_examples/minimal_working_example/Dockerfile +++ b/tests/lint_examples/minimal_working_example/Dockerfile @@ -1,4 +1,4 @@ -FROM nfcore/base:1.7 +FROM nfcore/base:dev MAINTAINER Phil Ewels LABEL authors="phil.ewels@scilifelab.se" \ description="Docker image containing all requirements for the nf-core tools pipeline" From 27cda76c7c83877ec5e2c128b980362154078184 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 7 Oct 2019 20:39:55 +0200 Subject: [PATCH 008/224] Update blacklist --- CHANGELOG.md | 8 +++++++- bin/blacklist.json | 7 ++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e83f0ac7..7f15ef2c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,13 @@ ## v1.8dev -..nothing yet.. +### Syncing + +* Updated Blacklist of synced pipelines + +### Other + +* Updated Base Dockerfile to Conda 4.7.10 ## v1.7 diff --git a/bin/blacklist.json b/bin/blacklist.json index 47959b9e4..8602e2cef 100644 --- a/bin/blacklist.json +++ b/bin/blacklist.json @@ -12,6 +12,11 @@ "lncpipe", "ddamsproteomics", "scrnaseq", - "guideseq" + "guideseq", + "nanodemux", + "mnaseseq", + "bactmap", + "kmermaid", + "crisprvar" ] } From 864cc8f8d4b837cd9b186845b51d23cedccb2dcf Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 7 Oct 2019 20:40:03 +0200 Subject: [PATCH 009/224] Update base Image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7e3babe4b..0a62abdf7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM continuumio/miniconda:4.6.14 +FROM continuumio/miniconda:4.7.10 LABEL authors="phil.ewels@scilifelab.se,alexander.peltzer@qbic.uni-tuebingen.de" \ description="Docker image containing base requirements for the nfcore pipelines" From 6870ed90c78a18ddff0535d519a7cdec8167ccda Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 8 Oct 2019 09:26:21 +0200 Subject: [PATCH 010/224] Adjust according to @drpatel --- bin/blacklist.json | 9 --------- 1 file changed, 9 deletions(-) diff --git a/bin/blacklist.json b/bin/blacklist.json index 8602e2cef..430c91d3d 100644 --- a/bin/blacklist.json +++ b/bin/blacklist.json @@ -1,20 +1,11 @@ { "pipelines": [ - "epitopeprediction", "exoseq", "neutronstar", - "smrnaseq", "vipr", - "sarek", "neutronstar", "proteomicslfq", "clinvap", - "lncpipe", - "ddamsproteomics", - "scrnaseq", - "guideseq", - "nanodemux", - "mnaseseq", "bactmap", "kmermaid", "crisprvar" From 514dac66e0c1c38169f1348e2f638d713f5d0cba Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 10 Oct 2019 12:41:58 +0200 Subject: [PATCH 011/224] Fix #418 --- CHANGELOG.md | 4 ++++ .../{{cookiecutter.name_noslash}}/conf/igenomes.config | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f15ef2c6..e982c820b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ * Updated Base Dockerfile to Conda 4.7.10 +### Template + +* Fixed incorrect paths in iGenomes config as described in issue [418](https://github.com/nf-core/tools/issues/418) + ## v1.7 ### PyPI package description diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/igenomes.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/igenomes.config index 392f25073..6edcc5b68 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/igenomes.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/igenomes.config @@ -24,8 +24,8 @@ params { fasta = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/WholeGenomeFasta/genome.fa" gtf = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Annotation/Genes/genes.gtf" star = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCh37/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCh37/Sequence/BWAIndex/" + bowtie2 = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/Bowtie2Index/" + bwa = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/BWAIndex/" } 'TAIR10' { bed12 = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Annotation/Genes/genes.bed" From bd927ba8ae9ca5b180c2c503df3b5317a5f6ee67 Mon Sep 17 00:00:00 2001 From: "Raghu kamat(RK)" Date: Thu, 10 Oct 2019 10:44:49 -0400 Subject: [PATCH 012/224] Issue #420: Fixing the broken slack link and pointing it to the correct slack channel --- CHANGELOG.md | 2 +- .../{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md | 3 ++- .../pipeline-template/{{cookiecutter.name_noslash}}/README.md | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf38320a1..2c4681eed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -217,7 +217,7 @@ Very large release containing lots of work from the first nf-core hackathon, hel * New pipelines are now created using the command `nf-core create` * The nf-core template and associated linting are now controlled under the same version system * Large number of template updates and associated linting changes - * New simplified cookicutter variable usage + * New simplified cookiecutter variable usage * Refactored documentation - simplified and reduced duplication * Better `manifest` variables instead of `params` for pipeline name and version * New integrated nextflow version checking diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md index 937aed821..20b93ab6c 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md @@ -44,4 +44,5 @@ If there are any failures then the automated tests fail. These tests are run both with the latest available version of Nextflow and also the minimum required version that is stated in the pipeline code. ## Getting help -For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.name }}#documentation) and don't hesitate to get in touch on the [{{ cookiecutter.name }} pipeline channel](https://nfcore.slack.com/channels/{{ cookiecutter.name }}) on [Slack](https://nf-co.re/join/slack/). +For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.short_name }}#documentation) and don't hesitate to get in touch on the [{{ cookiecutter.name }} pipeline channel](https://nfcore.slack.com/channels/{{ cookiecutter.name }}) on Slack. + diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md index b219a0c3a..f2f750668 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md @@ -56,7 +56,7 @@ The {{ cookiecutter.name }} pipeline comes with documentation about the pipeline If you would like to contribute to this pipeline, please see the [contributing guidelines](.github/CONTRIBUTING.md). -For further information or help, don't hesitate to get in touch on [Slack](https://nfcore.slack.com/channels/{{ cookiecutter.name }}) (you can join with [this invite](https://nf-co.re/join/slack)). +For further information or help, don't hesitate to get in touch on [Slack](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) (you can join with [this invite](https://nf-co.re/join/slack)). ## Citation From f48d1e4531af361ddbbcb2e35c071138bcc6b43d Mon Sep 17 00:00:00 2001 From: "Raghu kamat(RK)" Date: Thu, 10 Oct 2019 10:49:44 -0400 Subject: [PATCH 013/224] reverting the documentation in contributing.md and correcting the slack link --- .../{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md index 20b93ab6c..4cf0b042f 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md @@ -44,5 +44,5 @@ If there are any failures then the automated tests fail. These tests are run both with the latest available version of Nextflow and also the minimum required version that is stated in the pipeline code. ## Getting help -For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.short_name }}#documentation) and don't hesitate to get in touch on the [{{ cookiecutter.name }} pipeline channel](https://nfcore.slack.com/channels/{{ cookiecutter.name }}) on Slack. +For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.name }}#documentation) and don't hesitate to get in touch on the [{{ cookiecutter.short_name }} pipeline channel](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) on [Slack](https://nf-co.re/join/slack/). From ca3b7a70d02a135bff816506943c389600e0dbf6 Mon Sep 17 00:00:00 2001 From: "Raghu kamat(RK)" Date: Thu, 10 Oct 2019 10:50:42 -0400 Subject: [PATCH 014/224] removing extra line --- .../{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md index 4cf0b042f..8623af62c 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md @@ -45,4 +45,3 @@ These tests are run both with the latest available version of Nextflow and also ## Getting help For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.name }}#documentation) and don't hesitate to get in touch on the [{{ cookiecutter.short_name }} pipeline channel](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) on [Slack](https://nf-co.re/join/slack/). - From 72d65a6404d40f61aea71d2381e1ae8bd9c184e9 Mon Sep 17 00:00:00 2001 From: "Raghu kamat(RK)" Date: Thu, 10 Oct 2019 10:54:06 -0400 Subject: [PATCH 015/224] reverting the accidental change made to the pipeline name --- .../{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md index 8623af62c..63ec44141 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md @@ -44,4 +44,4 @@ If there are any failures then the automated tests fail. These tests are run both with the latest available version of Nextflow and also the minimum required version that is stated in the pipeline code. ## Getting help -For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.name }}#documentation) and don't hesitate to get in touch on the [{{ cookiecutter.short_name }} pipeline channel](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) on [Slack](https://nf-co.re/join/slack/). +For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.name }}#documentation) and don't hesitate to get in touch on the [{{ cookiecutter.name }} pipeline channel](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) on [Slack](https://nf-co.re/join/slack/). \ No newline at end of file From 712af13840485c2a2ad118ae1e549172af187b92 Mon Sep 17 00:00:00 2001 From: "Raghu kamat(RK)" Date: Thu, 10 Oct 2019 10:44:49 -0400 Subject: [PATCH 016/224] Issue #420: Fixing the broken slack link and pointing it to the correct slack channel --- CHANGELOG.md | 2 +- .../{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md | 2 +- .../pipeline-template/{{cookiecutter.name_noslash}}/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf38320a1..2c4681eed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -217,7 +217,7 @@ Very large release containing lots of work from the first nf-core hackathon, hel * New pipelines are now created using the command `nf-core create` * The nf-core template and associated linting are now controlled under the same version system * Large number of template updates and associated linting changes - * New simplified cookicutter variable usage + * New simplified cookiecutter variable usage * Refactored documentation - simplified and reduced duplication * Better `manifest` variables instead of `params` for pipeline name and version * New integrated nextflow version checking diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md index 937aed821..63ec44141 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md @@ -44,4 +44,4 @@ If there are any failures then the automated tests fail. These tests are run both with the latest available version of Nextflow and also the minimum required version that is stated in the pipeline code. ## Getting help -For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.name }}#documentation) and don't hesitate to get in touch on the [{{ cookiecutter.name }} pipeline channel](https://nfcore.slack.com/channels/{{ cookiecutter.name }}) on [Slack](https://nf-co.re/join/slack/). +For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.name }}#documentation) and don't hesitate to get in touch on the [{{ cookiecutter.name }} pipeline channel](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) on [Slack](https://nf-co.re/join/slack/). \ No newline at end of file diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md index b219a0c3a..f2f750668 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md @@ -56,7 +56,7 @@ The {{ cookiecutter.name }} pipeline comes with documentation about the pipeline If you would like to contribute to this pipeline, please see the [contributing guidelines](.github/CONTRIBUTING.md). -For further information or help, don't hesitate to get in touch on [Slack](https://nfcore.slack.com/channels/{{ cookiecutter.name }}) (you can join with [this invite](https://nf-co.re/join/slack)). +For further information or help, don't hesitate to get in touch on [Slack](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) (you can join with [this invite](https://nf-co.re/join/slack)). ## Citation From 4270be102039aa2c67eb25e5bdaf79d2fe7db6ba Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 14 Oct 2019 14:19:43 +0200 Subject: [PATCH 017/224] Create: Supply short_name to template. Fixes error introduced in nf-core/tools#421 --- nf_core/create.py | 1 + nf_core/pipeline-template/cookiecutter.json | 1 + 2 files changed, 2 insertions(+) diff --git a/nf_core/create.py b/nf_core/create.py index 5b0802eb8..e44a7105f 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -81,6 +81,7 @@ def run_cookiecutter(self): 'author': self.author, 'name_noslash': self.name_noslash, 'name_docker': self.name_docker, + 'short_name': self.short_name, 'version': self.new_version, 'nf_core_version': nf_core.__version__ }, diff --git a/nf_core/pipeline-template/cookiecutter.json b/nf_core/pipeline-template/cookiecutter.json index dfc2f668a..4d10d7404 100644 --- a/nf_core/pipeline-template/cookiecutter.json +++ b/nf_core/pipeline-template/cookiecutter.json @@ -4,6 +4,7 @@ "author": "Rocky Balboa", "name_noslash": "{{ cookiecutter.name.replace('/', '-') }}", "name_docker": "{{ cookiecutter.name_docker }}", + "short_name": "{{ cookiecutter.short_name }}", "version": "1.0dev", "nf_core_version": "{{ cookiecutter.nf_core_version }}" } From 6385e473e5abe18c92af1b55cc4bce55ceaf4605 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 14 Oct 2019 14:21:13 +0200 Subject: [PATCH 018/224] Changelog --- CHANGELOG.md | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ceb2a69b8..65c6eaddc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,23 +2,19 @@ ## v1.8dev -### Syncing +### Tools helper code * Updated Blacklist of synced pipelines -### Other - -* Updated Base Dockerfile to Conda 4.7.10 - ### Template * Fixed incorrect paths in iGenomes config as described in issue [418](https://github.com/nf-core/tools/issues/418) -## v1.7 +### Other -### PyPI package description +* Updated Base Dockerfile to Conda 4.7.10 -* The readme should now be rendered properly on PyPI. +## v1.7 ### Tools helper code @@ -31,6 +27,7 @@ * When listing pipelines, a nicer message is given for the rare case of a detached `HEAD` ref in a locally pulled pipeline. [#297](https://github.com/nf-core/tools/issues/297) * The `download` command can now compress files into a single archive. * `nf-core create` now fetches a logo for the pipeline from the nf-core website +* The readme should now be rendered properly on PyPI. ### Syncing From 0d0f1db0a935cd3ea5936ae78fd05dfb42e349b8 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 14 Oct 2019 15:00:47 +0100 Subject: [PATCH 019/224] Highlight citation --- .../pipeline-template/{{cookiecutter.name_noslash}}/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md index f2f750668..f27b38b1f 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md @@ -64,4 +64,4 @@ For further information or help, don't hesitate to get in touch on [Slack](https You can cite the `nf-core` pre-print as follows: -Ewels PA, Peltzer A, Fillinger S, Alneberg JA, Patel H, Wilm A, Garcia MU, Di Tommaso P, Nahnsen S. **nf-core: Community curated bioinformatics pipelines**. *bioRxiv*. 2019. p. 610741. [doi: 10.1101/610741](https://www.biorxiv.org/content/10.1101/610741v1). +> Ewels PA, Peltzer A, Fillinger S, Alneberg JA, Patel H, Wilm A, Garcia MU, Di Tommaso P, Nahnsen S. **nf-core: Community curated bioinformatics pipelines**. *bioRxiv*. 2019. p. 610741. [doi: 10.1101/610741](https://www.biorxiv.org/content/10.1101/610741v1). From baa33dcf38135fcfc765a81fba3f2b450afcb097 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 14 Oct 2019 15:01:00 +0100 Subject: [PATCH 020/224] Update igenomes files --- .../conf/igenomes.config | 473 +++++++++++++----- 1 file changed, 339 insertions(+), 134 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/igenomes.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/igenomes.config index 6edcc5b68..37217cf4e 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/igenomes.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/igenomes.config @@ -9,184 +9,389 @@ params { // illumina iGenomes reference file paths - // TODO nf-core: Add new reference types and strip out those that are not needed genomes { 'GRCh37' { - bed12 = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Annotation/Genes/genes.bed" + mito_name = "MT" + macs_gsize = "2.7e9" + blacklist = "${baseDir}/assets/blacklists/GRCh37-blacklist.bed" + } + 'GRCh38' { + fasta = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Annotation/Genes/genes.bed" + mito_name = "chrM" + macs_gsize = "2.7e9" + blacklist = "${baseDir}/assets/blacklists/hg38-blacklist.bed" } 'GRCm38' { - bed12 = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Annotation/Genes/genes.bed" + mito_name = "MT" + macs_gsize = "1.87e9" + blacklist = "${baseDir}/assets/blacklists/GRCm38-blacklist.bed" } 'TAIR10' { - bed12 = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Annotation/Genes/genes.bed" + mito_name = "Mt" } 'EB2' { - bed12 = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Annotation/Genes/genes.bed" } 'UMD3.1' { - bed12 = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/BWAIndex/" - + fasta = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Annotation/Genes/genes.bed" + mito_name = "MT" } 'WBcel235' { - bed12 = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Annotation/Genes/genes.bed" + mito_name = "MtDNA" + macs_gsize = "9e7" } 'CanFam3.1' { - bed12 = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Annotation/Genes/genes.bed" + mito_name = "MT" } 'GRCz10' { - bed12 = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Annotation/Genes/genes.bed" + mito_name = "MT" } 'BDGP6' { - bed12 = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Annotation/Genes/genes.bed" + mito_name = "M" + macs_gsize = "1.2e8" } 'EquCab2' { - bed12 = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Annotation/Genes/genes.bed" + mito_name = "MT" } 'EB1' { - bed12 = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Annotation/Genes/genes.bed" } 'Galgal4' { - bed12 = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Annotation/Genes/genes.bed" + mito_name = "MT" } 'Gm01' { - bed12 = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Annotation/Genes/genes.bed" } 'Mmul_1' { - bed12 = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Annotation/Genes/genes.bed" + mito_name = "MT" } 'IRGSP-1.0' { - bed12 = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Annotation/Genes/genes.bed" + mito_name = "Mt" } 'CHIMP2.1.4' { - bed12 = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Annotation/Genes/genes.bed" + mito_name = "MT" } 'Rnor_6.0' { - bed12 = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Annotation/Genes/genes.bed" + mito_name = "MT" } 'R64-1-1' { - bed12 = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Annotation/Genes/genes.bed" + mito_name = "MT" + macs_gsize = "1.2e7" } 'EF2' { - bed12 = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Annotation/Genes/genes.bed" + mito_name = "MT" + macs_gsize = "1.21e7" } 'Sbi1' { - bed12 = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Annotation/Genes/genes.bed" } 'Sscrofa10.2' { - bed12 = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Annotation/Genes/genes.bed" + mito_name = "MT" } 'AGPv3' { - bed12 = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Annotation/Genes/genes.bed" - fasta = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/WholeGenomeFasta/genome.fa" - gtf = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Annotation/Genes/genes.gtf" - star = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/STARIndex/" - bowtie2 = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/Bowtie2Index/" - bwa = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/BWAIndex/" + fasta = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Annotation/Genes/genes.bed" + mito_name = "Mt" + } + 'hg38' { + fasta = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Annotation/Genes/genes.bed" + mito_name = "chrM" + macs_gsize = "2.7e9" + blacklist = "${baseDir}/assets/blacklists/hg38-blacklist.bed" + } + 'hg19' { + fasta = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Annotation/Genes/genes.bed" + mito_name = "chrM" + macs_gsize = "2.7e9" + blacklist = "${baseDir}/assets/blacklists/hg19-blacklist.bed" + } + 'mm10' { + fasta = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Annotation/Genes/genes.bed" + mito_name = "chrM" + macs_gsize = "1.87e9" + blacklist = "${baseDir}/assets/blacklists/mm10-blacklist.bed" + } + 'bosTau8' { + fasta = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Annotation/Genes/genes.bed" + mito_name = "chrM" + } + 'ce10' { + fasta = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Annotation/Genes/genes.bed" + mito_name = "chrM" + macs_gsize = "9e7" + } + 'canFam3' { + fasta = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Annotation/Genes/genes.bed" + mito_name = "chrM" + } + 'danRer10' { + fasta = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Annotation/Genes/genes.bed" + mito_name = "chrM" + } + 'dm6' { + fasta = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Annotation/Genes/genes.bed" + mito_name = "chrM" + macs_gsize = "1.2e8" + } + 'equCab2' { + fasta = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Annotation/Genes/genes.bed" + mito_name = "chrM" + } + 'galGal4' { + fasta = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Annotation/Genes/genes.bed" + mito_name = "chrM" + } + 'panTro4' { + fasta = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Annotation/Genes/genes.bed" + mito_name = "chrM" + } + 'rn6' { + fasta = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Annotation/Genes/genes.bed" + mito_name = "chrM" + } + 'sacCer3' { + fasta = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/BismarkIndex/" + mito_name = "chrM" + macs_gsize = "1.2e7" + } + 'susScr3' { + fasta = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/BWAIndex/genome.fa" + bowtie2 = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/Bowtie2Index/" + star = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/STARIndex/" + bismark = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/BismarkIndex/" + gtf = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Annotation/Genes/genes.gtf" + bed12 = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Annotation/Genes/genes.bed" + mito_name = "chrM" } } } From 8e1f6fc22aedb63d1b4c90145643133c8f939b4d Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 14 Oct 2019 15:01:17 +0100 Subject: [PATCH 021/224] Add max_multiqc_email_size parameter --- .../{{cookiecutter.name_noslash}}/docs/usage.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index cade6b7ce..a52aaca6c 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -27,6 +27,7 @@ * [`--outdir`](#--outdir) * [`--email`](#--email) * [`--email_on_fail`](#--email_on_fail) + * [`--max_multiqc_email_size`](#--max_multiqc_email_size) * [`-name`](#-name) * [`-resume`](#-resume) * [`-c`](#-c) @@ -215,6 +216,9 @@ Set this parameter to your e-mail address to get a summary e-mail with details o ### `--email_on_fail` This works exactly as with `--email`, except emails are only sent if the workflow is not successful. +### `max_multiqc_email_size` +Theshold size for MultiQC report to be attached in notification email. If file generated by pipeline exceeds the threshold, it will not be attached (Default: 25MB). + ### `-name` Name for the pipeline run. If not specified, Nextflow will automatically generate a random mnemonic. From 29b35038ebf917f0baae183dd047c9a7281d5166 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 14 Oct 2019 15:01:28 +0100 Subject: [PATCH 022/224] Add max_multiqc_email_size parameter --- .../{{cookiecutter.name_noslash}}/main.nf | 76 +++++++++---------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index 29baacd38..e189fad49 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -36,7 +36,7 @@ def helpMessage() { --outdir The output directory where the results will be saved --email Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits --email_on_fail Same as --email, except only send mail if the workflow is not successful - --maxMultiqcEmailFileSize Theshold size for MultiQC report to be attached in notification email. If file generated by pipeline exceeds the threshold, it will not be attached (Default: 25MB) + --max_multiqc_email_size Theshold size for MultiQC report to be attached in notification email. If file generated by pipeline exceeds the threshold, it will not be attached (Default: 25MB) -name Name for the pipeline run. If not specified, Nextflow will automatically generate a random mnemonic. AWSBatch options: @@ -75,17 +75,17 @@ if (params.fasta) { ch_fasta = file(params.fasta, checkIfExists: true) } // this has the bonus effect of catching both -name and --name custom_runName = params.name if (!(workflow.runName ==~ /[a-z]+_[a-z]+/)) { - custom_runName = workflow.runName + custom_runName = workflow.runName } if ( workflow.profile == 'awsbatch') { - // AWSBatch sanity checking - if (!params.awsqueue || !params.awsregion) exit 1, "Specify correct --awsqueue and --awsregion parameters on AWSBatch!" - // Check outdir paths to be S3 buckets if running on AWSBatch - // related: https://github.com/nextflow-io/nextflow/issues/813 - if (!params.outdir.startsWith('s3:')) exit 1, "Outdir not on S3 - specify S3 Bucket to run on AWSBatch!" - // Prevent trace files to be stored on S3 since S3 does not support rolling files. - if (workflow.tracedir.startsWith('s3:')) exit 1, "Specify a local tracedir or run without trace! S3 cannot be used for tracefiles." + // AWSBatch sanity checking + if (!params.awsqueue || !params.awsregion) exit 1, "Specify correct --awsqueue and --awsregion parameters on AWSBatch!" + // Check outdir paths to be S3 buckets if running on AWSBatch + // related: https://github.com/nextflow-io/nextflow/issues/813 + if (!params.outdir.startsWith('s3:')) exit 1, "Outdir not on S3 - specify S3 Bucket to run on AWSBatch!" + // Prevent trace files to be stored on S3 since S3 does not support rolling files. + if (workflow.tracedir.startsWith('s3:')) exit 1, "Specify a local tracedir or run without trace! S3 cannot be used for tracefiles." } // Stage config files @@ -101,19 +101,19 @@ if (params.readPaths) { .from(params.readPaths) .map { row -> [ row[0], [ file(row[1][0], checkIfExists: true) ] ] } .ifEmpty { exit 1, "params.readPaths was empty - no input files supplied" } - .into { read_files_fastqc; read_files_trimming } + .into { ch_read_files_fastqc; ch_read_files_trimming } } else { Channel .from(params.readPaths) .map { row -> [ row[0], [ file(row[1][0], checkIfExists: true), file(row[1][1], checkIfExists: true) ] ] } .ifEmpty { exit 1, "params.readPaths was empty - no input files supplied" } - .into { read_files_fastqc; read_files_trimming } + .into { ch_read_files_fastqc; ch_read_files_trimming } } } else { Channel .fromFilePairs( params.reads, size: params.singleEnd ? 1 : 2 ) .ifEmpty { exit 1, "Cannot find any reads matching: ${params.reads}\nNB: Path needs to be enclosed in quotes!\nIf this is single-end data, please specify --singleEnd on the command line." } - .into { read_files_fastqc; read_files_trimming } + .into { ch_read_files_fastqc; ch_read_files_trimming } } // Header log info @@ -133,17 +133,17 @@ summary['Working dir'] = workflow.workDir summary['Script dir'] = workflow.projectDir summary['User'] = workflow.userName if (workflow.profile == 'awsbatch') { - summary['AWS Region'] = params.awsregion - summary['AWS Queue'] = params.awsqueue + summary['AWS Region'] = params.awsregion + summary['AWS Queue'] = params.awsqueue } summary['Config Profile'] = workflow.profile if (params.config_profile_description) summary['Config Description'] = params.config_profile_description if (params.config_profile_contact) summary['Config Contact'] = params.config_profile_contact if (params.config_profile_url) summary['Config URL'] = params.config_profile_url if (params.email || params.email_on_fail) { - summary['E-mail Address'] = params.email - summary['E-mail on failure'] = params.email_on_fail - summary['MultiQC maxsize'] = params.maxMultiqcEmailFileSize + summary['E-mail Address'] = params.email + summary['E-mail on failure'] = params.email_on_fail + summary['MultiQC maxsize'] = params.max_multiqc_email_size } log.info summary.collect { k,v -> "${k.padRight(18)}: $v" }.join("\n") log.info "-\033[2m--------------------------------------------------\033[0m-" @@ -179,7 +179,7 @@ process get_software_versions { } output: - file 'software_versions_mqc.yaml' into software_versions_yaml + file 'software_versions_mqc.yaml' into ch_software_versions_yaml file "software_versions.csv" script: @@ -203,10 +203,10 @@ process fastqc { saveAs: { filename -> filename.indexOf(".zip") > 0 ? "zips/$filename" : "$filename" } input: - set val(name), file(reads) from read_files_fastqc + set val(name), file(reads) from ch_read_files_fastqc output: - file "*_fastqc.{zip,html}" into fastqc_results + file "*_fastqc.{zip,html}" into ch_fastqc_results script: """ @@ -223,12 +223,12 @@ process multiqc { input: file multiqc_config from ch_multiqc_config // TODO nf-core: Add in log files from your new processes for MultiQC to find! - file ('fastqc/*') from fastqc_results.collect().ifEmpty([]) - file ('software_versions/*') from software_versions_yaml.collect() + file ('fastqc/*') from ch_fastqc_results.collect().ifEmpty([]) + file ('software_versions/*') from ch_software_versions_yaml.collect() file workflow_summary from create_workflow_summary(summary) output: - file "*multiqc_report.html" into multiqc_report + file "*multiqc_report.html" into ch_multiqc_report file "*_data" file "multiqc_plots" @@ -267,7 +267,7 @@ workflow.onComplete { // Set up the e-mail variables def subject = "[{{ cookiecutter.name }}] Successful: $workflow.runName" if (!workflow.success) { - subject = "[{{ cookiecutter.name }}] FAILED: $workflow.runName" + subject = "[{{ cookiecutter.name }}] FAILED: $workflow.runName" } def email_fields = [:] email_fields['version'] = workflow.manifest.version @@ -293,12 +293,12 @@ workflow.onComplete { email_fields['summary']['Nextflow Build'] = workflow.nextflow.build email_fields['summary']['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp - // TODO nf-core: If not using MultiQC, strip out this code (including params.maxMultiqcEmailFileSize) + // TODO nf-core: If not using MultiQC, strip out this code (including params.max_multiqc_email_size) // On success try attach the multiqc report def mqc_report = null try { if (workflow.success) { - mqc_report = multiqc_report.getVal() + mqc_report = ch_multiqc_report.getVal() if (mqc_report.getClass() == ArrayList) { log.warn "[{{ cookiecutter.name }}] Found multiple reports from process 'multiqc', will use only one" mqc_report = mqc_report[0] @@ -326,7 +326,7 @@ workflow.onComplete { def email_html = html_template.toString() // Render the sendmail template - def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, baseDir: "$baseDir", mqcFile: mqc_report, mqcMaxSize: params.maxMultiqcEmailFileSize.toBytes() ] + def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, baseDir: "$baseDir", mqcFile: mqc_report, mqcMaxSize: params.max_multiqc_email_size.toBytes() ] def sf = new File("$baseDir/assets/sendmail_template.txt") def sendmail_template = engine.createTemplate(sf).make(smail_fields) def sendmail_html = sendmail_template.toString() @@ -334,21 +334,21 @@ workflow.onComplete { // Send the HTML e-mail if (email_address) { try { - if ( params.plaintext_email ){ throw GroovyException('Send plaintext e-mail, not HTML') } - // Try to send HTML e-mail using sendmail - [ 'sendmail', '-t' ].execute() << sendmail_html - log.info "[{{ cookiecutter.name }}] Sent summary e-mail to $email_address (sendmail)" + if ( params.plaintext_email ){ throw GroovyException('Send plaintext e-mail, not HTML') } + // Try to send HTML e-mail using sendmail + [ 'sendmail', '-t' ].execute() << sendmail_html + log.info "[{{ cookiecutter.name }}] Sent summary e-mail to $email_address (sendmail)" } catch (all) { - // Catch failures and try with plaintext - [ 'mail', '-s', subject, email_address ].execute() << email_txt - log.info "[{{ cookiecutter.name }}] Sent summary e-mail to $email_address (mail)" + // Catch failures and try with plaintext + [ 'mail', '-s', subject, email_address ].execute() << email_txt + log.info "[{{ cookiecutter.name }}] Sent summary e-mail to $email_address (mail)" } } // Write summary e-mail HTML to a file def output_d = new File( "${params.outdir}/pipeline_info/" ) if (!output_d.exists()) { - output_d.mkdirs() + output_d.mkdirs() } def output_hf = new File( output_d, "pipeline_report.html" ) output_hf.withWriter { w -> w << email_html } @@ -361,9 +361,9 @@ workflow.onComplete { c_red = params.monochrome_logs ? '' : "\033[0;31m"; if (workflow.stats.ignoredCount > 0 && workflow.success) { - log.info "${c_purple}Warning, pipeline completed, but with errored process(es) ${c_reset}" - log.info "${c_red}Number of ignored errored process(es) : ${workflow.stats.ignoredCount} ${c_reset}" - log.info "${c_green}Number of successfully ran process(es) : ${workflow.stats.succeedCount} ${c_reset}" + log.info "${c_purple}Warning, pipeline completed, but with errored process(es) ${c_reset}" + log.info "${c_red}Number of ignored errored process(es) : ${workflow.stats.ignoredCount} ${c_reset}" + log.info "${c_green}Number of successfully ran process(es) : ${workflow.stats.succeedCount} ${c_reset}" } if (workflow.success) { From 92a8e79afb1453e777804ef2892136c7f4df5723 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 14 Oct 2019 15:01:42 +0100 Subject: [PATCH 023/224] Add max_multiqc_email_size parameter --- .../{{cookiecutter.name_noslash}}/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config index 6ce293dab..30bc8b8d9 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config @@ -20,7 +20,7 @@ params { multiqc_config = "$baseDir/assets/multiqc_config.yaml" email = false email_on_fail = false - maxMultiqcEmailFileSize = 25.MB + max_multiqc_email_size = 25.MB plaintext_email = false monochrome_logs = false help = false From 57e8f703f78ecff7d5429bce2c3d16bbf59460b8 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 14 Oct 2019 15:03:37 +0100 Subject: [PATCH 024/224] Update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65c6eaddc..5d1b6d050 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ ### Template * Fixed incorrect paths in iGenomes config as described in issue [418](https://github.com/nf-core/tools/issues/418) +* Add UCSC genomes to `igenomes.config` and add paths to all genome indices +* Change `maxMultiqcEmailFileSize` parameter to `max_multiqc_email_size` ### Other From 24e388fb9f7e3e02c4210856d79e26dc14e21c3a Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 14 Oct 2019 22:43:46 +0100 Subject: [PATCH 025/224] Add new para --- .../pipeline-template/{{cookiecutter.name_noslash}}/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md index f27b38b1f..df92c7f0d 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md @@ -64,4 +64,5 @@ For further information or help, don't hesitate to get in touch on [Slack](https You can cite the `nf-core` pre-print as follows: + > Ewels PA, Peltzer A, Fillinger S, Alneberg JA, Patel H, Wilm A, Garcia MU, Di Tommaso P, Nahnsen S. **nf-core: Community curated bioinformatics pipelines**. *bioRxiv*. 2019. p. 610741. [doi: 10.1101/610741](https://www.biorxiv.org/content/10.1101/610741v1). From f3a83c457dce19cf866ae0df903867ec3c8f5c4d Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 14 Oct 2019 22:44:18 +0100 Subject: [PATCH 026/224] Add -- before param --- .../{{cookiecutter.name_noslash}}/docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index a52aaca6c..42369033d 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -216,7 +216,7 @@ Set this parameter to your e-mail address to get a summary e-mail with details o ### `--email_on_fail` This works exactly as with `--email`, except emails are only sent if the workflow is not successful. -### `max_multiqc_email_size` +### `--max_multiqc_email_size` Theshold size for MultiQC report to be attached in notification email. If file generated by pipeline exceeds the threshold, it will not be attached (Default: 25MB). ### `-name` From 78a8380212b3fabb42324b162d117c32de1c2bbf Mon Sep 17 00:00:00 2001 From: drpatelh Date: Tue, 15 Oct 2019 21:57:04 +0100 Subject: [PATCH 027/224] Export Conda environment in Dockerfile --- .../pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile index f57f757da..edc016682 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile @@ -5,3 +5,4 @@ LABEL authors="{{ cookiecutter.author }}" \ COPY environment.yml / RUN conda env create -f /environment.yml && conda clean -a ENV PATH /opt/conda/envs/{{ cookiecutter.name_noslash }}-{{ cookiecutter.version }}/bin:$PATH +RUN conda env export --name {{ cookiecutter.name_noslash }}-{{ cookiecutter.version }} > {{ cookiecutter.name_noslash }}-{{ cookiecutter.version }}.yml From 71b69fab283fbbf9f37f8fd27a26626505a43722 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Tue, 15 Oct 2019 21:59:04 +0100 Subject: [PATCH 028/224] Update CHANGELOG --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d1b6d050..d0b4281fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,10 @@ ### Template -* Fixed incorrect paths in iGenomes config as described in issue [418](https://github.com/nf-core/tools/issues/418) +* Fixed incorrect paths in iGenomes config as described in issue [#418](https://github.com/nf-core/tools/issues/418) * Add UCSC genomes to `igenomes.config` and add paths to all genome indices * Change `maxMultiqcEmailFileSize` parameter to `max_multiqc_email_size` +* Export conda environment in Docker file [#349](https://github.com/nf-core/tools/issues/349) ### Other From cf9c662a1bfe1a3be55264d405b151acc93bac45 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Tue, 15 Oct 2019 22:13:31 +0100 Subject: [PATCH 029/224] Reorder commands --- .../pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile index edc016682..5c4e2dd93 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile @@ -4,5 +4,5 @@ LABEL authors="{{ cookiecutter.author }}" \ COPY environment.yml / RUN conda env create -f /environment.yml && conda clean -a -ENV PATH /opt/conda/envs/{{ cookiecutter.name_noslash }}-{{ cookiecutter.version }}/bin:$PATH RUN conda env export --name {{ cookiecutter.name_noslash }}-{{ cookiecutter.version }} > {{ cookiecutter.name_noslash }}-{{ cookiecutter.version }}.yml +ENV PATH /opt/conda/envs/{{ cookiecutter.name_noslash }}-{{ cookiecutter.version }}/bin:$PATH From cc1695edec268c370e1d2b311ff16a6d1b03817e Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 16 Oct 2019 14:58:04 +0100 Subject: [PATCH 030/224] Add docker,singularity to run profile --- .../{{cookiecutter.name_noslash}}/conf/test.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test.config index 9ce6312ae..70b44fa4d 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test.config @@ -4,7 +4,7 @@ * ------------------------------------------------- * Defines bundled input files and everything required * to run a fast and simple test. Use as follows: - * nextflow run {{ cookiecutter.name }} -profile test + * nextflow run {{ cookiecutter.name }} -profile test, */ params { @@ -14,11 +14,11 @@ params { max_cpus = 2 max_memory = 6.GB max_time = 48.h - + // Input data // TODO nf-core: Specify the paths to your test data on nf-core/test-datasets // TODO nf-core: Give any required params for the test so that command line flags are not needed - singleEnd = false + single_end = false readPaths = [ ['Testdata', ['https://github.com/nf-core/test-datasets/raw/exoseq/testdata/Testdata_R1.tiny.fastq.gz', 'https://github.com/nf-core/test-datasets/raw/exoseq/testdata/Testdata_R2.tiny.fastq.gz']], ['SRR389222', ['https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', 'https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub2.fastq.gz']] From 8504f64f2f7ceadc1495f40da8472eb7d969ff32 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 16 Oct 2019 14:58:26 +0100 Subject: [PATCH 031/224] Change snakeCase to camel_case --- .../{{cookiecutter.name_noslash}}/docs/usage.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index 42369033d..0ba7a1434 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -12,11 +12,11 @@ * [Main arguments](#main-arguments) * [`-profile`](#-profile) * [`--reads`](#--reads) - * [`--singleEnd`](#--singleend) + * [`--single_end`](#--single_end) * [Reference genomes](#reference-genomes) * [`--genome` (using iGenomes)](#--genome-using-igenomes) * [`--fasta`](#--fasta) - * [`--igenomesIgnore`](#--igenomesignore) + * [`--igenomes_ignore`](#--igenomes_ignore) * [Job resources](#job-resources) * [Automatic resubmission](#automatic-resubmission) * [Custom resource requests](#custom-resource-requests) @@ -125,11 +125,11 @@ Please note the following requirements: If left unspecified, a default pattern is used: `data/*{1,2}.fastq.gz` -### `--singleEnd` -By default, the pipeline expects paired-end data. If you have single-end data, you need to specify `--singleEnd` on the command line when you launch the pipeline. A normal glob pattern, enclosed in quotation marks, can then be used for `--reads`. For example: +### `--single_end` +By default, the pipeline expects paired-end data. If you have single-end data, you need to specify `--single_end` on the command line when you launch the pipeline. A normal glob pattern, enclosed in quotation marks, can then be used for `--reads`. For example: ```bash ---singleEnd --reads '*.fastq' +--single_end --reads '*.fastq' ``` It is not possible to run a mixture of single-end and paired-end files in one run. @@ -180,7 +180,7 @@ If you prefer, you can specify the full path to your reference genome when you r --fasta '[path to Fasta reference]' ``` -### `--igenomesIgnore` +### `--igenomes_ignore` Do not load `igenomes.config` when running the pipeline. You may choose this option if you observe clashes between custom parameters and those supplied in `igenomes.config`. ## Job resources From 8e6d91a742a2dfb13498e0720af91610ca2223ee Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 16 Oct 2019 14:58:36 +0100 Subject: [PATCH 032/224] Change snakeCase to camel_case --- .../{{cookiecutter.name_noslash}}/main.nf | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index e189fad49..12aa4b1c7 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -23,21 +23,21 @@ def helpMessage() { Mandatory arguments: --reads Path to input data (must be surrounded with quotes) -profile Configuration profile to use. Can use multiple (comma separated) - Available: conda, docker, singularity, awsbatch, test and more. + Available: conda, docker, singularity, awsbatch, test and more Options: --genome Name of iGenomes reference - --singleEnd Specifies that the input is single end reads + --single_end Specifies that the input is single-end reads - References If not specified in the configuration file or you wish to overwrite any of the references. - --fasta Path to Fasta reference + References If not specified in the configuration file or you wish to overwrite any of the references + --fasta Path to fasta reference Other options: --outdir The output directory where the results will be saved --email Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits --email_on_fail Same as --email, except only send mail if the workflow is not successful --max_multiqc_email_size Theshold size for MultiQC report to be attached in notification email. If file generated by pipeline exceeds the threshold, it will not be attached (Default: 25MB) - -name Name for the pipeline run. If not specified, Nextflow will automatically generate a random mnemonic. + -name Name for the pipeline run. If not specified, Nextflow will automatically generate a random mnemonic AWSBatch options: --awsqueue The AWSBatch JobQueue that needs to be set when running on AWSBatch @@ -78,7 +78,7 @@ if (!(workflow.runName ==~ /[a-z]+_[a-z]+/)) { custom_runName = workflow.runName } -if ( workflow.profile == 'awsbatch') { +if (workflow.profile == 'awsbatch') { // AWSBatch sanity checking if (!params.awsqueue || !params.awsregion) exit 1, "Specify correct --awsqueue and --awsregion parameters on AWSBatch!" // Check outdir paths to be S3 buckets if running on AWSBatch @@ -96,7 +96,7 @@ ch_output_docs = file("$baseDir/docs/output.md", checkIfExists: true) * Create a channel for input read files */ if (params.readPaths) { - if (params.singleEnd) { + if (params.single_end) { Channel .from(params.readPaths) .map { row -> [ row[0], [ file(row[1][0], checkIfExists: true) ] ] } @@ -111,8 +111,8 @@ if (params.readPaths) { } } else { Channel - .fromFilePairs( params.reads, size: params.singleEnd ? 1 : 2 ) - .ifEmpty { exit 1, "Cannot find any reads matching: ${params.reads}\nNB: Path needs to be enclosed in quotes!\nIf this is single-end data, please specify --singleEnd on the command line." } + .fromFilePairs(params.reads, size: params.single_end ? 1 : 2) + .ifEmpty { exit 1, "Cannot find any reads matching: ${params.reads}\nNB: Path needs to be enclosed in quotes!\nIf this is single-end data, please specify --single_end on the command line." } .into { ch_read_files_fastqc; ch_read_files_trimming } } @@ -124,7 +124,7 @@ summary['Run Name'] = custom_runName ?: workflow.runName // TODO nf-core: Report custom parameters here summary['Reads'] = params.reads summary['Fasta Ref'] = params.fasta -summary['Data Type'] = params.singleEnd ? 'Single-End' : 'Paired-End' +summary['Data Type'] = params.single_end ? 'Single-End' : 'Paired-End' summary['Max Resources'] = "$params.max_memory memory, $params.max_cpus cpus, $params.max_time time per job" if (workflow.containerEngine) summary['Container'] = "$workflow.containerEngine - $workflow.container" summary['Output dir'] = params.outdir @@ -334,7 +334,7 @@ workflow.onComplete { // Send the HTML e-mail if (email_address) { try { - if ( params.plaintext_email ){ throw GroovyException('Send plaintext e-mail, not HTML') } + if (params.plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } // Try to send HTML e-mail using sendmail [ 'sendmail', '-t' ].execute() << sendmail_html log.info "[{{ cookiecutter.name }}] Sent summary e-mail to $email_address (sendmail)" @@ -346,13 +346,13 @@ workflow.onComplete { } // Write summary e-mail HTML to a file - def output_d = new File( "${params.outdir}/pipeline_info/" ) + def output_d = new File("${params.outdir}/pipeline_info/") if (!output_d.exists()) { output_d.mkdirs() } - def output_hf = new File( output_d, "pipeline_report.html" ) + def output_hf = new File(output_d, "pipeline_report.html") output_hf.withWriter { w -> w << email_html } - def output_tf = new File( output_d, "pipeline_report.txt" ) + def output_tf = new File(output_d, "pipeline_report.txt") output_tf.withWriter { w -> w << email_txt } c_reset = params.monochrome_logs ? '' : "\033[0m"; @@ -376,7 +376,7 @@ workflow.onComplete { } -def nfcoreHeader(){ +def nfcoreHeader() { // Log colors ANSI codes c_reset = params.monochrome_logs ? '' : "\033[0m"; c_dim = params.monochrome_logs ? '' : "\033[2m"; @@ -399,7 +399,7 @@ def nfcoreHeader(){ """.stripIndent() } -def checkHostname(){ +def checkHostname() { def c_reset = params.monochrome_logs ? '' : "\033[0m" def c_white = params.monochrome_logs ? '' : "\033[0;37m" def c_red = params.monochrome_logs ? '' : "\033[1;91m" From c5f07b0cdbd6bd6f7762d8ac27296f2d5b2df737 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 16 Oct 2019 14:58:43 +0100 Subject: [PATCH 033/224] Change snakeCase to camel_case --- .../{{cookiecutter.name_noslash}}/nextflow.config | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config index 30bc8b8d9..9d21dc0d7 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config @@ -12,7 +12,7 @@ params { // TODO nf-core: Specify your pipeline's command line flags genome = false reads = "data/*{1,2}.fastq.gz" - singleEnd = false + single_end = false outdir = './results' // Boilerplate options @@ -28,7 +28,7 @@ params { tracedir = "${params.outdir}/pipeline_info" awsqueue = false awsregion = 'eu-west-1' - igenomesIgnore = false + igenomes_ignore = false custom_config_version = 'master' custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" hostnames = false @@ -56,7 +56,8 @@ profiles { conda { process.conda = "$baseDir/environment.yml" } debug { process.beforeScript = 'echo $HOSTNAME' } docker { docker.enabled = true } - singularity { singularity.enabled = true } + singularity { singularity.enabled = true + singularity.autoMounts = true } test { includeConfig 'conf/test.config' } } @@ -66,7 +67,7 @@ profiles { docker.runOptions = '-u \$(id -u):\$(id -g)' // Load igenomes.config if required -if (!params.igenomesIgnore) { +if (!params.igenomes_ignore) { includeConfig 'conf/igenomes.config' } From 92f2ae2aec1404ee6b85e08d08071c835e0ad059 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 16 Oct 2019 15:05:35 +0100 Subject: [PATCH 034/224] Update CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0b4281fa..e93fce6f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ * Add UCSC genomes to `igenomes.config` and add paths to all genome indices * Change `maxMultiqcEmailFileSize` parameter to `max_multiqc_email_size` * Export conda environment in Docker file [#349](https://github.com/nf-core/tools/issues/349) +* Change remaining parameters from `camelCase` to `snakeCase` [#39](https://github.com/nf-core/hic/issues/39) + * `--singleEnd` to `--single_end` + * `--igenomesIgnore` to `--igenomes_ignore` +* Add `autoMounts=true` to default singularity profile ### Other From 1e463c8e8dcd2aa09d70988cc6bc1fa1e5139cae Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 22 Oct 2019 13:06:52 +0200 Subject: [PATCH 035/224] Fix branch protection in TEMPLATE Fix part of #432 --- .../pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml index 31c5205eb..063f590d7 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml @@ -9,7 +9,7 @@ matrix: before_install: # PRs to master are only ok if coming from dev branch - - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && ([ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ] || [ $TRAVIS_PULL_REQUEST_BRANCH = "patch" ]))' + - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ]) || [ $TRAVIS_PULL_REQUEST_BRANCH = "patch" ]' # Pull the docker image first so the test doesn't wait for this - docker pull {{ cookiecutter.name_docker }}:dev # Fake the tag locally so that the pipeline runs properly From 8a35286076abe4eb18777c236f64ef9579eb1c95 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 13:32:18 +0100 Subject: [PATCH 036/224] Remove markdownlint checks --- .../{{cookiecutter.name_noslash}}/.github/markdownlint.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml index e052a635a..efa2c61d8 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml @@ -1,8 +1,6 @@ # Markdownlint configuration file default: true, line-length: false -no-multiple-blanks: 0 -blanks-around-headers: false blanks-around-lists: false header-increment: false no-duplicate-header: From 4614150c402db0a9dbd2257e4f0141460392ae53 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 13:34:26 +0100 Subject: [PATCH 037/224] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e93fce6f1..da1cd6f7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ * `--singleEnd` to `--single_end` * `--igenomesIgnore` to `--igenomes_ignore` * Add `autoMounts=true` to default singularity profile +* Add in `markdownlint` checks that were being ignored by default ### Other From 2538f2840586fd0b90c1740cca595fa72416bbab Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 14:19:39 +0100 Subject: [PATCH 038/224] Add lines between headers --- .../pipeline-template/{{cookiecutter.name_noslash}}/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CHANGELOG.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CHANGELOG.md index 334d08780..1b11643be 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CHANGELOG.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CHANGELOG.md @@ -1,4 +1,5 @@ # {{ cookiecutter.name }}: Changelog ## v{{ cookiecutter.version }} - [date] + Initial release of {{ cookiecutter.name }}, created with the [nf-core](http://nf-co.re/) template. From d273a6a78873dc1efd6f245240b2900eb81c8f05 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 14:19:47 +0100 Subject: [PATCH 039/224] Add lines between headers --- .../{{cookiecutter.name_noslash}}/docs/output.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/output.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/output.md index c9b9b9e4e..aff5a2693 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/output.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/output.md @@ -5,6 +5,7 @@ This document describes the output produced by the pipeline. Most of the plots a ## Pipeline overview + The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes data using the following steps: @@ -12,6 +13,7 @@ and processes data using the following steps: * [MultiQC](#multiqc) - aggregate report, describing results of the whole pipeline ## FastQC + [FastQC](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/) gives general quality metrics about your reads. It provides information about the quality score distribution across your reads, the per base sequence content (%T/A/G/C). You get information about adapter contamination and other overrepresented sequences. For further reading and documentation see the [FastQC help](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/). @@ -27,6 +29,7 @@ For further reading and documentation see the [FastQC help](http://www.bioinform ## MultiQC + [MultiQC](http://multiqc.info) is a visualisation tool that generates a single HTML report summarising all samples in your project. Most of the pipeline QC results are visualised in the report and further statistics are available in within the report data directory. The pipeline has special steps which allow the software versions used to be reported in the MultiQC output for future traceability. From 6141f3b5c16612ad130ca0208350361b66c09bd2 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 14:19:52 +0100 Subject: [PATCH 040/224] Add lines between headers --- .../docs/usage.md | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index 0ba7a1434..ce9470748 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -41,8 +41,8 @@ * [`--multiqc_config`](#--multiqc_config) - ## Introduction + Nextflow handles job submissions on SLURM or other environments, and supervises running the jobs. Thus the Nextflow process must run until the pipeline is finished. We recommend that you put the process running in the background through `screen` / `tmux` or similar tool. Alternatively you can run nextflow within a cluster job submitted your job scheduler. It is recommended to limit the Nextflow Java virtual machines memory. We recommend adding the following line to your environment (typically in `~/.bashrc` or `~./bash_profile`): @@ -54,6 +54,7 @@ NXF_OPTS='-Xms1g -Xmx4g' ## Running the pipeline + The typical command for running the pipeline is as follows: ```bash @@ -72,6 +73,7 @@ results # Finished results (configurable, see below) ``` ### Updating the pipeline + When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline: ```bash @@ -79,16 +81,17 @@ nextflow pull {{ cookiecutter.name }} ``` ### Reproducibility + It's a good idea to specify a pipeline version when running the pipeline on your data. This ensures that a specific version of the pipeline code and software are used when you run your pipeline. If you keep using the same tag, you'll be running the same version of the pipeline, even if there have been changes to the code since. First, go to the [{{ cookiecutter.name }} releases page](https://github.com/{{ cookiecutter.name }}/releases) and find the latest version number - numeric only (eg. `1.3.1`). Then specify this when running the pipeline with `-r` (one hyphen) - eg. `-r 1.3.1`. This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. - ## Main arguments ### `-profile` + Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. Note that multiple profiles can be loaded, for example: `-profile docker` - the order of arguments is important! If `-profile` is not specified at all the pipeline will be run locally and expects all software to be installed and available on the `PATH`. @@ -111,6 +114,7 @@ If `-profile` is not specified at all the pipeline will be run locally and expec ### `--reads` + Use this to specify the location of your input FastQ files. For example: ```bash @@ -126,6 +130,7 @@ Please note the following requirements: If left unspecified, a default pattern is used: `data/*{1,2}.fastq.gz` ### `--single_end` + By default, the pipeline expects paired-end data. If you have single-end data, you need to specify `--single_end` on the command line when you launch the pipeline. A normal glob pattern, enclosed in quotation marks, can then be used for `--reads`. For example: ```bash @@ -134,12 +139,12 @@ By default, the pipeline expects paired-end data. If you have single-end data, y It is not possible to run a mixture of single-end and paired-end files in one run. - ## Reference genomes The pipeline config files come bundled with paths to the illumina iGenomes reference index files. If running with docker or AWS, the configuration is set up to use the [AWS-iGenomes](https://ewels.github.io/AWS-iGenomes/) resource. ### `--genome` (using iGenomes) + There are 31 different species supported in the iGenomes references. To run the pipeline, you must specify which to use with the `--genome` flag. You can find the keys to specify the genomes in the [iGenomes config file](../conf/igenomes.config). Common genomes that are supported are: @@ -174,6 +179,7 @@ params { ### `--fasta` + If you prefer, you can specify the full path to your reference genome when you run the pipeline: ```bash @@ -181,13 +187,17 @@ If you prefer, you can specify the full path to your reference genome when you r ``` ### `--igenomes_ignore` + Do not load `igenomes.config` when running the pipeline. You may choose this option if you observe clashes between custom parameters and those supplied in `igenomes.config`. ## Job resources + ### Automatic resubmission + Each step in the pipeline has a default set of requirements for number of CPUs, memory and time. For most of the steps in the pipeline, if the job exits with an error code of `143` (exceeded requested resources) it will automatically resubmit with higher requests (2 x original, then 3 x original). If it still fails after three times then the pipeline is stopped. ### Custom resource requests + Wherever process-specific requirements are set in the pipeline, the default value can be changed by creating a custom config file. See the files hosted at [`nf-core/configs`](https://github.com/nf-core/configs/tree/master/conf) for examples. If you are likely to be running `nf-core` pipelines regularly it may be a good idea to request that your custom config file is uploaded to the `nf-core/configs` git repository. Before you do this please can you test that the config file works with your pipeline of choice using the `-c` parameter (see definition below). You can then create a pull request to the `nf-core/configs` repository with the addition of your config file, associated documentation file (see examples in [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs)), and amending [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) to include your custom profile. @@ -195,6 +205,7 @@ If you are likely to be running `nf-core` pipelines regularly it may be a good i If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack/). ## AWS Batch specific parameters + Running the pipeline on AWS Batch requires a couple of specific parameters to be set according to your AWS Batch configuration. Please use the `-awsbatch` profile and then specify all of the following parameters. ### `--awsqueue` The JobQueue that you intend to use on AWS Batch. @@ -208,18 +219,23 @@ Please make sure to also set the `-w/--work-dir` and `--outdir` parameters to a ### `--outdir` + The output directory where the results will be saved. ### `--email` + Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits. If set in your user config file (`~/.nextflow/config`) then you don't need to specify this on the command line for every run. ### `--email_on_fail` + This works exactly as with `--email`, except emails are only sent if the workflow is not successful. ### `--max_multiqc_email_size` + Theshold size for MultiQC report to be attached in notification email. If file generated by pipeline exceeds the threshold, it will not be attached (Default: 25MB). ### `-name` + Name for the pipeline run. If not specified, Nextflow will automatically generate a random mnemonic. This is used in the MultiQC report (if not default) and in the summary HTML / e-mail (always). @@ -227,6 +243,7 @@ This is used in the MultiQC report (if not default) and in the summary HTML / e- **NB:** Single hyphen (core Nextflow option) ### `-resume` + Specify this when restarting a pipeline. Nextflow will used cached results from any pipeline steps where the inputs are the same, continuing from where it got to previously. You can also supply a run name to resume a specific run: `-resume [run-name]`. Use the `nextflow log` command to show previous run names. @@ -234,6 +251,7 @@ You can also supply a run name to resume a specific run: `-resume [run-name]`. U **NB:** Single hyphen (core Nextflow option) ### `-c` + Specify the path to a specific config file (this is a core NextFlow command). **NB:** Single hyphen (core Nextflow option) @@ -241,6 +259,7 @@ Specify the path to a specific config file (this is a core NextFlow command). Note - you can use this to override pipeline defaults. ### `--custom_config_version` + Provide git commit id for custom Institutional configs hosted at `nf-core/configs`. This was implemented for reproducibility purposes. Default is set to `master`. ```bash @@ -249,6 +268,7 @@ Provide git commit id for custom Institutional configs hosted at `nf-core/config ``` ### `--custom_config_base` + If you're running offline, nextflow will not be able to fetch the institutional config files from the internet. If you don't need them, then this is not a problem. If you do need them, you should download the files from the repo and tell nextflow where to find them with the @@ -269,22 +289,28 @@ nextflow run /path/to/pipeline/ --custom_config_base /path/to/my/configs/configs > files + singularity containers + institutional configs in one go for you, to make this process easier. ### `--max_memory` + Use to set a top-limit for the default memory requirement for each process. Should be a string in the format integer-unit. eg. `--max_memory '8.GB'` ### `--max_time` + Use to set a top-limit for the default time requirement for each process. Should be a string in the format integer-unit. eg. `--max_time '2.h'` ### `--max_cpus` + Use to set a top-limit for the default CPU requirement for each process. Should be a string in the format integer-unit. eg. `--max_cpus 1` ### `--plaintext_email` + Set to receive plain-text e-mails instead of HTML formatted. ### `--monochrome_logs` + Set to disable colourful command line output and live life in monochrome. ### `--multiqc_config` + Specify a path to a custom MultiQC configuration file. From e80e1d3cc9f22affbd6f4b6282510804427c3722 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 14:25:00 +0100 Subject: [PATCH 041/224] Fix missed spacing --- .../{{cookiecutter.name_noslash}}/docs/output.md | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/output.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/output.md index aff5a2693..f6bfa82bf 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/output.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/output.md @@ -27,7 +27,6 @@ For further reading and documentation see the [FastQC help](http://www.bioinform * `zips/sample_fastqc.zip` * zip file containing the FastQC report, tab-delimited data file and plot images - ## MultiQC [MultiQC](http://multiqc.info) is a visualisation tool that generates a single HTML report summarising all samples in your project. Most of the pipeline QC results are visualised in the report and further statistics are available in within the report data directory. From 52668450cc1e1855dc93c978dfbaae9786cba083 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 14:25:11 +0100 Subject: [PATCH 042/224] Fix missed spacing --- .../{{cookiecutter.name_noslash}}/docs/usage.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index ce9470748..3c93032d0 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -207,9 +207,13 @@ If you have any questions or issues please send us a message on [Slack](https:// ## AWS Batch specific parameters Running the pipeline on AWS Batch requires a couple of specific parameters to be set according to your AWS Batch configuration. Please use the `-awsbatch` profile and then specify all of the following parameters. + ### `--awsqueue` + The JobQueue that you intend to use on AWS Batch. + ### `--awsregion` + The AWS region to run your job in. Default is set to `eu-west-1` but can be adjusted to your needs. Please make sure to also set the `-w/--work-dir` and `--outdir` parameters to a S3 storage bucket of your choice - you'll get an error message notifying you if you didn't. From c77458e6db8c277c19969f52e8a9976c864525ad Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 14:32:56 +0100 Subject: [PATCH 043/224] Remove additional entries from markdownlint file --- .../{{cookiecutter.name_noslash}}/.github/markdownlint.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml index efa2c61d8..96b12a703 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml @@ -1,7 +1,5 @@ # Markdownlint configuration file default: true, line-length: false -blanks-around-lists: false -header-increment: false no-duplicate-header: siblings_only: true From c40847433b9bd8c6f96b9fb5ac10e01d4fc1344b Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 14:45:53 +0100 Subject: [PATCH 044/224] Remove fields tools markdownlint file --- .github/markdownlint.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/markdownlint.yml b/.github/markdownlint.yml index 9e84d0e78..c6b3f58f0 100644 --- a/.github/markdownlint.yml +++ b/.github/markdownlint.yml @@ -1,10 +1,7 @@ # Markdownlint configuration file default: true, line-length: false -no-multiple-blanks: 0 -blanks-around-headers: false -blanks-around-lists: false -header-increment: false no-duplicate-header: siblings_only: true no-bare-urls: false # tools only - the {{ jinja variables }} break URLs and cause this to error +commands-show-output: false # tools only - suppresses error messages for usage of $ in main README From 6f6ea1a4e4b179cb530e50513471601e9d306754 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 15:34:04 +0100 Subject: [PATCH 045/224] Fix header spacing --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4403afcce..f3d81a59d 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,6 @@ A python package with helper tools for the nf-core community. * [`nf-core lint` - Check pipeline code against nf-core guidelines](#linting-a-workflow) * [`nf-core bump-version` - Update nf-core pipeline version number](#bumping-a-pipeline-version-number) - The nf-core tools package is written in Python and can be imported and used within other packages. For documentation of the internal Python functions, please refer to the [Tools Python API docs](https://nf-co.re/tools-docs/). @@ -56,6 +55,7 @@ pip install -e . ``` ## Listing pipelines + The command `nf-core list` shows all available nf-core pipelines along with their latest version, when that was published and how recently the pipeline code was pulled to your local system (if at all). An example of the output from the command is as follows: @@ -139,6 +139,7 @@ nf-core/mag 8 dev - - Finally, to return machine-readable JSON output, use the `--json` flag. ## Launch a pipeline + Some nextflow pipelines have a considerable number of command line flags that can be used. To help with this, the `nf-core launch` command uses an interactive command-line wizard tool to prompt you for values for running nextflow and the pipeline parameters. @@ -202,8 +203,8 @@ Launching `nf-core/rnaseq` [evil_engelbart] - revision: 37f260d360 [master] [..truncated..] ``` - ## Downloading pipelines for offline use + Sometimes you may need to run an nf-core pipeline on a server or HPC system that has no internet connection. In this case you will need to fetch the pipeline files first, then manually transfer them to your system. To make this process easier and ensure accurate retrieval of correctly versioned code and software containers, we have written a download helper tool. Simply specify the name of the nf-core pipeline and it will be downloaded to your current working directory. @@ -255,6 +256,7 @@ nf-core-methylseq-1.0/ ``` ## Pipeline software licences + Sometimes it's useful to see the software licences of the tools used in a pipeline. You can use the `licences` subcommand to fetch and print the software licence from each conda / PyPI package used in an nf-core pipeline. ```console @@ -292,6 +294,7 @@ samtools 1.8 MIT ``` ## Creating a new workflow + The `create` subcommand makes a new workflow using the nf-core base template. With a given pipeline name, description and author, it makes a starter pipeline which follows nf-core best practices. @@ -335,8 +338,8 @@ Please see the [nf-core documentation](https://nf-co.re/developers/adding_pipeli Note that if the required arguments for `nf-core create` are not given, it will interactively prompt for them. If you prefer, you can supply them as command line arguments. See `nf-core create --help` for more information. - ## Linting a workflow + The `lint` subcommand checks a given pipeline for all nf-core community guidelines. This is the same test that is used on the automated continuous integration tests. @@ -366,7 +369,6 @@ WARNING: Test Warnings: You can find extensive documentation about each of the lint tests in the [lint errors documentation](https://nf-co.re/errors). - ## Bumping a pipeline version number When releasing a new version of a nf-core pipeline, version numbers have to be updated in several different places. The helper command `nf-core bump-version` automates this for you to avoid manual errors (and frustration!). From 01192676f4c74f8725f9b9cddfc69e1257134974 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 15:34:10 +0100 Subject: [PATCH 046/224] Fix header spacing --- docs/lint_errors.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/lint_errors.md b/docs/lint_errors.md index 9d80f54b1..141c5c2c5 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -3,6 +3,7 @@ This page contains detailed descriptions of the tests done by the [nf-core/tools](https://github.com/nf-core/tools) package. Linting errors should show URLs next to any failures that link to the relevant heading below. ## Error #1 - File not found ## {#1} + nf-core pipelines should adhere to a common file structure for consistency. The lint test looks for the following required files: * `nextflow.config` @@ -27,13 +28,14 @@ The following files are suggested but not a hard requirement. If they are missin * `conf/base.config` * A `conf` directory with at least one config called `base.config` - ## Error #2 - Docker file check failed ## {#2} + Pipelines should have a files called `Dockerfile` in their root directory. The file is used for automated docker image builds. This test checks that the file exists and contains at least the string `FROM` (`Dockerfile`). ## Error #3 - Licence check failed ## {#3} + nf-core pipelines must ship with an open source [MIT licence](https://choosealicense.com/licenses/mit/). This test fails if the following conditions are not met: @@ -46,6 +48,7 @@ This test fails if the following conditions are not met: * `[year]`, `[fullname]`, ``, ``, `` or `` ## Error #4 - Nextflow config check failed ## {#4} + nf-core pipelines are required to be configured with a minimal set of variable names. This test fails or throws warnings if required variables are not set. @@ -104,6 +107,7 @@ The following variables are depreciated and fail the test if they are still pres * The old method for specifying the dockerhub container address. Replaced by `process.container` ## Error #5 - Continuous Integration configuration ## {#5} + nf-core pipelines must have CI testing with Travis or Circle CI. This test fails if the following happens: @@ -131,6 +135,7 @@ This test fails if the following happens: ``` ## Error #6 - Repository `README.md` tests ## {#6} + The `README.md` files for a project are very important and must meet some requirements: * Nextflow badge From f0060a6b1082cd8cf3dc408de411cf495b756804 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 15:39:06 +0100 Subject: [PATCH 047/224] Fix issue 434 --- .../{{cookiecutter.name_noslash}}/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md index df92c7f0d..3cdbc366e 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md @@ -21,14 +21,16 @@ ii. Install one of [`docker`](https://docs.docker.com/engine/installation/), [`s iii. Download the pipeline and test it on a minimal dataset with a single command ```bash -nextflow run {{ cookiecutter.name }} -profile test, +nextflow run {{ cookiecutter.name }} -profile test, ``` +> Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile institute` in the command above and this will enable either `docker` or `singularity`, and set the appropriate execution settings for your local cluster. + iv. Start running your own analysis! ```bash -nextflow run {{ cookiecutter.name }} -profile --reads '*_R{1,2}.fastq.gz' --genome GRCh37 +nextflow run {{ cookiecutter.name }} -profile --reads '*_R{1,2}.fastq.gz' --genome GRCh37 ``` See [usage docs](docs/usage.md) for all of the available options when running the pipeline. From 9d52a8970c8369eb0a22ff7d3a6e5f14d0ce3cf5 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 15:43:54 +0100 Subject: [PATCH 048/224] Change wording --- .../pipeline-template/{{cookiecutter.name_noslash}}/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md index 3cdbc366e..03526397e 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md @@ -24,7 +24,7 @@ iii. Download the pipeline and test it on a minimal dataset with a single comman nextflow run {{ cookiecutter.name }} -profile test, ``` -> Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile institute` in the command above and this will enable either `docker` or `singularity`, and set the appropriate execution settings for your local cluster. +> Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile institute` in the command above and this will enable either `docker` or `singularity`, and set the appropriate execution settings for your local compute environment. iv. Start running your own analysis! From 7639e89bd529f32fbcc754fcb6d8f9a155ded616 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 15:48:34 +0100 Subject: [PATCH 049/224] Update indents --- .../{{cookiecutter.name_noslash}}/main.nf | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index 12aa4b1c7..0c7edf721 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -174,9 +174,9 @@ ${summary.collect { k,v -> "
$k
${v ?: ' - if (filename.indexOf(".csv") > 0) filename - else null - } + if (filename.indexOf(".csv") > 0) filename + else null + } output: file 'software_versions_mqc.yaml' into ch_software_versions_yaml @@ -200,7 +200,9 @@ process fastqc { tag "$name" label 'process_medium' publishDir "${params.outdir}/fastqc", mode: 'copy', - saveAs: { filename -> filename.indexOf(".zip") > 0 ? "zips/$filename" : "$filename" } + saveAs: { filename -> + filename.indexOf(".zip") > 0 ? "zips/$filename" : "$filename" + } input: set val(name), file(reads) from ch_read_files_fastqc From aaef6f52ba5909079d3212c305d0b996f2caa6e9 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 16:08:12 +0100 Subject: [PATCH 050/224] Try removing duplicate header restriction --- .github/markdownlint.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/markdownlint.yml b/.github/markdownlint.yml index c6b3f58f0..a4392c189 100644 --- a/.github/markdownlint.yml +++ b/.github/markdownlint.yml @@ -1,7 +1,5 @@ # Markdownlint configuration file default: true, line-length: false -no-duplicate-header: - siblings_only: true no-bare-urls: false # tools only - the {{ jinja variables }} break URLs and cause this to error commands-show-output: false # tools only - suppresses error messages for usage of $ in main README From def6fbe2ea2b8af4fd545294b76b4034956f9557 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 16:08:49 +0100 Subject: [PATCH 051/224] Try removing duplicate header restriction --- .../{{cookiecutter.name_noslash}}/.github/markdownlint.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml index 96b12a703..737d9f234 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml @@ -1,5 +1,3 @@ # Markdownlint configuration file default: true, line-length: false -no-duplicate-header: - siblings_only: true From 8c93327734f018cb32c39edaa8d62f3bbfceb62e Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 16:11:24 +0100 Subject: [PATCH 052/224] Reword based on @ewels comment --- .../pipeline-template/{{cookiecutter.name_noslash}}/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md index 03526397e..51ea8ab3e 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md @@ -24,7 +24,7 @@ iii. Download the pipeline and test it on a minimal dataset with a single comman nextflow run {{ cookiecutter.name }} -profile test, ``` -> Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile institute` in the command above and this will enable either `docker` or `singularity`, and set the appropriate execution settings for your local compute environment. +> Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile institute` in your command. This will enable either `docker` or `singularity` and set the appropriate execution settings for your local compute environment. iv. Start running your own analysis! From 04f4577455649685cdafc08753eac70f1f47f35c Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 16:14:34 +0100 Subject: [PATCH 053/224] Add duplicate header restriction back in for CHANGELOG --- .github/markdownlint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/markdownlint.yml b/.github/markdownlint.yml index a4392c189..c6b3f58f0 100644 --- a/.github/markdownlint.yml +++ b/.github/markdownlint.yml @@ -1,5 +1,7 @@ # Markdownlint configuration file default: true, line-length: false +no-duplicate-header: + siblings_only: true no-bare-urls: false # tools only - the {{ jinja variables }} break URLs and cause this to error commands-show-output: false # tools only - suppresses error messages for usage of $ in main README From 1e75a345f2008774525d57e331cdb0aac28bd216 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 16:14:43 +0100 Subject: [PATCH 054/224] Add duplicate header restriction back in for CHANGELOG --- .../{{cookiecutter.name_noslash}}/.github/markdownlint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml index 737d9f234..96b12a703 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/markdownlint.yml @@ -1,3 +1,5 @@ # Markdownlint configuration file default: true, line-length: false +no-duplicate-header: + siblings_only: true From e29227e93120013b56073dff0b00beab69143eb2 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 25 Oct 2019 16:36:08 +0100 Subject: [PATCH 055/224] Fix tpyo --- .../{{cookiecutter.name_noslash}}/docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index 3c93032d0..11727fffe 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -236,7 +236,7 @@ This works exactly as with `--email`, except emails are only sent if the workflo ### `--max_multiqc_email_size` -Theshold size for MultiQC report to be attached in notification email. If file generated by pipeline exceeds the threshold, it will not be attached (Default: 25MB). +Threshold size for MultiQC report to be attached in notification email. If file generated by pipeline exceeds the threshold, it will not be attached (Default: 25MB). ### `-name` From 71110dfebb8843d18318fc8d9edcc7baf6e911d4 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Oct 2019 10:44:09 +0100 Subject: [PATCH 056/224] Linting: single_end and igenomes_ignore now must be snake case --- CHANGELOG.md | 1 + docs/lint_errors.md | 6 ++++-- nf_core/lint.py | 10 ++++++---- .../minimal_working_example/nextflow.config | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da1cd6f7f..a21cd8a34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * Change remaining parameters from `camelCase` to `snakeCase` [#39](https://github.com/nf-core/hic/issues/39) * `--singleEnd` to `--single_end` * `--igenomesIgnore` to `--igenomes_ignore` + * Having the old camelCase versions of these will now throw an error * Add `autoMounts=true` to default singularity profile * Add in `markdownlint` checks that were being ignored by default diff --git a/docs/lint_errors.md b/docs/lint_errors.md index 141c5c2c5..eb87630e8 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -93,9 +93,9 @@ The following variables throw warnings if missing: * If the pipeline version number contains the string `dev`, the dockerhub tag must be `:dev` * `params.reads` * Input parameter to specify input data (typically FastQ files / pairs) -* `params.singleEnd` +* `params.single_end` * Specify to work with single-end sequence data instead of default paired-end - * Used with Nextflow: `.fromFilePairs( params.reads, size: params.singleEnd ? 1 : 2 )` + * Used with Nextflow: `.fromFilePairs( params.reads, size: params.single_end ? 1 : 2 )` The following variables are depreciated and fail the test if they are still present: @@ -105,6 +105,8 @@ The following variables are depreciated and fail the test if they are still pres * The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` * `params.container` * The old method for specifying the dockerhub container address. Replaced by `process.container` +* `singleEnd` and `igenomesIgnore` + * Now using `snake_case` for all command line options ## Error #5 - Continuous Integration configuration ## {#5} diff --git a/nf_core/lint.py b/nf_core/lint.py index 5b65e5688..3b8cb09f8 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -112,11 +112,11 @@ class PipelineLint(object): params.help = false params.outdir = './results' params.bam = false - params.singleEnd = false + params.single_end = false params.seqtype = 'dna' params.solver = 'glpk' params.igenomes_base = './iGenomes' - params.clusterOptions = false + params.cluster_options = false ... """ def __init__(self, path): @@ -351,13 +351,15 @@ def check_nextflow_config(self): 'dag.file', 'params.reads', 'process.container', - 'params.singleEnd' + 'params.single_end' ] # Old depreciated vars - fail if present config_fail_ifdefined = [ 'params.version', 'params.nf_required_version', - 'params.container' + 'params.container', + 'params.singleEnd', + 'params.igenomesIgnore' ] # Get the nextflow config for this pipeline diff --git a/tests/lint_examples/minimal_working_example/nextflow.config b/tests/lint_examples/minimal_working_example/nextflow.config index b0f3a01f5..b942c502b 100644 --- a/tests/lint_examples/minimal_working_example/nextflow.config +++ b/tests/lint_examples/minimal_working_example/nextflow.config @@ -2,7 +2,7 @@ params { outdir = './results' reads = "data/*.fastq" - singleEnd = false + single_end = false custom_config_version = 'master' custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" } From 6975ad27f230ddfe5dae81270c589a5fa7058dbf Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Oct 2019 11:31:55 +0100 Subject: [PATCH 057/224] Revert clusterOptions --- nf_core/lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index 3b8cb09f8..f7c6ff44a 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -116,7 +116,7 @@ class PipelineLint(object): params.seqtype = 'dna' params.solver = 'glpk' params.igenomes_base = './iGenomes' - params.cluster_options = false + params.clusterOptions = false ... """ def __init__(self, path): From ae3f1a84baeb7334d9f31217612120aca4f81d7e Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Oct 2019 11:45:23 +0100 Subject: [PATCH 058/224] Template: -ansi-log false in travis --- CHANGELOG.md | 1 + .../pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da1cd6f7f..d131dcaf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ * `--igenomesIgnore` to `--igenomes_ignore` * Add `autoMounts=true` to default singularity profile * Add in `markdownlint` checks that were being ignored by default +* Disable ansi logging in the travis CI tests. ### Other diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml index 31c5205eb..db1995103 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml @@ -39,4 +39,4 @@ script: # Lint the documentation - markdownlint ${TRAVIS_BUILD_DIR} -c ${TRAVIS_BUILD_DIR}/.github/markdownlint.yml # Run the pipeline with the test profile - - nextflow run ${TRAVIS_BUILD_DIR} -profile test,docker + - nextflow run ${TRAVIS_BUILD_DIR} -profile test,docker -ansi-log false From 3abd86dcab788961975356662a0f2ad82c683665 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 9 Oct 2019 17:14:12 +0200 Subject: [PATCH 059/224] nf-core sync: start work on refactoring / rewriting sync code --- nf_core/sync.py | 263 ++++++++++++++++++++++++++++++++++++++++++++++++ scripts/nf-core | 41 +++++++- 2 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 nf_core/sync.py diff --git a/nf_core/sync.py b/nf_core/sync.py new file mode 100644 index 000000000..2097f92da --- /dev/null +++ b/nf_core/sync.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python +"""Bumps the version number in all appropriate files for +a nf-core pipeline. +""" + +import git +import json +import logging +import nf_core +import os +import requests +import shutil +import sys +import tempfile + +class PipelineSync(object): + """Object to hold syncing information and results. + + Args: + pipeline_dir (str): The path to the Nextflow pipeline root directory + make_template_branch (bool): Set this to `True` to create a `TEMPLATE` branch if it is not found + make_pr (bool): Set this to `True` to create a GitHub pull-request with the changes + + Attributes: + gh_base_url (str): The GitHub base url or the nf-core project + github_pr_url_templ (str): The API endpoint for creating pull requests + path_parent_dir (str): Current script directory + sync_errors (list): List of errors from sync + pr_errors (list): List of errors from PR + """ + + def __init__(self, pipeline_dir, make_template_branch=False, from_branch=None, make_pr=False): + """ Initialise syncing object """ + + self.pipeline_dir = os.path.abspath(pipeline_dir) + self.make_template_branch = make_template_branch + self.from_branch = from_branch + self.make_pr = make_pr + + self.gh_base_url = "https://{token}@github.com/nf-core/{pipeline}" + self.github_pr_url_templ = "https://api.github.com/repos/nf-core/{pipeline}/pulls" + + self.sync_error = False + self.pr_error = False + + def sync(self): + """ Find workflow attributes, create a new template pipeline on TEMPLATE + """ + logging.info("Pipeline directory: {}".format(self.pipeline_dir)) + # Check that the pipeline_dir is a git repo + try: + self.repo = git.Repo(self.pipeline_dir) + except git.exc.InvalidGitRepositoryError as e: + self.sync_error = True + logging.error("'{}' does not appear to be a git repository".format(self.pipeline_dir)) + return False + + + # get current branch so we can switch back later + self.original_branch = self.repo.active_branch.name + logging.debug("Original pipeline repository branch is '{}'".format(self.original_branch)) + + # Check to see if there are uncommitted changes on current branch + if self.repo.is_dirty(untracked_files=True): + self.sync_error = True + logging.error("Uncommitted changes found in pipeline directory!\nPlease commit these before running nf-core sync") + return False + + # Try to check out target branch (eg. `origin/dev`) + try: + if self.from_branch and self.repo.active_branch.name != self.from_branch: + logging.info("Checking out workflow branch '{}'".format(self.from_branch)) + self.repo.git.checkout(self.from_branch) + except git.exc.GitCommandError: + self.sync_error = True + logging.error("Branch `{}` not found!".format(self.from_branch)) + return False + + # Fetch workflow variables + logging.info("Fetching workflow config variables") + self.wf_config = nf_core.utils.fetch_wf_config(self.pipeline_dir) + + # Try to check out the `TEMPLATE` branch + try: + self.repo.git.checkout("origin/TEMPLATE", b="TEMPLATE") + except git.exc.GitCommandError: + + # Try to check out an existing local branch called TEMPLATE + try: + self.repo.git.checkout("TEMPLATE") + except git.exc.GitCommandError: + + # Failed, if we're not making a new branch just die + if not self.make_template_branch: + self.sync_error = True + logging.error("Could not check out branch 'origin/TEMPLATE'") + return False + + # Branch and force is set, fire function to create `TEMPLATE` branch + else: + logging.debug("Could not check out origin/TEMPLATE!") + logging.info("Creating orphan TEMPLATE branch") + try: + self.repo.git.checkout('--orphan', 'TEMPLATE') + except git.exc.GitCommandError as e: + self.sync_error = True + logging.error("Could not create 'TEMPLATE' branch:\n{}".format(e)) + return False + + # Delete everything + logging.info("Deleting all files in TEMPLATE branch") + for the_file in os.listdir(self.pipeline_dir): + if the_file == '.git': + continue + file_path = os.path.join(self.pipeline_dir, the_file) + logging.debug("Deleting {}".format(file_path)) + try: + if os.path.isfile(file_path): + os.unlink(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + except Exception as e: + self.sync_error = True + logging.error(e) + return False + + # Make a new pipeline using nf_core.create + + # Make a pull request if we've been asked to + + # Reset: Check out original branch again + + + +# def update_template_branch(self): +# """ Check out `TEMPLATE` and make a new pipeline using nf_core.create """ +# +# try: +# syncutils.template.NfcoreTemplate( +# pipeline['name'], +# branch=DEF_TEMPLATE_BRANCH, +# repo_url=GH_BASE_URL.format(token=os.environ["NF_CORE_BOT"], pipeline=pipeline['name']) +# ).sync() +# except Exception as e: +# sync_errors.append((pipeline['name'], e)) +# +# def create_pull_request(self): +# name = pipeline.get('name') +# for errored_pipeline, _ in sync_errors: +# if name == errored_pipeline: +# return +# response = create_pullrequest(name, token=os.environ["NF_CORE_BOT"]) +# if response.status_code != 201: +# pr_errors.append((name, response.status_code, response.content)) +# else: +# print("Created pull-request for pipeline \'{pipeline}\' successfully." +# .format(pipeline=name)) +# +# +# +# +# +# +#def create_pullrequest(pipeline, origin="dev", template="TEMPLATE", token="", user="nf-core"): +# """Create a pull request to a base branch (default: dev), +# from a head branch (default: TEMPLATE) +# +# Returns: An instance of class requests.Response +# """ +# content = {} +# content['title'] = "Important pipeline nf-core update! (version {tag})".format(tag=os.environ['TRAVIS_TAG']) +# content['body'] = "Some important changes have been made in the nf-core pipelines templates.\n" \ +# "Please make sure to merge this in ASAP and make a new minor release of your pipeline.\n\n" \ +# "Follow the link [nf-core/tools](https://github.com/nf-core/tools/releases/tag/{})".format(os.environ['TRAVIS_TAG']) +# content['head'] = "{}".format(template) +# content['base'] = origin +# return requests.post(url=GITHUB_PR_URL_TEMPL.format(pipeline=pipeline), +# data=json.dumps(content), +# auth=requests.auth.TTPBasicAuth(user, token)) +# +# +#def filter_blacklisted_pipelines_from_list(pipelines, blacklisted_pipelines): +# filtered_pipelines = [] +# for pipeline in pipelines: +# if not pipeline.get('name'): +# print("No attribute \'name\' for pipeline found: {}".format(pipeline)) +# else: +# filtered_pipelines.append(pipeline) if pipeline.get('name') not in blacklisted_pipelines \ +# else filtered_pipelines +# return filtered_pipelines +# +# +#def fetch_black_listed_pipelines_from_file(file_path): +# with open(file_path) as fh: +# blacklist = json.load(fh) +# return blacklist.get('pipelines') +# +# +#def fetch_nfcore_workflows_from_website(url): +# try: +# res = requests.get(url) +# pipelines = res.json().get('remote_workflows') +# except Exception as e: +# print("Could not get remote workflows. Reason was: {}".format(e)) +# pipelines = [] +# return pipelines +# +# +#def update_template_branch_for_pipeline(pipeline): +# try: +# syncutils.template.NfcoreTemplate( +# pipeline['name'], +# branch=DEF_TEMPLATE_BRANCH, +# repo_url=GH_BASE_URL.format(token=os.environ["NF_CORE_BOT"], pipeline=pipeline['name']) +# ).sync() +# except Exception as e: +# sync_errors.append((pipeline['name'], e)) +# +# +#def create_pullrequest_if_update_sucessful(pipeline): +# name = pipeline.get('name') +# for errored_pipeline, _ in sync_errors: +# if name == errored_pipeline: +# return +# response = create_pullrequest(name, token=os.environ["NF_CORE_BOT"]) +# if response.status_code != 201: +# pr_errors.append((name, response.status_code, response.content)) +# else: +# print("Created pull-request for pipeline \'{pipeline}\' successfully." +# .format(pipeline=name)) +# +# +#def main(): +# assert os.environ['TRAVIS_TAG'] +# assert os.environ['NF_CORE_BOT'] +# +# blacklisted_pipeline_names = fetch_black_listed_pipelines_from_file(PATH_PARENT_DIR + "/blacklist.json") +# +# pipelines = fetch_nfcore_workflows_from_website(NF_CORE_PIPELINE_INFO) +# +# if len(sys.argv) > 1: +# pipeline_to_sync = sys.argv[1] +# filtered_pipelines = [pipeline for pipeline in pipelines if pipeline_to_sync in pipeline.get('name')] +# else: +# filtered_pipelines = filter_blacklisted_pipelines_from_list(pipelines, blacklisted_pipeline_names) +# +# for pipeline in filtered_pipelines: +# print("Update template branch for pipeline '{pipeline}'... ".format(pipeline=pipeline['name'])) +# update_template_branch_for_pipeline(pipeline) +# print("Trying to open pull request for pipeline {}...".format(pipeline['name'])) +# create_pullrequest_if_update_sucessful(pipeline) +# +# for pipeline, exception in sync_errors: +# print("WARNING!!!! Sync for pipeline {name} failed.".format(name=pipeline)) +# print(exception) +# +# for pipeline, return_code, content in pr_errors: +# print("WARNING!!!! Pull-request for pipeline \'{pipeline}\' failed," +# " got return code {return_code}." +# .format(pipeline=pipeline, return_code=return_code)) +# print(content) +# +# sys.exit(0) diff --git a/scripts/nf-core b/scripts/nf-core index 9202c456d..bc8cd04de 100755 --- a/scripts/nf-core +++ b/scripts/nf-core @@ -9,7 +9,14 @@ import os import re import nf_core -import nf_core.bump_version, nf_core.create, nf_core.download, nf_core.launch, nf_core.licences, nf_core.lint, nf_core.list +import nf_core.bump_version +import nf_core.create +import nf_core.download +import nf_core.launch +import nf_core.licences +import nf_core.lint +import nf_core.list +import nf_core.sync import logging @@ -269,6 +276,38 @@ def bump_version(pipeline_dir, new_version, nextflow): nf_core.bump_version.bump_nextflow_version(lint_obj, new_version) +@nf_core_cli.command('sync', help_priority=8) +@click.argument( + 'pipeline_dir', + type = click.Path(exists=True), + required = True, + metavar = "" +) +@click.option( + '-t', '--make-template-branch', + is_flag = True, + default = False, + help = "Create a TEMPLATE branch if none is found." +) +@click.option( + '-b', '--from-branch', + type = str, + help = 'The pipeline git branch to use to get workflow vars' +) +@click.option( + '-p', '--pull-request', + is_flag = True, + default = False, + help = "Make a GitHub pull-request with the changes" +) +def sync(pipeline_dir, make_template_branch, from_branch, pull_request): + """ Sync a pipeline TEMPLATE branch with the nf-core template""" + sync_obj = nf_core.sync.PipelineSync(pipeline_dir, make_template_branch, from_branch, pull_request) + sync_obj.sync() + if sync_obj.sync_error or sync_obj.pr_error: + sys.exit(1) + + if __name__ == '__main__': click.echo(click.style("\n ,--.", fg='green')+click.style("/",fg='black')+click.style(",-.", fg='green')) From d9a1adf605638ffa5960510ecdcf25e77838ae9e Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 9 Oct 2019 17:20:09 +0200 Subject: [PATCH 060/224] Check required config vars and make a pipeline --- nf_core/sync.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/nf_core/sync.py b/nf_core/sync.py index 2097f92da..0b9eb35ef 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -80,6 +80,14 @@ def sync(self): logging.info("Fetching workflow config variables") self.wf_config = nf_core.utils.fetch_wf_config(self.pipeline_dir) + # Check that we have the required variables + for rvar in ['manifest.name','manifest.description','manifest.version','manifest.author']: + if rvar not in self.wf_config: + self.sync_error = True + logging.error("Workflow config variable `{}` not found!".format(rvar)) + return False + + # Try to check out the `TEMPLATE` branch try: self.repo.git.checkout("origin/TEMPLATE", b="TEMPLATE") @@ -125,6 +133,16 @@ def sync(self): return False # Make a new pipeline using nf_core.create + logging.info("Making a new template pipeline using pipeline variables") + nf_core.create.PipelineCreate( + name = self.wf_config['manifest.name'], + description = self.wf_config['manifest.description'], + new_version = self.wf_config['manifest.version'], + no_git = True, + force = True, + outdir = self.pipeline_dir, + author = self.wf_config['manifest.author'], + ).init_pipeline() # Make a pull request if we've been asked to From 69acd4692d4535e16ff32a8bc6fc40e702d3b838 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 11 Oct 2019 10:14:27 +0200 Subject: [PATCH 061/224] Add a bit more of the functionality - untested --- nf_core/sync.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 0b9eb35ef..1780fb10b 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -19,6 +19,7 @@ class PipelineSync(object): Args: pipeline_dir (str): The path to the Nextflow pipeline root directory make_template_branch (bool): Set this to `True` to create a `TEMPLATE` branch if it is not found + from_branch (str): The branch to use to fetch config vars. If not set, will use current active branch make_pr (bool): Set this to `True` to create a GitHub pull-request with the changes Attributes: @@ -35,6 +36,7 @@ def __init__(self, pipeline_dir, make_template_branch=False, from_branch=None, m self.pipeline_dir = os.path.abspath(pipeline_dir) self.make_template_branch = make_template_branch self.from_branch = from_branch + self.orphan_branch = False self.make_pr = make_pr self.gh_base_url = "https://{token}@github.com/nf-core/{pipeline}" @@ -46,7 +48,9 @@ def __init__(self, pipeline_dir, make_template_branch=False, from_branch=None, m def sync(self): """ Find workflow attributes, create a new template pipeline on TEMPLATE """ + logging.info("Pipeline directory: {}".format(self.pipeline_dir)) + # Check that the pipeline_dir is a git repo try: self.repo = git.Repo(self.pipeline_dir) @@ -55,6 +59,15 @@ def sync(self): logging.error("'{}' does not appear to be a git repository".format(self.pipeline_dir)) return False + # If we've been asked to make a PR, check that we have the credentials + if self.make_pr: + try: + assert length(str(os.environ['NF_CORE_BOT'])) > 5 + except (IndexError, AssertionError) as e: + self.sync_error = True + logging.error("Environment variable `$NF_CORE_BOT` is not set - cannot make PR") + return False + # get current branch so we can switch back later self.original_branch = self.repo.active_branch.name @@ -110,6 +123,7 @@ def sync(self): logging.info("Creating orphan TEMPLATE branch") try: self.repo.git.checkout('--orphan', 'TEMPLATE') + self.orphan_branch = True except git.exc.GitCommandError as e: self.sync_error = True logging.error("Could not create 'TEMPLATE' branch:\n{}".format(e)) @@ -144,9 +158,43 @@ def sync(self): author = self.wf_config['manifest.author'], ).init_pipeline() - # Make a pull request if we've been asked to + # Commit changes if we have any + if not self.repo.is_dirty(untracked_files=True): + logging.info("Template contains no changes - no new commit created") + else: + try: + self.repo.git.add(A=True) + self.repo.index.commit("Template update for nf-core/tools version {}".format(nf_core.__version__)) + except Exception as e: + self.sync_error = True + logging.error("Could not commit changes to TEMPLATE:\n{}".format(e)) + return False + + # Push and make a pull request if we've been asked to + if self.make_pr: + self.repo.git.push() + # + # TODO - MAKE PR + # + # + # + # # Reset: Check out original branch again + logging.debug("Checking out original branch: '{}'".format(self.original_branch)) + try: + self.repo.git.checkout(self.original_branch) + except git.exc.GitCommandError as e: + self.sync_error = True + logging.error("Could not reset to original branch `{}`:\n{}".format(self.from_branch, e)) + return False + + # Finish up + if not self.make_pr: + git_merge_cmd = 'git merge TEMPLATE' + if self.orphan_branch: + git_merge_cmd += ' --allow-unrelated-histories' + logging.info("Now try to merge the updates in to your pipeline:\n {}".format(git_merge_cmd)) From 2cf6ec5b20999451aa78430ae8b1cb193dc56df1 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 12 Oct 2019 00:29:36 +0200 Subject: [PATCH 062/224] Sync: push changes, get PR variables --- nf_core/sync.py | 78 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 26 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 1780fb10b..fa762de55 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -8,6 +8,7 @@ import logging import nf_core import os +import re import requests import shutil import sys @@ -37,10 +38,13 @@ def __init__(self, pipeline_dir, make_template_branch=False, from_branch=None, m self.make_template_branch = make_template_branch self.from_branch = from_branch self.orphan_branch = False + self.made_changes = False self.make_pr = make_pr - self.gh_base_url = "https://{token}@github.com/nf-core/{pipeline}" - self.github_pr_url_templ = "https://api.github.com/repos/nf-core/{pipeline}/pulls" + self.gh_username = None + self.gh_repo = None + self.gh_base_url = "https://{token}@github.com/{username}/{pipeline}" + self.github_pr_url_templ = "https://api.github.com/repos/{username}/{pipeline}/pulls" self.sync_error = False self.pr_error = False @@ -59,16 +63,6 @@ def sync(self): logging.error("'{}' does not appear to be a git repository".format(self.pipeline_dir)) return False - # If we've been asked to make a PR, check that we have the credentials - if self.make_pr: - try: - assert length(str(os.environ['NF_CORE_BOT'])) > 5 - except (IndexError, AssertionError) as e: - self.sync_error = True - logging.error("Environment variable `$NF_CORE_BOT` is not set - cannot make PR") - return False - - # get current branch so we can switch back later self.original_branch = self.repo.active_branch.name logging.debug("Original pipeline repository branch is '{}'".format(self.original_branch)) @@ -115,6 +109,7 @@ def sync(self): if not self.make_template_branch: self.sync_error = True logging.error("Could not check out branch 'origin/TEMPLATE'") + logging.info("Use flag --make-template-branch to attempt to create this branch") return False # Branch and force is set, fire function to create `TEMPLATE` branch @@ -124,6 +119,9 @@ def sync(self): try: self.repo.git.checkout('--orphan', 'TEMPLATE') self.orphan_branch = True + if self.make_pr: + self.make_pr = False + logging.info("Will not attempt to make a PR - orphan branch must be merged manually first") except git.exc.GitCommandError as e: self.sync_error = True logging.error("Could not create 'TEMPLATE' branch:\n{}".format(e)) @@ -149,13 +147,13 @@ def sync(self): # Make a new pipeline using nf_core.create logging.info("Making a new template pipeline using pipeline variables") nf_core.create.PipelineCreate( - name = self.wf_config['manifest.name'], - description = self.wf_config['manifest.description'], - new_version = self.wf_config['manifest.version'], + name = self.wf_config['manifest.name'].strip('\"').strip("\'"), + description = self.wf_config['manifest.description'].strip('\"').strip("\'"), + new_version = self.wf_config['manifest.version'].strip('\"').strip("\'"), no_git = True, force = True, outdir = self.pipeline_dir, - author = self.wf_config['manifest.author'], + author = self.wf_config['manifest.author'].strip('\"').strip("\'"), ).init_pipeline() # Commit changes if we have any @@ -165,20 +163,48 @@ def sync(self): try: self.repo.git.add(A=True) self.repo.index.commit("Template update for nf-core/tools version {}".format(nf_core.__version__)) + self.made_changes = True + logging.info("Committed changes to TEMPLATE branch") except Exception as e: self.sync_error = True logging.error("Could not commit changes to TEMPLATE:\n{}".format(e)) return False # Push and make a pull request if we've been asked to - if self.make_pr: - self.repo.git.push() - # - # TODO - MAKE PR - # - # - # - # + if self.make_pr and self.made_changes: + logging.info("Pushing TEMPLATE branch to remote") + try: + self.repo.git.push() + except git.exc.GitCommandError as e: + if self.make_template_branch: + try: + self.repo.git.push('--set-upstream', 'origin', 'TEMPLATE') + except git.exc.GitCommandError as e: + logging.error("Could not push TEMPLATE branch:\n {}".format(e)) + self.pr_error = True + else: + logging.error("Could not push TEMPLATE branch:\n {}".format(e)) + self.pr_error = True + + # Figure out the GitHub username to make a PR if we can + gh_ssh_username_match = re.search(r'git@github\.com:([^\/]+)/([^\/]+)\.git$', self.repo.remotes.origin.url) + if gh_ssh_username_match: + self.gh_username = gh_ssh_username_match.group(1) + self.gh_repo = gh_ssh_username_match.group(2) + gh_url_username_match = re.search(r'https://github\.com/([^\/]+)/([^\/]+)\.git$', self.repo.remotes.origin.url) + if gh_url_username_match: + self.gh_username = gh_url_username_match.group(1) + self.gh_repo = gh_url_username_match.group(2) + + # If we've been asked to make a PR, check that we have the credentials + try: + assert len(str(os.environ['NF_CORE_BOT'])) > 5 + ### TODO - MAKE THE PR ### + except (KeyError, AssertionError) as e: + self.pr_error = True + logging.error("Environment variable `$NF_CORE_BOT` is not set - cannot make PR") + if self.gh_username and self.gh_repo: + logging.info("Make a PR at the following URL:\n https://github.com/{}/{}/compare/{}...TEMPLATE".format(self.gh_username, self.gh_repo, self.original_branch)) # Reset: Check out original branch again logging.debug("Checking out original branch: '{}'".format(self.original_branch)) @@ -190,11 +216,11 @@ def sync(self): return False # Finish up - if not self.make_pr: + if not self.make_pr and self.made_changes: git_merge_cmd = 'git merge TEMPLATE' if self.orphan_branch: git_merge_cmd += ' --allow-unrelated-histories' - logging.info("Now try to merge the updates in to your pipeline:\n {}".format(git_merge_cmd)) + logging.info("Now try to merge the updates in to your pipeline:\n cd {}\n {}".format(self.pipeline_dir, git_merge_cmd)) From 1189b26a1c85bbe6c7504d1d34714df7b93919d0 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 14 Oct 2019 10:55:50 +0200 Subject: [PATCH 063/224] Split everything into functions. More testing --- nf_core/sync.py | 388 +++++++++++++++++++++++------------------------- scripts/nf-core | 6 +- 2 files changed, 190 insertions(+), 204 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index fa762de55..f756e3c0a 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -14,6 +14,16 @@ import sys import tempfile +class SyncException(Exception): + """Exception raised when there was an error with TEMPLATE branch synchronisation + """ + pass + +class PullRequestException(Exception): + """Exception raised when there was an error creating a Pull-Request on GitHub.com + """ + pass + class PipelineSync(object): """Object to hold syncing information and results. @@ -24,44 +34,92 @@ class PipelineSync(object): make_pr (bool): Set this to `True` to create a GitHub pull-request with the changes Attributes: - gh_base_url (str): The GitHub base url or the nf-core project - github_pr_url_templ (str): The API endpoint for creating pull requests - path_parent_dir (str): Current script directory - sync_errors (list): List of errors from sync - pr_errors (list): List of errors from PR + pipeline_dir (str): Path to target pipeline directory + from_branch (str): Repo branch to use when collecting workflow variables. Default: active branch. + make_template_branch (bool): Whether to try to create TEMPLATE branch if not found + orphan_branch (bool): Whether an orphan branch was made when creating TEMPLATE + made_changes (bool): Whether making the new template pipeline introduced any changes + make_pr (bool): Whether to try to automatically make a PR on GitHub.com + required_config_vars (list): List of nextflow variables required to make template pipeline + gh_auth_token (str): Authorisation token used to make PR with GitHub API + gh_username (str): GitHub username + gh_repo (str): GitHub repository name """ def __init__(self, pipeline_dir, make_template_branch=False, from_branch=None, make_pr=False): """ Initialise syncing object """ self.pipeline_dir = os.path.abspath(pipeline_dir) - self.make_template_branch = make_template_branch self.from_branch = from_branch + self.make_template_branch = make_template_branch self.orphan_branch = False self.made_changes = False self.make_pr = make_pr - + self.required_config_vars = [ + 'manifest.name', + 'manifest.description', + 'manifest.version', + 'manifest.author' + ] + + self.gh_auth_token = None self.gh_username = None self.gh_repo = None - self.gh_base_url = "https://{token}@github.com/{username}/{pipeline}" - self.github_pr_url_templ = "https://api.github.com/repos/{username}/{pipeline}/pulls" - self.sync_error = False - self.pr_error = False + # Currently no other way to set this token except $NF_CORE_BOT + if self.gh_auth_token is None: + try: + self.gh_auth_token = os.environ['NF_CORE_BOT'] + except KeyError: + pass def sync(self): """ Find workflow attributes, create a new template pipeline on TEMPLATE """ - logging.info("Pipeline directory: {}".format(self.pipeline_dir)) + config_log_msg = "Pipeline directory: {}".format(self.pipeline_dir) + if self.from_branch: + config_log_msg += "\n Using branch `{}` to fetch workflow variables".format(self.from_branch) + if self.make_template_branch: + config_log_msg += "\n Will attempt to create `TEMPLATE` branch if not found" + if self.make_pr: + config_log_msg += "\n Will attempt to automatically create a pull request on GitHub.com" + logging.info(config_log_msg) + + self.inspect_sync_dir() + + self.get_wf_config() + + self.checkout_template_branch() + + self.make_template_pipeline() + + self.commit_template_changes() + + # Push and make a pull request if we've been asked to + if self.make_pr: + try: + self.push_template_branch() + self.make_pull_request() + except PullRequestException as e: + # Keep going - we want to clean up the target directory still + logging.error(e) + + self.reset_target_dir() + + if not self.make_pr: + self.git_merge_help() + + def inspect_sync_dir(self): + """Takes a look at the target directory for syncing. Checks that it's a git repo + and makes sure that there are no uncommitted changes. + """ # Check that the pipeline_dir is a git repo try: self.repo = git.Repo(self.pipeline_dir) except git.exc.InvalidGitRepositoryError as e: - self.sync_error = True - logging.error("'{}' does not appear to be a git repository".format(self.pipeline_dir)) - return False + raise SyncException("'{}' does not appear to be a git repository".format(self.pipeline_dir)) # get current branch so we can switch back later self.original_branch = self.repo.active_branch.name @@ -69,32 +127,46 @@ def sync(self): # Check to see if there are uncommitted changes on current branch if self.repo.is_dirty(untracked_files=True): - self.sync_error = True - logging.error("Uncommitted changes found in pipeline directory!\nPlease commit these before running nf-core sync") - return False + raise SyncException("Uncommitted changes found in pipeline directory!\nPlease commit these before running nf-core sync") + def get_wf_config(self): + """Check out the target branch if requested and fetch the nextflow config. + Check that we have the required config variables. + """ # Try to check out target branch (eg. `origin/dev`) try: if self.from_branch and self.repo.active_branch.name != self.from_branch: logging.info("Checking out workflow branch '{}'".format(self.from_branch)) self.repo.git.checkout(self.from_branch) except git.exc.GitCommandError: - self.sync_error = True - logging.error("Branch `{}` not found!".format(self.from_branch)) - return False + raise SyncException("Branch `{}` not found!".format(self.from_branch)) + + # Figure out the GitHub username and repo name from the 'origin' remote if we can + try: + gh_ssh_username_match = re.search(r'git@github\.com:([^\/]+)/([^\/]+)\.git$', self.repo.remotes.origin.url) + if gh_ssh_username_match: + self.gh_username = gh_ssh_username_match.group(1) + self.gh_repo = gh_ssh_username_match.group(2) + gh_url_username_match = re.search(r'https://github\.com/([^\/]+)/([^\/]+)\.git$', self.repo.remotes.origin.url) + if gh_url_username_match: + self.gh_username = gh_url_username_match.group(1) + self.gh_repo = gh_url_username_match.group(2) + except AttributeError as e: + logging.debug("Could not find repository URL for remote called 'origin'") # Fetch workflow variables logging.info("Fetching workflow config variables") self.wf_config = nf_core.utils.fetch_wf_config(self.pipeline_dir) # Check that we have the required variables - for rvar in ['manifest.name','manifest.description','manifest.version','manifest.author']: + for rvar in self.required_config_vars: if rvar not in self.wf_config: - self.sync_error = True - logging.error("Workflow config variable `{}` not found!".format(rvar)) - return False - + raise SyncException("Workflow config variable `{}` not found!".format(rvar)) + def checkout_template_branch(self): + """Try to check out the TEMPLATE branch. If it fails, try origin/TEMPLATE. + If it still fails and --make-template-branch was given, create it as an orphan branch. + """ # Try to check out the `TEMPLATE` branch try: self.repo.git.checkout("origin/TEMPLATE", b="TEMPLATE") @@ -107,10 +179,10 @@ def sync(self): # Failed, if we're not making a new branch just die if not self.make_template_branch: - self.sync_error = True - logging.error("Could not check out branch 'origin/TEMPLATE'") - logging.info("Use flag --make-template-branch to attempt to create this branch") - return False + raise SyncException( + "Could not check out branch 'origin/TEMPLATE'" \ + "\nUse flag --make-template-branch to attempt to create this branch" + ) # Branch and force is set, fire function to create `TEMPLATE` branch else: @@ -121,11 +193,13 @@ def sync(self): self.orphan_branch = True if self.make_pr: self.make_pr = False - logging.info("Will not attempt to make a PR - orphan branch must be merged manually first") + logging.warning("Will not attempt to make a PR - orphan branch must be merged manually first") except git.exc.GitCommandError as e: - self.sync_error = True - logging.error("Could not create 'TEMPLATE' branch:\n{}".format(e)) - return False + raise SyncException("Could not create 'TEMPLATE' branch:\n{}".format(e)) + + def make_template_pipeline(self): + """Delete all files and make a fresh template using the workflow variables + """ # Delete everything logging.info("Deleting all files in TEMPLATE branch") @@ -140,12 +214,16 @@ def sync(self): elif os.path.isdir(file_path): shutil.rmtree(file_path) except Exception as e: - self.sync_error = True - logging.error(e) - return False + raise SyncException(e) # Make a new pipeline using nf_core.create logging.info("Making a new template pipeline using pipeline variables") + + # Suppress log messages from the pipeline creation method + orig_loglevel = logging.getLogger().getEffectiveLevel() + if orig_loglevel == getattr(logging, 'INFO'): + logging.getLogger().setLevel(logging.ERROR) + nf_core.create.PipelineCreate( name = self.wf_config['manifest.name'].strip('\"').strip("\'"), description = self.wf_config['manifest.description'].strip('\"').strip("\'"), @@ -156,6 +234,12 @@ def sync(self): author = self.wf_config['manifest.author'].strip('\"').strip("\'"), ).init_pipeline() + # Reset logging + logging.getLogger().setLevel(orig_loglevel) + + def commit_template_changes(self): + """If we have any changes with the new template files, make a git commit + """ # Commit changes if we have any if not self.repo.is_dirty(untracked_files=True): logging.info("Template contains no changes - no new commit created") @@ -166,12 +250,14 @@ def sync(self): self.made_changes = True logging.info("Committed changes to TEMPLATE branch") except Exception as e: - self.sync_error = True - logging.error("Could not commit changes to TEMPLATE:\n{}".format(e)) - return False + raise SyncException("Could not commit changes to TEMPLATE:\n{}".format(e)) - # Push and make a pull request if we've been asked to - if self.make_pr and self.made_changes: + def push_template_branch(self): + """If we made any changes, push the TEMPLATE branch to the default remote + and try to make a PR. If we don't have the auth token, try to figure out a URL + for the PR and print this to the console. + """ + if self.made_changes: logging.info("Pushing TEMPLATE branch to remote") try: self.repo.git.push() @@ -180,176 +266,74 @@ def sync(self): try: self.repo.git.push('--set-upstream', 'origin', 'TEMPLATE') except git.exc.GitCommandError as e: - logging.error("Could not push TEMPLATE branch:\n {}".format(e)) - self.pr_error = True + raise PullRequestException("Could not push TEMPLATE branch:\n {}".format(e)) else: - logging.error("Could not push TEMPLATE branch:\n {}".format(e)) - self.pr_error = True + raise PullRequestException("Could not push TEMPLATE branch:\n {}".format(e)) + else: + logging.debug("No changes to TEMPLATE - skipping push to remote") - # Figure out the GitHub username to make a PR if we can - gh_ssh_username_match = re.search(r'git@github\.com:([^\/]+)/([^\/]+)\.git$', self.repo.remotes.origin.url) - if gh_ssh_username_match: - self.gh_username = gh_ssh_username_match.group(1) - self.gh_repo = gh_ssh_username_match.group(2) - gh_url_username_match = re.search(r'https://github\.com/([^\/]+)/([^\/]+)\.git$', self.repo.remotes.origin.url) - if gh_url_username_match: - self.gh_username = gh_url_username_match.group(1) - self.gh_repo = gh_url_username_match.group(2) + def make_pull_request(self): + """Create a pull request to a base branch (default: dev), + from a head branch (default: TEMPLATE) - # If we've been asked to make a PR, check that we have the credentials - try: - assert len(str(os.environ['NF_CORE_BOT'])) > 5 - ### TODO - MAKE THE PR ### - except (KeyError, AssertionError) as e: - self.pr_error = True - logging.error("Environment variable `$NF_CORE_BOT` is not set - cannot make PR") - if self.gh_username and self.gh_repo: - logging.info("Make a PR at the following URL:\n https://github.com/{}/{}/compare/{}...TEMPLATE".format(self.gh_username, self.gh_repo, self.original_branch)) + Returns: An instance of class requests.Response + """ + if not self.made_changes: + logging.debug("No changes to TEMPLATE - skipping PR creation") + + # Check that we know the github username and repo name + try: + assert self.gh_username is not None + assert self.gh_repo is not None + except AssertionError: + raise PullRequestException("Could not find GitHub username and repo from git remote 'origin'") + + # If we've been asked to make a PR, check that we have the credentials + try: + assert self.gh_auth_token is not None + except AssertionError: + logging.info("Make a PR at the following URL:\n https://github.com/{}/{}/compare/{}...TEMPLATE".format(self.gh_username, self.gh_repo, self.original_branch)) + raise PullRequestException("No GitHub authentication token set - cannot make PR") + + pr_content = { + 'title': "Important! Template update for nf-core/tools template, version {}".format(nf_core.__version__), + 'body': "Some important changes have been made in the nf-core/tools pipeline template.\n" \ + "Please make sure to merge this pull-request as soon as possible.\n" \ + "Once complete, make a new minor release of your pipeline.\n\n" \ + "For more information, please see the [nf-core/tools v{tag} release page](https://github.com/nf-core/tools/releases/tag/{tag}).".format(tag=nf_core.__version__), + 'head': "TEMPLATE", + 'base': self.from_branch + } + return requests.post( + url = "https://api.github.com/repos/{}/{}/pulls".format(self.gh_username, self.gh_repo), + data = json.dumps(pr_content), + auth = HTTPBasicAuth(self.gh_username, self.gh_auth_token) + ) + + def reset_target_dir(self): + """Reset the target pipeline directory. Check out the original branch. + """ # Reset: Check out original branch again logging.debug("Checking out original branch: '{}'".format(self.original_branch)) try: self.repo.git.checkout(self.original_branch) except git.exc.GitCommandError as e: - self.sync_error = True - logging.error("Could not reset to original branch `{}`:\n{}".format(self.from_branch, e)) - return False + raise SyncException("Could not reset to original branch `{}`:\n{}".format(self.from_branch, e)) - # Finish up - if not self.make_pr and self.made_changes: + def git_merge_help(self): + """Print a command line help message with instructions on how to merge changes + """ + if self.made_changes: git_merge_cmd = 'git merge TEMPLATE' + manual_sync_link = '' if self.orphan_branch: git_merge_cmd += ' --allow-unrelated-histories' - logging.info("Now try to merge the updates in to your pipeline:\n cd {}\n {}".format(self.pipeline_dir, git_merge_cmd)) - - - -# def update_template_branch(self): -# """ Check out `TEMPLATE` and make a new pipeline using nf_core.create """ -# -# try: -# syncutils.template.NfcoreTemplate( -# pipeline['name'], -# branch=DEF_TEMPLATE_BRANCH, -# repo_url=GH_BASE_URL.format(token=os.environ["NF_CORE_BOT"], pipeline=pipeline['name']) -# ).sync() -# except Exception as e: -# sync_errors.append((pipeline['name'], e)) -# -# def create_pull_request(self): -# name = pipeline.get('name') -# for errored_pipeline, _ in sync_errors: -# if name == errored_pipeline: -# return -# response = create_pullrequest(name, token=os.environ["NF_CORE_BOT"]) -# if response.status_code != 201: -# pr_errors.append((name, response.status_code, response.content)) -# else: -# print("Created pull-request for pipeline \'{pipeline}\' successfully." -# .format(pipeline=name)) -# -# -# -# -# -# -#def create_pullrequest(pipeline, origin="dev", template="TEMPLATE", token="", user="nf-core"): -# """Create a pull request to a base branch (default: dev), -# from a head branch (default: TEMPLATE) -# -# Returns: An instance of class requests.Response -# """ -# content = {} -# content['title'] = "Important pipeline nf-core update! (version {tag})".format(tag=os.environ['TRAVIS_TAG']) -# content['body'] = "Some important changes have been made in the nf-core pipelines templates.\n" \ -# "Please make sure to merge this in ASAP and make a new minor release of your pipeline.\n\n" \ -# "Follow the link [nf-core/tools](https://github.com/nf-core/tools/releases/tag/{})".format(os.environ['TRAVIS_TAG']) -# content['head'] = "{}".format(template) -# content['base'] = origin -# return requests.post(url=GITHUB_PR_URL_TEMPL.format(pipeline=pipeline), -# data=json.dumps(content), -# auth=requests.auth.TTPBasicAuth(user, token)) -# -# -#def filter_blacklisted_pipelines_from_list(pipelines, blacklisted_pipelines): -# filtered_pipelines = [] -# for pipeline in pipelines: -# if not pipeline.get('name'): -# print("No attribute \'name\' for pipeline found: {}".format(pipeline)) -# else: -# filtered_pipelines.append(pipeline) if pipeline.get('name') not in blacklisted_pipelines \ -# else filtered_pipelines -# return filtered_pipelines -# -# -#def fetch_black_listed_pipelines_from_file(file_path): -# with open(file_path) as fh: -# blacklist = json.load(fh) -# return blacklist.get('pipelines') -# -# -#def fetch_nfcore_workflows_from_website(url): -# try: -# res = requests.get(url) -# pipelines = res.json().get('remote_workflows') -# except Exception as e: -# print("Could not get remote workflows. Reason was: {}".format(e)) -# pipelines = [] -# return pipelines -# -# -#def update_template_branch_for_pipeline(pipeline): -# try: -# syncutils.template.NfcoreTemplate( -# pipeline['name'], -# branch=DEF_TEMPLATE_BRANCH, -# repo_url=GH_BASE_URL.format(token=os.environ["NF_CORE_BOT"], pipeline=pipeline['name']) -# ).sync() -# except Exception as e: -# sync_errors.append((pipeline['name'], e)) -# -# -#def create_pullrequest_if_update_sucessful(pipeline): -# name = pipeline.get('name') -# for errored_pipeline, _ in sync_errors: -# if name == errored_pipeline: -# return -# response = create_pullrequest(name, token=os.environ["NF_CORE_BOT"]) -# if response.status_code != 201: -# pr_errors.append((name, response.status_code, response.content)) -# else: -# print("Created pull-request for pipeline \'{pipeline}\' successfully." -# .format(pipeline=name)) -# -# -#def main(): -# assert os.environ['TRAVIS_TAG'] -# assert os.environ['NF_CORE_BOT'] -# -# blacklisted_pipeline_names = fetch_black_listed_pipelines_from_file(PATH_PARENT_DIR + "/blacklist.json") -# -# pipelines = fetch_nfcore_workflows_from_website(NF_CORE_PIPELINE_INFO) -# -# if len(sys.argv) > 1: -# pipeline_to_sync = sys.argv[1] -# filtered_pipelines = [pipeline for pipeline in pipelines if pipeline_to_sync in pipeline.get('name')] -# else: -# filtered_pipelines = filter_blacklisted_pipelines_from_list(pipelines, blacklisted_pipeline_names) -# -# for pipeline in filtered_pipelines: -# print("Update template branch for pipeline '{pipeline}'... ".format(pipeline=pipeline['name'])) -# update_template_branch_for_pipeline(pipeline) -# print("Trying to open pull request for pipeline {}...".format(pipeline['name'])) -# create_pullrequest_if_update_sucessful(pipeline) -# -# for pipeline, exception in sync_errors: -# print("WARNING!!!! Sync for pipeline {name} failed.".format(name=pipeline)) -# print(exception) -# -# for pipeline, return_code, content in pr_errors: -# print("WARNING!!!! Pull-request for pipeline \'{pipeline}\' failed," -# " got return code {return_code}." -# .format(pipeline=pipeline, return_code=return_code)) -# print(content) -# -# sys.exit(0) + manual_sync_link = "\n\nFor more information, please see:\nhttps://nf-co.re/developers/sync#merge-template-into-main-branches" + logging.info( + "Now try to merge the updates in to your pipeline:\n cd {}\n {}{}".format( + self.pipeline_dir, + git_merge_cmd, + manual_sync_link + ) + ) diff --git a/scripts/nf-core b/scripts/nf-core index bc8cd04de..2da5a1fcd 100755 --- a/scripts/nf-core +++ b/scripts/nf-core @@ -303,8 +303,10 @@ def bump_version(pipeline_dir, new_version, nextflow): def sync(pipeline_dir, make_template_branch, from_branch, pull_request): """ Sync a pipeline TEMPLATE branch with the nf-core template""" sync_obj = nf_core.sync.PipelineSync(pipeline_dir, make_template_branch, from_branch, pull_request) - sync_obj.sync() - if sync_obj.sync_error or sync_obj.pr_error: + try: + sync_obj.sync() + except (nf_core.sync.SyncException, nf_core.sync.PullRequestException) as e: + logging.error(e) sys.exit(1) From 8263819576f0250322f212da302e6560eb8d499c Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 14 Oct 2019 11:19:23 +0200 Subject: [PATCH 064/224] Sync: Fix PR code. Better error handling and logging. --- nf_core/sync.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index f756e3c0a..8d9a1925f 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -141,6 +141,13 @@ def get_wf_config(self): except git.exc.GitCommandError: raise SyncException("Branch `{}` not found!".format(self.from_branch)) + # If not specified, get the name of the active branch + if not self.from_branch: + try: + self.from_branch = self.repo.active_branch.name + except git.exc.GitCommandError as e: + logging.error("Could not find active repo branch: ".format(e)) + # Figure out the GitHub username and repo name from the 'origin' remote if we can try: gh_ssh_username_match = re.search(r'git@github\.com:([^\/]+)/([^\/]+)\.git$', self.repo.remotes.origin.url) @@ -295,20 +302,24 @@ def make_pull_request(self): logging.info("Make a PR at the following URL:\n https://github.com/{}/{}/compare/{}...TEMPLATE".format(self.gh_username, self.gh_repo, self.original_branch)) raise PullRequestException("No GitHub authentication token set - cannot make PR") + logging.info("Submitting a pull request via the GitHub API") pr_content = { - 'title': "Important! Template update for nf-core/tools template, version {}".format(nf_core.__version__), - 'body': "Some important changes have been made in the nf-core/tools pipeline template.\n" \ - "Please make sure to merge this pull-request as soon as possible.\n" \ + 'title': "Important! Template update for nf-core/tools v{}".format(nf_core.__version__), + 'body': "Some important changes have been made in the nf-core/tools pipeline template. " \ + "Please make sure to merge this pull-request as soon as possible. " \ "Once complete, make a new minor release of your pipeline.\n\n" \ "For more information, please see the [nf-core/tools v{tag} release page](https://github.com/nf-core/tools/releases/tag/{tag}).".format(tag=nf_core.__version__), 'head': "TEMPLATE", 'base': self.from_branch } - return requests.post( + r = requests.post( url = "https://api.github.com/repos/{}/{}/pulls".format(self.gh_username, self.gh_repo), data = json.dumps(pr_content), - auth = HTTPBasicAuth(self.gh_username, self.gh_auth_token) + auth = requests.auth.HTTPBasicAuth(self.gh_username, self.gh_auth_token) ) + if r.status_code != 200: + raise PullRequestException("GitHub API returned code {}: {}".format(r.status_code, r.text)) + logging.debug(r.json) def reset_target_dir(self): """Reset the target pipeline directory. Check out the original branch. From 44202d370c337cd6bb30e30634f893abaf4c9b06 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 14 Oct 2019 11:33:32 +0200 Subject: [PATCH 065/224] Sync: Add github username, repo and auth as cli args --- nf_core/sync.py | 16 +++++++++------- scripts/nf-core | 21 ++++++++++++++++++--- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 8d9a1925f..ad7f47e0b 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -32,6 +32,9 @@ class PipelineSync(object): make_template_branch (bool): Set this to `True` to create a `TEMPLATE` branch if it is not found from_branch (str): The branch to use to fetch config vars. If not set, will use current active branch make_pr (bool): Set this to `True` to create a GitHub pull-request with the changes + gh_username (str): GitHub username + gh_repo (str): GitHub repository name + gh_auth_token (str): Authorisation token used to make PR with GitHub API Attributes: pipeline_dir (str): Path to target pipeline directory @@ -41,12 +44,13 @@ class PipelineSync(object): made_changes (bool): Whether making the new template pipeline introduced any changes make_pr (bool): Whether to try to automatically make a PR on GitHub.com required_config_vars (list): List of nextflow variables required to make template pipeline - gh_auth_token (str): Authorisation token used to make PR with GitHub API gh_username (str): GitHub username gh_repo (str): GitHub repository name + gh_auth_token (str): Authorisation token used to make PR with GitHub API """ - def __init__(self, pipeline_dir, make_template_branch=False, from_branch=None, make_pr=False): + def __init__(self, pipeline_dir, make_template_branch=False, from_branch=None, make_pr=False, + gh_username=None, gh_repo=None, gh_auth_token=None): """ Initialise syncing object """ self.pipeline_dir = os.path.abspath(pipeline_dir) @@ -62,11 +66,9 @@ def __init__(self, pipeline_dir, make_template_branch=False, from_branch=None, m 'manifest.author' ] - self.gh_auth_token = None - self.gh_username = None - self.gh_repo = None - - # Currently no other way to set this token except $NF_CORE_BOT + self.gh_username = gh_username + self.gh_repo = gh_repo + self.gh_auth_token = gh_auth_token if self.gh_auth_token is None: try: self.gh_auth_token = os.environ['NF_CORE_BOT'] diff --git a/scripts/nf-core b/scripts/nf-core index 2da5a1fcd..c2c657950 100755 --- a/scripts/nf-core +++ b/scripts/nf-core @@ -292,15 +292,30 @@ def bump_version(pipeline_dir, new_version, nextflow): @click.option( '-b', '--from-branch', type = str, - help = 'The pipeline git branch to use to get workflow vars' + help = 'The git branch to use to fetch workflow vars.' ) @click.option( '-p', '--pull-request', is_flag = True, default = False, - help = "Make a GitHub pull-request with the changes" + help = "Make a GitHub pull-request with the changes." ) -def sync(pipeline_dir, make_template_branch, from_branch, pull_request): +@click.option( + '-u', '--username', + type = str, + help = 'GitHub username for the PR.' +) +@click.option( + '-r', '--repository', + type = str, + help = 'GitHub repository name for the PR.' +) +@click.option( + '-a', '--auth-token', + type = str, + help = 'GitHub API personal access token.' +) +def sync(pipeline_dir, make_template_branch, from_branch, pull_request, username, repository, auth_token): """ Sync a pipeline TEMPLATE branch with the nf-core template""" sync_obj = nf_core.sync.PipelineSync(pipeline_dir, make_template_branch, from_branch, pull_request) try: From c9cc812ec90007de2b38f0b80a2f5ee8e13b54c3 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 14 Oct 2019 12:04:13 +0200 Subject: [PATCH 066/224] Sync: added sync-all command --- nf_core/sync.py | 70 +++++++++++++++++++++++++++++++++++++++++++++++-- scripts/nf-core | 16 +++++++++++ 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index ad7f47e0b..fdef6d15c 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -"""Bumps the version number in all appropriate files for -a nf-core pipeline. +"""Synchronise a pipeline TEMPLATE branch with the template. """ import git @@ -350,3 +349,70 @@ def git_merge_help(self): manual_sync_link ) ) + + + + +def sync_all_pipelines(gh_username='nf-core-bot', gh_auth_token=None): + """Sync all nf-core pipelines + """ + + # Get remote workflows + wfs = nf_core.list.Workflows() + wfs.get_remote_workflows() + + successful_syncs = [] + failed_syncs = [] + + # Set up a working directory + tmpdir = tempfile.mkdtemp() + + # Let's do some updating! + for wf in wfs.remote_workflows: + + logging.info("Syncing {}".format(wf.full_name)) + + # Make a local working directory + wf_local_path = os.path.join(tmpdir, wf.name) + os.mkdir(wf_local_path) + logging.debug("Sync working directory: {}".format(wf_local_path)) + + # Clone the repo + wf_remote_url = "https://{}@github.com/nf-core/{}".format(gh_auth_token, wf.name) + repo = git.Repo.clone_from(wf_remote_url, wf_local_path) + assert repo + + # Suppress log messages from the pipeline creation method + orig_loglevel = logging.getLogger().getEffectiveLevel() + if orig_loglevel == getattr(logging, 'INFO'): + logging.getLogger().setLevel(logging.ERROR) + + # Sync the repo + logging.debug("Running template sync") + sync_obj = nf_core.sync.PipelineSync( + pipeline_dir=wf_local_path, + from_branch='dev', + make_pr=True, + gh_username=gh_username + ) + try: + sync_obj.sync() + except (nf_core.sync.SyncException, nf_core.sync.PullRequestException) as e: + logging.error("Sync failed for {}:\n{}".format(wf.full_name, e)) + failed_syncs.append(wf.name) + else: + logging.debug("Sync successful for {}".format(wf.full_name)) + successful_syncs.append(wf.name) + + # Reset logging + logging.getLogger().setLevel(orig_loglevel) + + # Clean up + logging.debug("Removing work directory: {}".format(wf_local_path)) + shutil.rmtree(wf_local_path) + + logging.info("Successfully synchronised {} pipelines".format(len(successful_syncs))) + + if len(failed_syncs) > 0: + failed_list = '\n - '.join(failed_syncs) + logging.error("Errors whilst synchronising {} pipelines:\n - {}".format(len(failed_syncs), failed_list)) diff --git a/scripts/nf-core b/scripts/nf-core index c2c657950..ed48c86d4 100755 --- a/scripts/nf-core +++ b/scripts/nf-core @@ -324,6 +324,22 @@ def sync(pipeline_dir, make_template_branch, from_branch, pull_request, username logging.error(e) sys.exit(1) +@nf_core_cli.command('sync-all', help_priority=9) +@click.option( + '-u', '--username', + type = str, + default = 'nf-core-bot', + help = 'GitHub username for the PRs.' +) +@click.option( + '-a', '--auth-token', + type = str, + help = 'GitHub API personal access token.' +) +def sync(username, auth_token): + """ Sync template for all nf-core pipelines. nf-core admins only.""" + nf_core.sync.sync_all_pipelines() + if __name__ == '__main__': From d3dea2c3d6c2410638f512a203ff7c37e8b24693 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 14 Oct 2019 12:23:01 +0200 Subject: [PATCH 067/224] Sync: Update readme --- README.md | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/README.md b/README.md index f3d81a59d..1cc40174c 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ A python package with helper tools for the nf-core community. * [`nf-core create` - Create a new workflow from the nf-core template](#creating-a-new-workflow) * [`nf-core lint` - Check pipeline code against nf-core guidelines](#linting-a-workflow) * [`nf-core bump-version` - Update nf-core pipeline version number](#bumping-a-pipeline-version-number) +* [`nf-core sync` - Synchronise a pipeline TEMPLATE branch](#sync-a-pipeline-with-the-template) +* [`nf-core sync-all` - Sync all nf-core pipelines and trigger PRs](#sync-all-nf-core-pipelines) The nf-core tools package is written in Python and can be imported and used within other packages. For documentation of the internal Python functions, please refer to the [Tools Python API docs](https://nf-co.re/tools-docs/). @@ -431,6 +433,84 @@ INFO: Updating version in Singularity To change the required version of Nextflow instead of the pipeline version number, use the flag `--nextflow`. +## Sync a pipeline with the template + +Over time, the main nf-core pipeline template is updated. To keep all nf-core pipelines up to date, +we synchronise these updates automatically when new versions of nf-core/tools are released. +This is done by maintaining a special `TEMPLATE` branch, containing a vanilla copy of the nf-core template +with only the variables used when it first ran (name, description etc.). This branch is updated and a +pull-request can be made with just the updates from the main template code. + +This command takes a pipeline directory and attempts to run this synchronisation. +Usage is `nf-core sync `, eg: + +```console +$ nf-core sync my_pipeline/ + + ,--./,-. + ___ __ __ __ ___ /,-._.--~\ + |\ | |__ __ / ` / \ |__) |__ } { + | \| | \__, \__/ | \ |___ \`-._,-`-, + `._,._,' + + +INFO: Pipeline directory: /path/to/my_pipeline + +INFO: Fetching workflow config variables + +INFO: Deleting all files in TEMPLATE branch + +INFO: Making a new template pipeline using pipeline variables + +INFO: Committed changes to TEMPLATE branch + +INFO: Now try to merge the updates in to your pipeline: + cd /path/to/my_pipeline + git merge TEMPLATE +``` + +If your pipeline repository does not already have a `TEMPLATE` branch, you can instruct +the command to try to create one by giving the `--make-template-branch` flag. +If it has to, the sync tool will then create an orphan branch - see the +[nf-core website sync documentation](https://nf-co.re/developers/sync) for details on +how to handle this. + +By default, the tool will collect workflow variables from the current branch in your +pipeline directory. You can supply the `--from-branch` flag to specific a different branch. + +Finally, if you give the `--pull-request` flag, the command will push any changes to the remote +and attempt to create a pull request using the GitHub API. The GitHub username and repository +name will be fetched from the remote url (see `git remote -v | grep origin`), or can be supplied +with `--username` and `--repository`. + +To create the pull request, a personal access token is required for API authentication. +These can be created at [https://github.com/settings/tokens](https://github.com/settings/tokens). +Supply this using the `--auth-token` flag, or setting it as the environment variable `NF_CORE_BOT`: +`export NF_CORE_BOT=my_auth_token`. + +## Sync all nf-core pipelines + +This command is used by the nf-core/tools release automation to synchronise all nf-core pipelines +with the newest version of the template. It requires authentication as either the nf-core-bot account +or as an nf-core administrator. + +```console +$ nf-core sync-all + + ,--./,-. + ___ __ __ __ ___ /,-._.--~\ + |\ | |__ __ / ` / \ |__) |__ } { + | \| | \__, \__/ | \ |___ \`-._,-`-, + `._,._,' + + +INFO: Syncing nf-core/ampliseq + +[...] + +INFO: Successfully synchronised [n] pipelines +``` + ## Citation If you use `nf-core tools` in your work, please cite the `nf-core` preprint as follows: From 5dc11340e05de6ec72ba5f0dc0d80df320e81200 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 14 Oct 2019 12:24:47 +0200 Subject: [PATCH 068/224] Remove old sync code, update travis --- .travis.yml | 2 +- CHANGELOG.md | 2 +- bin/sync | 129 -------------------------------------- bin/syncutils/__init__.py | 1 - bin/syncutils/template.py | 87 ------------------------- bin/syncutils/utils.py | 64 ------------------- 6 files changed, 2 insertions(+), 283 deletions(-) delete mode 100755 bin/sync delete mode 100644 bin/syncutils/__init__.py delete mode 100644 bin/syncutils/template.py delete mode 100644 bin/syncutils/utils.py diff --git a/.travis.yml b/.travis.yml index 70d4a9547..f5a2e545e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,4 +57,4 @@ deploy: # Sync pipelines with possible nf-core template changes after_deploy: - - ./bin/sync + - nf-core sync-all diff --git a/CHANGELOG.md b/CHANGELOG.md index da1cd6f7f..d86528932 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Tools helper code -* Updated Blacklist of synced pipelines +* Refactored the template synchronisation code to be part of the main nf-core tool ### Template diff --git a/bin/sync b/bin/sync deleted file mode 100755 index 6dda0ed62..000000000 --- a/bin/sync +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python - -import json -import os -import requests -from requests.auth import HTTPBasicAuth -import sys -import syncutils.template -import syncutils.utils - -# Set the default nf-core pipeline template branch -DEF_TEMPLATE_BRANCH = "TEMPLATE" -# The GitHub base url or the nf-core project -GH_BASE_URL = "https://{token}@github.com/nf-core/{pipeline}" -# The JSON file is updated on every push event on the nf-core GitHub project -NF_CORE_PIPELINE_INFO = "http://nf-co.re/pipelines.json" -# The API endpoint for creating pull requests -GITHUB_PR_URL_TEMPL = "https://api.github.com/repos/nf-core/{pipeline}/pulls" -# Current script dir -PATH_PARENT_DIR = os.path.dirname(os.path.realpath(__file__)) - -sync_errors = [] -pr_errors = [] - - -def create_pullrequest(pipeline, origin="dev", template="TEMPLATE", token="", user="nf-core"): - """Create a pull request to a base branch (default: dev), - from a head branch (default: TEMPLATE) - - Returns: An instance of class requests.Response - """ - content = {} - content['title'] = "Important pipeline nf-core update! (version {tag})".format(tag=os.environ['TRAVIS_TAG']) - content['body'] = "Some important changes have been made in the nf-core pipelines templates.\n" \ - "Please make sure to merge this in ASAP and make a new minor release of your pipeline.\n\n" \ - "Follow the link [nf-core/tools](https://github.com/nf-core/tools/releases/tag/{})".format(os.environ['TRAVIS_TAG']) - content['head'] = "{}".format(template) - content['base'] = origin - return requests.post(url=GITHUB_PR_URL_TEMPL.format(pipeline=pipeline), - data=json.dumps(content), - auth=HTTPBasicAuth(user, token)) - - -def filter_blacklisted_pipelines_from_list(pipelines, blacklisted_pipelines): - filtered_pipelines = [] - for pipeline in pipelines: - if not pipeline.get('name'): - print("No attribute \'name\' for pipeline found: {}".format(pipeline)) - else: - filtered_pipelines.append(pipeline) if pipeline.get('name') not in blacklisted_pipelines \ - else filtered_pipelines - return filtered_pipelines - - -def fetch_black_listed_pipelines_from_file(file_path): - with open(file_path) as fh: - blacklist = json.load(fh) - return blacklist.get('pipelines') - - -def fetch_nfcore_workflows_from_website(url): - try: - res = requests.get(url) - pipelines = res.json().get('remote_workflows') - except Exception as e: - print("Could not get remote workflows. Reason was: {}".format(e)) - pipelines = [] - return pipelines - - -def update_template_branch_for_pipeline(pipeline): - try: - syncutils.template.NfcoreTemplate( - pipeline['name'], - branch=DEF_TEMPLATE_BRANCH, - repo_url=GH_BASE_URL.format(token=os.environ["NF_CORE_BOT"], pipeline=pipeline['name']) - ).sync() - except Exception as e: - sync_errors.append((pipeline['name'], e)) - - -def create_pullrequest_if_update_sucessful(pipeline): - name = pipeline.get('name') - for errored_pipeline, _ in sync_errors: - if name == errored_pipeline: - return - response = create_pullrequest(name, token=os.environ["NF_CORE_BOT"]) - if response.status_code != 201: - pr_errors.append((name, response.status_code, response.content)) - else: - print("Created pull-request for pipeline \'{pipeline}\' successfully." - .format(pipeline=name)) - - -def main(): - assert os.environ['TRAVIS_TAG'] - assert os.environ['NF_CORE_BOT'] - - blacklisted_pipeline_names = fetch_black_listed_pipelines_from_file(PATH_PARENT_DIR + "/blacklist.json") - - pipelines = fetch_nfcore_workflows_from_website(NF_CORE_PIPELINE_INFO) - - if len(sys.argv) > 1: - pipeline_to_sync = sys.argv[1] - filtered_pipelines = [pipeline for pipeline in pipelines if pipeline_to_sync in pipeline.get('name')] - else: - filtered_pipelines = filter_blacklisted_pipelines_from_list(pipelines, blacklisted_pipeline_names) - - for pipeline in filtered_pipelines: - print("Update template branch for pipeline '{pipeline}'... ".format(pipeline=pipeline['name'])) - update_template_branch_for_pipeline(pipeline) - print("Trying to open pull request for pipeline {}...".format(pipeline['name'])) - create_pullrequest_if_update_sucessful(pipeline) - - for pipeline, exception in sync_errors: - print("WARNING!!!! Sync for pipeline {name} failed.".format(name=pipeline)) - print(exception) - - for pipeline, return_code, content in pr_errors: - print("WARNING!!!! Pull-request for pipeline \'{pipeline}\' failed," - " got return code {return_code}." - .format(pipeline=pipeline, return_code=return_code)) - print(content) - - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/bin/syncutils/__init__.py b/bin/syncutils/__init__.py deleted file mode 100644 index d3f5a12fa..000000000 --- a/bin/syncutils/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/bin/syncutils/template.py b/bin/syncutils/template.py deleted file mode 100644 index 69fc1bca9..000000000 --- a/bin/syncutils/template.py +++ /dev/null @@ -1,87 +0,0 @@ -import tempfile -from syncutils import utils -import git -import os -import shutil - -import nf_core.create - -TEMPLATE_BRANCH = "TEMPLATE" - - -class NfcoreTemplate: - """Updates the template content of an nf-core pipeline in - its `TEMPLATE` branch. - - Args: - pipeline: The pipeline name - - branch: The template branch name, default=`TEMPLATE` - - token: GitHub auth token - """ - def __init__(self, pipeline, branch=TEMPLATE_BRANCH, repo_url=""): - """Basic constructor - """ - self.pipeline = pipeline - self.repo_url = repo_url - self.branch = branch - self.tmpdir = tempfile.mkdtemp() - self.templatedir = tempfile.mkdtemp() - self.repo = git.Repo.clone_from(self.repo_url, self.tmpdir) - assert self.repo - - def sync(self): - """Execute the template update. - """ - context = self.context_from_nextflow(nf_project_dir=self.tmpdir) - self.update_child_template(self.templatedir, self.tmpdir, context=context) - self.commit_changes() - self.push_changes() - - def context_from_nextflow(self, nf_project_dir): - """Fetch a Nextflow pipeline's config settings. - - Returns: A cookiecutter-readable context (Python dictionary) - """ - # Check if we are on "master" (main pipeline code) - if self.repo.active_branch.name != "dev": - self.repo.git.checkout("origin/dev", b="dev") - - # Fetch the config variables from the Nextflow pipeline - config = utils.fetch_wf_config(wf_path=nf_project_dir) - - # Checkout again to configured template branch - self.repo.git.checkout("origin/{branch}".format(branch=self.branch), - b="{branch}".format(branch=self.branch)) - - return utils.create_context(config) - - def update_child_template(self, templatedir, target_dir, context=None): - """Apply the changes of the cookiecutter template - to the pipelines template branch. - """ - # Clear the pipeline's template branch content - for f in os.listdir(self.tmpdir): - if f == ".git": - continue - try: - shutil.rmtree(os.path.join(target_dir, f)) - except: - os.remove(os.path.join(target_dir, f)) - # Create the new template structure - nf_core.create.PipelineCreate( - name=context.get('pipeline_name'), - description=context.get('pipeline_short_description'), - new_version=context.get('version'), - no_git=True, - force=True, - outdir=target_dir, - author=context.get('author') - ).init_pipeline() - - def commit_changes(self): - """Commits the changes of the new template to the current branch. - """ - self.repo.git.add(A=True) - self.repo.index.commit("Update nf-core pipeline template.") - - def push_changes(self): - self.repo.git.push() diff --git a/bin/syncutils/utils.py b/bin/syncutils/utils.py deleted file mode 100644 index 8d3ac94e0..000000000 --- a/bin/syncutils/utils.py +++ /dev/null @@ -1,64 +0,0 @@ -import erro -import os -import requests -import subprocess - - -def fetch_wf_config(wf_path): - """ - Use nextflow to retrieve the nf configuration variables from a workflow - """ - config = dict() - # Call `nextflow config` and pipe stderr to /dev/null - try: - with open(os.devnull, 'w') as devnull: - nfconfig_raw = subprocess.check_output(['nextflow', 'config', '-flat', wf_path], stderr=devnull) - except OSError as e: - if e.errno == errno.ENOENT: - raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") - except subprocess.CalledProcessError as e: - raise AssertionError("`nextflow config` returned non-zero error code: %s,\n %s", e.returncode, e.output) - else: - for l in nfconfig_raw.splitlines(): - ul = l.decode() - k, v = ul.split(' = ', 1) - config[k] = v.replace("\'", "").replace("\"", "") - return config - - -def create_context(config): - """Consumes a flat Nextflow config file and will create - a context dictionary with information for the nf-core template creation. - - Returns: A dictionary with: - { - 'pipeline_name': '' - 'pipeline_short_description': '' - 'version': '' - } - """ - context = {} - context["pipeline_name"] = config.get("manifest.name") if config.get("manifest.name") else get_name_from_url(config.get("manifest.homePage")) - context["pipeline_short_description"] = config.get("manifest.description") - context["version"] = config.get("manifest.version") if config.get("manifest.version") else config.get("params.version") - context["author"] = config.get("manifest.author") if config.get("manifest.author") else "No author provided" - return context - - -def get_name_from_url(url): - return url.split("/")[-1] if url else "" - - -def repos_without_template_branch(pipeline_names): - pipelines_without_template = [] - for pipeline in pipeline_names: - api_call = "https://api.github.com/repos/nf-core/{}/branches".format(pipeline) - print("Fetching branch information for nf-core/{}...".format(pipeline)) - res = requests.get(api_call) - branch_list = res.json() - branch_names = [branch["name"] for branch in branch_list] - if "TEMPLATE" not in branch_names: - pipelines_without_template.append(pipeline) - print("WARNING: nf-core/{} had no TEMPLATE branch!".format(pipeline)) - - return pipelines_without_template From 40c754a3a77b3c754315114f119e2ce43ee41870 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 14 Oct 2019 23:10:55 +0200 Subject: [PATCH 069/224] Removed blacklist file --- bin/blacklist.json | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 bin/blacklist.json diff --git a/bin/blacklist.json b/bin/blacklist.json deleted file mode 100644 index 430c91d3d..000000000 --- a/bin/blacklist.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "pipelines": [ - "exoseq", - "neutronstar", - "vipr", - "neutronstar", - "proteomicslfq", - "clinvap", - "bactmap", - "kmermaid", - "crisprvar" - ] -} From 5e2c6d9c2661ff51123eb3dcf27dfef7a42320e8 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 14 Oct 2019 23:12:08 +0200 Subject: [PATCH 070/224] Rename API build script --- .travis.yml | 2 +- bin/{push.sh => build_api_docs.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename bin/{push.sh => build_api_docs.sh} (100%) diff --git a/.travis.yml b/.travis.yml index f5a2e545e..14c0c74b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ jobs: include: - stage: docs generation if: (branch = "master" OR branch = "dev") AND type = push AND repo = nf-core/tools - script: bash ./bin/push.sh + script: bash ./bin/build_api_docs.sh deploy: provider: pypi diff --git a/bin/push.sh b/bin/build_api_docs.sh similarity index 100% rename from bin/push.sh rename to bin/build_api_docs.sh From bc85f13c77984761b062cf8af46de7649e8f4296 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 14 Oct 2019 23:18:40 +0200 Subject: [PATCH 071/224] 'nf-core sync --all' instead of 'nf-core sync-all' --- README.md | 7 +++---- scripts/nf-core | 45 ++++++++++++++++++++++----------------------- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 1cc40174c..841e72428 100644 --- a/README.md +++ b/README.md @@ -488,14 +488,13 @@ These can be created at [https://github.com/settings/tokens](https://github.com/ Supply this using the `--auth-token` flag, or setting it as the environment variable `NF_CORE_BOT`: `export NF_CORE_BOT=my_auth_token`. -## Sync all nf-core pipelines - -This command is used by the nf-core/tools release automation to synchronise all nf-core pipelines +Finally, if `--all` is supplied, then the command attempts to pull and synchronise all nf-core workflows. +This is used by the nf-core/tools release automation to synchronise all nf-core pipelines with the newest version of the template. It requires authentication as either the nf-core-bot account or as an nf-core administrator. ```console -$ nf-core sync-all +$ nf-core sync --all ,--./,-. ___ __ __ __ ___ /,-._.--~\ diff --git a/scripts/nf-core b/scripts/nf-core index ed48c86d4..101446fb3 100755 --- a/scripts/nf-core +++ b/scripts/nf-core @@ -280,7 +280,6 @@ def bump_version(pipeline_dir, new_version, nextflow): @click.argument( 'pipeline_dir', type = click.Path(exists=True), - required = True, metavar = "" ) @click.option( @@ -315,31 +314,31 @@ def bump_version(pipeline_dir, new_version, nextflow): type = str, help = 'GitHub API personal access token.' ) -def sync(pipeline_dir, make_template_branch, from_branch, pull_request, username, repository, auth_token): - """ Sync a pipeline TEMPLATE branch with the nf-core template""" - sync_obj = nf_core.sync.PipelineSync(pipeline_dir, make_template_branch, from_branch, pull_request) - try: - sync_obj.sync() - except (nf_core.sync.SyncException, nf_core.sync.PullRequestException) as e: - logging.error(e) - sys.exit(1) - -@nf_core_cli.command('sync-all', help_priority=9) -@click.option( - '-u', '--username', - type = str, - default = 'nf-core-bot', - help = 'GitHub username for the PRs.' -) @click.option( - '-a', '--auth-token', - type = str, - help = 'GitHub API personal access token.' + '--all', + is_flag = True, + default = False, + help = "Sync template for all nf-core pipelines." ) -def sync(username, auth_token): - """ Sync template for all nf-core pipelines. nf-core admins only.""" - nf_core.sync.sync_all_pipelines() +def sync(pipeline_dir, make_template_branch, from_branch, pull_request, username, repository, auth_token, all): + """ Sync a pipeline TEMPLATE branch with the nf-core template""" + + # Pull and sync all nf-core pipelines + if all: + nf_core.sync.sync_all_pipelines() + else: + # Manually check for the required parameter + if not pipeline_dir: + logging.error(" is required") + sys.exit(1) + # Sync the given pipeline dir + sync_obj = nf_core.sync.PipelineSync(pipeline_dir, make_template_branch, from_branch, pull_request) + try: + sync_obj.sync() + except (nf_core.sync.SyncException, nf_core.sync.PullRequestException) as e: + logging.error(e) + sys.exit(1) if __name__ == '__main__': From d475f56f9a0a8bcb8f5e40956fd20a21e662e588 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Oct 2019 17:28:15 +0100 Subject: [PATCH 072/224] Fix mentions of nf-core sync-all --- .travis.yml | 2 +- README.md | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 14c0c74b3..26b28abab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,4 +57,4 @@ deploy: # Sync pipelines with possible nf-core template changes after_deploy: - - nf-core sync-all + - nf-core sync --all diff --git a/README.md b/README.md index 841e72428..58ab7ff4a 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,7 @@ A python package with helper tools for the nf-core community. * [`nf-core create` - Create a new workflow from the nf-core template](#creating-a-new-workflow) * [`nf-core lint` - Check pipeline code against nf-core guidelines](#linting-a-workflow) * [`nf-core bump-version` - Update nf-core pipeline version number](#bumping-a-pipeline-version-number) -* [`nf-core sync` - Synchronise a pipeline TEMPLATE branch](#sync-a-pipeline-with-the-template) -* [`nf-core sync-all` - Sync all nf-core pipelines and trigger PRs](#sync-all-nf-core-pipelines) +* [`nf-core sync` - Synchronise pipeline TEMPLATE branches](#sync-a-pipeline-with-the-template) The nf-core tools package is written in Python and can be imported and used within other packages. For documentation of the internal Python functions, please refer to the [Tools Python API docs](https://nf-co.re/tools-docs/). From 42fe7178cddcd1830554e31f56563c2b8d6f0fee Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 29 Oct 2019 10:23:13 +0100 Subject: [PATCH 073/224] Everything should be travis-ci.com now --- .github/CONTRIBUTING.md | 2 +- README.md | 2 +- docs/lint_errors.md | 2 +- .../{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index f2e172926..6f816f973 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -38,7 +38,7 @@ The HTML will then be generated in `docs/api/_build/html`. ## Tests -When you create a pull request with changes, [Travis CI](https://travis-ci.org/) will run automatic tests. +When you create a pull request with changes, [Travis CI](https://travis-ci.com/) will run automatic tests. Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then. There are two types of tests that run: diff --git a/README.md b/README.md index f3d81a59d..c6869ffde 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ![nf-core/tools](docs/images/nfcore-tools_logo.png) -[![Build Status](https://travis-ci.org/nf-core/tools.svg?branch=master)](https://travis-ci.org/nf-core/tools) +[![Build Status](https://travis-ci.com/nf-core/tools.svg?branch=master)](https://travis-ci.com/nf-core/tools) [![codecov](https://codecov.io/gh/nf-core/tools/branch/master/graph/badge.svg)](https://codecov.io/gh/nf-core/tools) [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg?style=flat-square)](http://bioconda.github.io/recipes/nf-core/README.html) diff --git a/docs/lint_errors.md b/docs/lint_errors.md index 141c5c2c5..afd986ae5 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -11,7 +11,7 @@ nf-core pipelines should adhere to a common file structure for consistency. The * `Dockerfile` * A docker build script to generate a docker image with the required software * `.travis.yml` or `circle.yml` - * A config file for automated continuous testing with either [Travis CI](https://travis-ci.org/) or [Circle CI](https://circleci.com/) + * A config file for automated continuous testing with either [Travis CI](https://travis-ci.com/) or [Circle CI](https://circleci.com/) * `LICENSE`, `LICENSE.md`, `LICENCE.md` or `LICENCE.md` * The MIT licence. Copy from [here](https://raw.githubusercontent.com/nf-core/tools/master/LICENSE). * `README.md` diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md index 63ec44141..fcaab6b1e 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md @@ -26,7 +26,7 @@ If you're not used to this workflow with git, you can start with some [basic doc ## Tests -When you create a pull request with changes, [Travis CI](https://travis-ci.org/) will run automatic tests. +When you create a pull request with changes, [Travis CI](https://travis-ci.com/) will run automatic tests. Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then. There are typically two types of tests that run: From cc365f3d45cd72f4aa07458452346be4025008c0 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 29 Oct 2019 10:24:23 +0100 Subject: [PATCH 074/224] UPdate changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d131dcaf7..a1171a9bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ ### Other * Updated Base Dockerfile to Conda 4.7.10 +* Entirely switched from Travis-Ci.org to Travis-Ci.com for template and tools ## v1.7 From ac7453a198b3325f03e999c53caba964c35cf3fe Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 1 Nov 2019 13:25:44 +0000 Subject: [PATCH 075/224] Move params section to nextflow.config --- .../{{cookiecutter.name_noslash}}/conf/base.config | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/base.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/base.config index 3c5aac463..2a2322c95 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/base.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/base.config @@ -47,12 +47,5 @@ process { withName:get_software_versions { cache = false } -} - -params { - // Defaults only, expecting to be overwritten - max_memory = 128.GB - max_cpus = 16 - max_time = 240.h - igenomes_base = 's3://ngi-igenomes/igenomes/' + } From 224f2b7fd7785c768c73c97850362bd2bb240db5 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 1 Nov 2019 13:25:51 +0000 Subject: [PATCH 076/224] Move params section to nextflow.config --- .../{{cookiecutter.name_noslash}}/nextflow.config | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config index 9d21dc0d7..246c3ea17 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config @@ -24,7 +24,7 @@ params { plaintext_email = false monochrome_logs = false help = false - igenomes_base = "./iGenomes" + igenomes_base = 's3://ngi-igenomes/igenomes/' tracedir = "${params.outdir}/pipeline_info" awsqueue = false awsregion = 'eu-west-1' @@ -35,6 +35,12 @@ params { config_profile_description = false config_profile_contact = false config_profile_url = false + + // Defaults only, expecting to be overwritten + max_memory = 128.GB + max_cpus = 16 + max_time = 240.h + } // Container slug. Stable releases should specify release tag! From 86a9920057f580c80367316e81704042a69320ac Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 1 Nov 2019 13:29:29 +0000 Subject: [PATCH 077/224] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1171a9bb..8629ec699 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ * Add `autoMounts=true` to default singularity profile * Add in `markdownlint` checks that were being ignored by default * Disable ansi logging in the travis CI tests. +* Move `params`section from `base.config` to `nextflow.config` ### Other From 2253590ad06920821402ec46a0fdf3661288a940 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 1 Nov 2019 18:24:35 +0000 Subject: [PATCH 078/224] Bump NF to 19.10.0 in template --- .../{{cookiecutter.name_noslash}}/.travis.yml | 2 +- .../{{cookiecutter.name_noslash}}/README.md | 2 +- .../{{cookiecutter.name_noslash}}/nextflow.config | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml index db1995103..1a70b7361 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml @@ -30,7 +30,7 @@ install: - sudo apt-get install npm && npm install -g markdownlint-cli env: - - NXF_VER='0.32.0' # Specify a minimum NF version that should be tested and work + - NXF_VER='19.10.0' # Specify a minimum NF version that should be tested and work - NXF_VER='' # Plus: get the latest NF version and check that it works script: diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md index 51ea8ab3e..76c8b23a7 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md @@ -3,7 +3,7 @@ **{{ cookiecutter.description }}**. [![Build Status](https://travis-ci.com/{{ cookiecutter.name }}.svg?branch=master)](https://travis-ci.com/{{ cookiecutter.name }}) -[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.32.0-brightgreen.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A519.10.0-brightgreen.svg)](https://www.nextflow.io/) [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](http://bioconda.github.io/) [![Docker](https://img.shields.io/docker/automated/{{ cookiecutter.name_docker }}.svg)](https://hub.docker.com/r/{{ cookiecutter.name_docker }}) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config index 246c3ea17..6d84edee4 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config @@ -77,6 +77,11 @@ if (!params.igenomes_ignore) { includeConfig 'conf/igenomes.config' } +// Export this variable to prevent local Python libraries from conflicting with those in the container +env { + PYTHONNOUSERSITE = 1 +} + // Capture exit codes from upstream processes when piping process.shell = ['/bin/bash', '-euo', 'pipefail'] @@ -103,7 +108,7 @@ manifest { homePage = 'https://github.com/{{ cookiecutter.name }}' description = '{{ cookiecutter.description }}' mainScript = 'main.nf' - nextflowVersion = '>=0.32.0' + nextflowVersion = '>=19.10.0' version = '{{ cookiecutter.version }}' } From a53b0b8fd5a466066ebfa8a0b286f6fd21a7129e Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 1 Nov 2019 18:25:15 +0000 Subject: [PATCH 079/224] Bump NF to 19.10.0 everywhere else --- tests/lint_examples/minimal_working_example/.travis.yml | 2 +- tests/lint_examples/minimal_working_example/README.md | 2 +- tests/lint_examples/minimal_working_example/nextflow.config | 2 +- tests/test_bump_version.py | 4 ++-- tests/test_lint.py | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/lint_examples/minimal_working_example/.travis.yml b/tests/lint_examples/minimal_working_example/.travis.yml index 85b43c60d..dbdd8e191 100644 --- a/tests/lint_examples/minimal_working_example/.travis.yml +++ b/tests/lint_examples/minimal_working_example/.travis.yml @@ -32,7 +32,7 @@ install: env: matrix: - - NXF_VER=0.32.0 + - NXF_VER=19.10.0 - NXF_VER='' script: diff --git a/tests/lint_examples/minimal_working_example/README.md b/tests/lint_examples/minimal_working_example/README.md index 76ff3403d..838a6faef 100644 --- a/tests/lint_examples/minimal_working_example/README.md +++ b/tests/lint_examples/minimal_working_example/README.md @@ -1,5 +1,5 @@ # The pipeline readme file -[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.32.0-brightgreen.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A519.10.0-brightgreen.svg)](https://www.nextflow.io/) [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](http://bioconda.github.io/) diff --git a/tests/lint_examples/minimal_working_example/nextflow.config b/tests/lint_examples/minimal_working_example/nextflow.config index b0f3a01f5..e1bee83d4 100644 --- a/tests/lint_examples/minimal_working_example/nextflow.config +++ b/tests/lint_examples/minimal_working_example/nextflow.config @@ -37,6 +37,6 @@ manifest { homePage = 'https://github.com/nf-core/tools' description = 'Minimal working example pipeline' mainScript = 'main.nf' - nextflowVersion = '>=0.32.0' + nextflowVersion = '>=19.10.0' version = '0.4' } diff --git a/tests/test_bump_version.py b/tests/test_bump_version.py index 4c5191f30..ae9dbe1b5 100644 --- a/tests/test_bump_version.py +++ b/tests/test_bump_version.py @@ -55,8 +55,8 @@ def test_multiple_patterns_found(datafiles): def test_successfull_nextflow_version_bump(datafiles): lint_obj = nf_core.lint.PipelineLint(str(datafiles)) lint_obj.pipeline_name = 'tools' - lint_obj.config['manifest.nextflowVersion'] = '0.32.0' + lint_obj.config['manifest.nextflowVersion'] = '19.10.0' nf_core.bump_version.bump_nextflow_version(lint_obj, '0.40') lint_obj_new = nf_core.lint.PipelineLint(str(datafiles)) lint_obj_new.check_nextflow_config() - assert lint_obj_new.config['manifest.nextflowVersion'] == "'>=0.40'" \ No newline at end of file + assert lint_obj_new.config['manifest.nextflowVersion'] == "'>=0.40'" diff --git a/tests/test_lint.py b/tests/test_lint.py index 273f2232a..16366e5f5 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -131,7 +131,7 @@ def test_config_variable_error(self): def test_ci_conf_pass(self): """Tests that the continous integration config checks work with a good example""" lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.minNextflowVersion = '0.32.0' + lint_obj.minNextflowVersion = '19.10.0' lint_obj.check_ci_config() expectations = {"failed": 0, "warned": 0, "passed": 3} self.assess_lint_status(lint_obj, **expectations) @@ -168,7 +168,7 @@ def test_missing_license_example(self): def test_readme_pass(self): """Tests that the pipeline README file checks work with a good example""" lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.minNextflowVersion = '0.32.0' + lint_obj.minNextflowVersion = '19.10.0' lint_obj.files = ['environment.yml'] lint_obj.check_readme() expectations = {"failed": 0, "warned": 0, "passed": 2} From bbc7fd421cd0b29a098d2d58d33f4801c2013d67 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 1 Nov 2019 18:28:55 +0000 Subject: [PATCH 080/224] Update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8629ec699..82e1b6de0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ * Add in `markdownlint` checks that were being ignored by default * Disable ansi logging in the travis CI tests. * Move `params`section from `base.config` to `nextflow.config` +* Use `env` scope to export `PYTHONNOUSERSITE` in `nextflow.config` to prevent conflicts with host Python environment. +* Bump minimum Nextflow version to `19.10.0` - required to properly use `env` scope in `nextflow.config` ### Other From 3d4ff94b2ee18e1c4de89b82d3964b0e30aa546e Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 4 Nov 2019 11:19:41 +0000 Subject: [PATCH 081/224] Add @ewels comments to Dockerfile --- .../{{cookiecutter.name_noslash}}/Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile index 5c4e2dd93..51149a641 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile @@ -2,7 +2,12 @@ FROM nfcore/base:{{ 'dev' if 'dev' in cookiecutter.nf_core_version else cookiecu LABEL authors="{{ cookiecutter.author }}" \ description="Docker image containing all requirements for {{ cookiecutter.name }} pipeline" +# Install the conda environment COPY environment.yml / RUN conda env create -f /environment.yml && conda clean -a -RUN conda env export --name {{ cookiecutter.name_noslash }}-{{ cookiecutter.version }} > {{ cookiecutter.name_noslash }}-{{ cookiecutter.version }}.yml + +# Add conda installation dir to PATH (instead of doing 'conda activate') ENV PATH /opt/conda/envs/{{ cookiecutter.name_noslash }}-{{ cookiecutter.version }}/bin:$PATH + +# Dump the details of the installed packages to a file for posterity +RUN conda env export --name {{ cookiecutter.name_noslash }}-{{ cookiecutter.version }} > {{ cookiecutter.name_noslash }}-{{ cookiecutter.version }}.yml From c5aef4c2d5863795709a8963edc5e6e1a413a8c7 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 6 Nov 2019 16:23:49 +0100 Subject: [PATCH 082/224] Fix #446 --- CHANGELOG.md | 1 + nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82e1b6de0..a8613dff2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Template * Fixed incorrect paths in iGenomes config as described in issue [#418](https://github.com/nf-core/tools/issues/418) +* Fixed [incorrect usage of non-existant parameter](https://github.com/nf-core/tools/issues/446) in the template * Add UCSC genomes to `igenomes.config` and add paths to all genome indices * Change `maxMultiqcEmailFileSize` parameter to `max_multiqc_email_size` * Export conda environment in Docker file [#349](https://github.com/nf-core/tools/issues/349) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index 0c7edf721..8ebb6dda6 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -85,7 +85,7 @@ if (workflow.profile == 'awsbatch') { // related: https://github.com/nextflow-io/nextflow/issues/813 if (!params.outdir.startsWith('s3:')) exit 1, "Outdir not on S3 - specify S3 Bucket to run on AWSBatch!" // Prevent trace files to be stored on S3 since S3 does not support rolling files. - if (workflow.tracedir.startsWith('s3:')) exit 1, "Specify a local tracedir or run without trace! S3 cannot be used for tracefiles." + if (params.tracedir.startsWith('s3:')) exit 1, "Specify a local tracedir or run without trace! S3 cannot be used for tracefiles." } // Stage config files From 57e71be3444b62b9d12dd735acda837bcc05eba8 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 6 Nov 2019 16:26:00 +0100 Subject: [PATCH 083/224] Use contains for AWSBatch check --- nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index 8ebb6dda6..050a0c19f 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -78,7 +78,7 @@ if (!(workflow.runName ==~ /[a-z]+_[a-z]+/)) { custom_runName = workflow.runName } -if (workflow.profile == 'awsbatch') { +if (workflow.profile.contains('awsbatch')) { // AWSBatch sanity checking if (!params.awsqueue || !params.awsregion) exit 1, "Specify correct --awsqueue and --awsregion parameters on AWSBatch!" // Check outdir paths to be S3 buckets if running on AWSBatch From bae5f33095516a7afe495745b740405137ef18e7 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Thu, 7 Nov 2019 09:37:44 +0000 Subject: [PATCH 084/224] Fix tpyo --- .../.github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/PULL_REQUEST_TEMPLATE.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/PULL_REQUEST_TEMPLATE.md index ef7cae0e0..3600b8f1d 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/PULL_REQUEST_TEMPLATE.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/PULL_REQUEST_TEMPLATE.md @@ -1,4 +1,4 @@ -Many thanks to contributing to {{ cookiecutter.name }}! +Many thanks for contributing to {{ cookiecutter.name }}! Please fill in the appropriate checklist below (delete whatever is not relevant). These are the most common things requested on pull requests (PRs). From 4dfe2a3f26bac7a29108ca9f2eeaceb017add4d5 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Thu, 7 Nov 2019 09:38:10 +0000 Subject: [PATCH 085/224] Fix tpyo --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 487fdf62e..9ad9e59a7 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,4 +1,4 @@ -Many thanks to contributing to nf-core/tools! +Many thanks for contributing to nf-core/tools! Please fill in the appropriate checklist below (delete whatever is not relevant). These are the most common things requested on pull requests (PRs). From 1e3afd8b9eb087eb2bec4bcb1ef735e1786c2aaf Mon Sep 17 00:00:00 2001 From: drpatelh Date: Thu, 7 Nov 2019 09:56:30 +0000 Subject: [PATCH 086/224] Update CHANGELOG --- .../{{cookiecutter.name_noslash}}/CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CHANGELOG.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CHANGELOG.md index 1b11643be..f1067c4ce 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CHANGELOG.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CHANGELOG.md @@ -1,5 +1,17 @@ # {{ cookiecutter.name }}: Changelog +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). + ## v{{ cookiecutter.version }} - [date] Initial release of {{ cookiecutter.name }}, created with the [nf-core](http://nf-co.re/) template. + +### `Added` + +### `Fixed` + +### `Dependencies` + +### `Deprecated` + From 435239b67e6eedd3f241da21ca43aa6eba405779 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Thu, 7 Nov 2019 09:57:27 +0000 Subject: [PATCH 087/224] Update CHANGELOG --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8613dff2..39163a7e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,8 @@ * Move `params`section from `base.config` to `nextflow.config` * Use `env` scope to export `PYTHONNOUSERSITE` in `nextflow.config` to prevent conflicts with host Python environment. * Bump minimum Nextflow version to `19.10.0` - required to properly use `env` scope in `nextflow.config` - +* Add link to [Semantic Versioning](http://semver.org/spec/v2.0.0.html) to CHANGELOG + ### Other * Updated Base Dockerfile to Conda 4.7.10 From 56b4de091ce4d6893441ff9376347a024384081a Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 8 Nov 2019 12:13:09 +0000 Subject: [PATCH 088/224] Fix Slack link --- .../{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md index fcaab6b1e..7833bc580 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md @@ -6,7 +6,7 @@ We try to manage the required tasks for {{ cookiecutter.name }} using GitHub iss However, don't be put off by this template - other more general issues and suggestions are welcome! Contributions to the code are even more welcome ;) -> If you need help using or modifying {{ cookiecutter.name }} then the best place to ask is on the pipeline channel on [Slack](https://nf-co.re/join/slack/). +> If you need help using or modifying {{ cookiecutter.name }} then the best place to ask is on the pipeline channel on [Slack](https://nf-co.re/join/slack). @@ -44,4 +44,4 @@ If there are any failures then the automated tests fail. These tests are run both with the latest available version of Nextflow and also the minimum required version that is stated in the pipeline code. ## Getting help -For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.name }}#documentation) and don't hesitate to get in touch on the [{{ cookiecutter.name }} pipeline channel](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) on [Slack](https://nf-co.re/join/slack/). \ No newline at end of file +For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.name }}#documentation) and don't hesitate to get in touch on the [{{ cookiecutter.name }} pipeline channel](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) on [Slack](https://nf-co.re/join/slack). From d527e883d2c6a1f34fd432800dc309dfa7018f4d Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 8 Nov 2019 12:13:19 +0000 Subject: [PATCH 089/224] Fix Slack link --- .../{{cookiecutter.name_noslash}}/CODE_OF_CONDUCT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CODE_OF_CONDUCT.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CODE_OF_CONDUCT.md index 1cda76009..cf930c8ac 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CODE_OF_CONDUCT.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CODE_OF_CONDUCT.md @@ -34,7 +34,7 @@ This Code of Conduct applies both within project spaces and in public spaces whe ## Enforcement -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team on [Slack](https://nf-co.re/join/slack/). The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team on [Slack](https://nf-co.re/join/slack). The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. From babe2271675b03a2a52abf3fa29cb169d60890fa Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 8 Nov 2019 12:13:28 +0000 Subject: [PATCH 090/224] Fix Slack link --- .../{{cookiecutter.name_noslash}}/docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index 11727fffe..c26833c8f 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -202,7 +202,7 @@ Wherever process-specific requirements are set in the pipeline, the default valu If you are likely to be running `nf-core` pipelines regularly it may be a good idea to request that your custom config file is uploaded to the `nf-core/configs` git repository. Before you do this please can you test that the config file works with your pipeline of choice using the `-c` parameter (see definition below). You can then create a pull request to the `nf-core/configs` repository with the addition of your config file, associated documentation file (see examples in [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs)), and amending [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) to include your custom profile. -If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack/). +If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack). ## AWS Batch specific parameters From 22eefc619412ec6508f5e043341d131f1cec7cf7 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 8 Nov 2019 13:17:43 +0000 Subject: [PATCH 091/224] Update CHANGELOG --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39163a7e6..f665e092d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,8 +22,8 @@ * Move `params`section from `base.config` to `nextflow.config` * Use `env` scope to export `PYTHONNOUSERSITE` in `nextflow.config` to prevent conflicts with host Python environment. * Bump minimum Nextflow version to `19.10.0` - required to properly use `env` scope in `nextflow.config` -* Add link to [Semantic Versioning](http://semver.org/spec/v2.0.0.html) to CHANGELOG - +* Add link to [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and [Semantic Versioning](http://semver.org/spec/v2.0.0.html) to CHANGELOG + ### Other * Updated Base Dockerfile to Conda 4.7.10 From a97048bfc5791b997e53988b3382ea50d68677ed Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 8 Nov 2019 17:15:01 +0100 Subject: [PATCH 092/224] Travis tests: Use nf-tower with nf-core@mailinator.com --- .../{{cookiecutter.name_noslash}}/.travis.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml index 1a70b7361..68f25d05c 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml @@ -30,8 +30,13 @@ install: - sudo apt-get install npm && npm install -g markdownlint-cli env: - - NXF_VER='19.10.0' # Specify a minimum NF version that should be tested and work - - NXF_VER='' # Plus: get the latest NF version and check that it works + # Tower token is to inspect runs on https://tower.nf + # Use public mailbox nf-core@mailinator.com to log in: https://www.mailinator.com/v3/index.jsp?zone=public&query=nf-core + # Specify a minimum NF version that should be tested and work + - NXF_VER='19.10.0' TOWER_ACCESS_TOKEN="1c1f493bc2703472d6f1b9f6fb9e9d117abab7b1" + # Plus: get the latest NF version and check that it works + - NXF_VER='' TOWER_ACCESS_TOKEN="1c1f493bc2703472d6f1b9f6fb9e9d117abab7b1" + script: # Lint the pipeline code @@ -39,4 +44,4 @@ script: # Lint the documentation - markdownlint ${TRAVIS_BUILD_DIR} -c ${TRAVIS_BUILD_DIR}/.github/markdownlint.yml # Run the pipeline with the test profile - - nextflow run ${TRAVIS_BUILD_DIR} -profile test,docker -ansi-log false + - nextflow run ${TRAVIS_BUILD_DIR} -profile test,docker -ansi-log false -name {{ cookiecutter.short_name }}-${TRAVIS_EVENT_TYPE}-${TRAVIS_PULL_REQUEST}-${TRAVIS_COMMIT:0:6}-test-description From d3611744224ae08a56d16cdfd0829064ae6bc180 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 8 Nov 2019 17:16:33 +0100 Subject: [PATCH 093/224] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8613dff2..68dad1e38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ * Move `params`section from `base.config` to `nextflow.config` * Use `env` scope to export `PYTHONNOUSERSITE` in `nextflow.config` to prevent conflicts with host Python environment. * Bump minimum Nextflow version to `19.10.0` - required to properly use `env` scope in `nextflow.config` +* Added support for nf-tower in the travis tests, using public mailbox nf-core@mailinator.com ### Other From d3ae7b1a3f89aba7fe6252fae1e65411539f08f4 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 8 Nov 2019 18:25:49 +0000 Subject: [PATCH 094/224] Remove blank line --- .../pipeline-template/{{cookiecutter.name_noslash}}/CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CHANGELOG.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CHANGELOG.md index f1067c4ce..fcbbfe48f 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CHANGELOG.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CHANGELOG.md @@ -14,4 +14,3 @@ Initial release of {{ cookiecutter.name }}, created with the [nf-core](http://nf ### `Dependencies` ### `Deprecated` - From 50a3de35c51bdabb6d09f4a9883bdbc5c0d31360 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 12 Nov 2019 22:42:21 +0100 Subject: [PATCH 095/224] Update wording in Dockerfile --- .../pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile index 51149a641..69d24cdc3 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/Dockerfile @@ -1,6 +1,6 @@ FROM nfcore/base:{{ 'dev' if 'dev' in cookiecutter.nf_core_version else cookiecutter.nf_core_version }} LABEL authors="{{ cookiecutter.author }}" \ - description="Docker image containing all requirements for {{ cookiecutter.name }} pipeline" + description="Docker image containing all software requirements for the {{ cookiecutter.name }} pipeline" # Install the conda environment COPY environment.yml / From ebde8946cb46ac9e36dcd5d6f86ba75ac4f8db9d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 12 Nov 2019 23:17:46 +0100 Subject: [PATCH 096/224] avoid duplicate workflow.container log workflow.container is mentioned twice as it is now in the initial summary log. Removed it from the final completion log messages. --- nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index 050a0c19f..8ca801854 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -290,7 +290,6 @@ workflow.onComplete { if (workflow.repository) email_fields['summary']['Pipeline repository Git URL'] = workflow.repository if (workflow.commitId) email_fields['summary']['Pipeline repository Git Commit'] = workflow.commitId if (workflow.revision) email_fields['summary']['Pipeline Git branch/tag'] = workflow.revision - if (workflow.container) email_fields['summary']['Docker image'] = workflow.container email_fields['summary']['Nextflow Version'] = workflow.nextflow.version email_fields['summary']['Nextflow Build'] = workflow.nextflow.build email_fields['summary']['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp From c7ae4bbeaa98c255aab84c27dc5182d5eb710486 Mon Sep 17 00:00:00 2001 From: phue Date: Sat, 16 Nov 2019 13:58:34 +0100 Subject: [PATCH 097/224] bump_version.py: bump version in conda env export At first, I added pattern matches for each of the occurences in the Dockerfile, however this looked kinda ugly because there were multiple 'Updating version in Dockerfile ..' lines printed. So I refactored the code a little to make sure there is only one logging.info line printed per file that was changed. While I was at it, I also added a bit of color to the output. --- nf_core/bump_version.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index c56c9f43a..415ab9fb9 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -8,6 +8,8 @@ import re import sys +import click + def bump_pipeline_version(lint_obj, new_version): """Bumps a pipeline version number. @@ -53,10 +55,10 @@ def bump_pipeline_version(lint_obj, new_version): nfconfig_newstr = "name: nf-core-{}-{}".format(lint_obj.pipeline_name.lower(), new_version) update_file_version("environment.yml", lint_obj, nfconfig_pattern, nfconfig_newstr) - # Update Dockerfile PATH - nfconfig_pattern = r"PATH\s+/opt/conda/envs/nf-core-{}-{}/bin:\$PATH".format(lint_obj.pipeline_name.lower(), current_version.replace('.',r'\.')) - nfconfig_newstr = "PATH /opt/conda/envs/nf-core-{}-{}/bin:$PATH".format(lint_obj.pipeline_name.lower(), new_version) - update_file_version("Dockerfile", lint_obj, nfconfig_pattern, nfconfig_newstr) + # Update Dockerfile ENV PATH and RUN conda env create + nfconfig_pattern = r"nf-core-{}-{}".format(lint_obj.pipeline_name.lower(), current_version.replace('.',r'\.')) + nfconfig_newstr = "nf-core-{}-{}".format(lint_obj.pipeline_name.lower(), new_version) + update_file_version("Dockerfile", lint_obj, nfconfig_pattern, nfconfig_newstr, allow_multiple=True) def bump_nextflow_version(lint_obj, new_version): @@ -113,14 +115,20 @@ def update_file_version(filename, lint_obj, pattern, newstr, allow_multiple=Fals content = fh.read() # Check that we have exactly one match - matches = re.findall(pattern, content) - if len(matches) == 0: + matches_pattern = re.findall("^.*{}.*$".format(pattern),content,re.MULTILINE) + if len(matches_pattern) == 0: raise SyntaxError("Could not find version number in {}: '{}'".format(filename, pattern)) - if len(matches) > 1 and not allow_multiple: + if len(matches_pattern) > 1 and not allow_multiple: raise SyntaxError("Found more than one version number in {}: '{}'".format(filename, pattern)) # Replace the match - logging.info("Updating version in {}\n - {}\n + {}".format(filename, matches[0], newstr)) new_content = re.sub(pattern, newstr, content) + matches_newstr = re.findall("^.*{}.*$".format(newstr),new_content,re.MULTILINE) + + logging.info("Updating version in {}\n".format(filename) + + click.style(" - {}\n".format("\n - ".join(matches_pattern).strip()), fg='red') + + click.style(" + {}\n".format("\n + ".join(matches_newstr).strip()), fg='green') + ) + with open(fn, 'w') as fh: fh.write(new_content) From 774c1791ea35529240def2d4e408fe095b54aa41 Mon Sep 17 00:00:00 2001 From: phue Date: Sat, 16 Nov 2019 14:14:07 +0100 Subject: [PATCH 098/224] update README.md --- README.md | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index c6869ffde..ccd411a8d 100644 --- a/README.md +++ b/README.md @@ -400,33 +400,32 @@ INFO: Changing version number: Current version number is '1.0dev' New version number will be '1.0' + INFO: Updating version in nextflow.config - version = '1.0dev' + version = '1.0' + INFO: Updating version in nextflow.config - - container = 'nfcore/mypipeline:dev' - + container = 'nfcore/mypipeline:1.0' + - process.container = 'nfcore/mypipeline:1.0dev' + + process.container = 'nfcore/mypipeline:1.0' + INFO: Updating version in .travis.yml - - docker tag nfcore/mypipeline:dev nfcore/mypipeline:latest - + docker tag nfcore/mypipeline:dev nfcore/mypipeline:1.0 + - - docker tag nfcore/mypipeline:dev nfcore/mypipeline:1.0dev + + - docker tag nfcore/mypipeline:dev nfcore/mypipeline:1.0 -INFO: Updating version in Singularity - - VERSION 1.0dev - + VERSION 1.0 INFO: Updating version in environment.yml - name: nf-core-mypipeline-1.0dev + name: nf-core-mypipeline-1.0 -INFO: Updating version in Dockerfile - - PATH /opt/conda/envs/nf-core-mypipeline-1.0dev/bin:$PATH - + PATH /opt/conda/envs/nf-core-mypipeline-1.0/bin:\$PATH -INFO: Updating version in Singularity - - PATH=/opt/conda/envs/nf-core-mypipeline-1.0dev/bin:$PATH - + PATH=/opt/conda/envs/nf-core-mypipeline-1.0/bin:\$PATH +INFO: Updating version in Dockerfile + - RUN conda env export --name nf-core-mypipeline-1.0dev > nf-core-mypipeline-1.0dev.yml + - ENV PATH /opt/conda/envs/nf-core-mypipeline-1.0dev/bin:$PATH + + RUN conda env export --name nf-core-mypipeline-1.0 > nf-core-mypipeline-1.0.yml + + ENV PATH /opt/conda/envs/nf-core-mypipeline-1.0/bin:$PATH ``` To change the required version of Nextflow instead of the pipeline version number, use the flag `--nextflow`. From b81680bf54deafc6bf3dbfb0909a84b18fd1e37e Mon Sep 17 00:00:00 2001 From: phue Date: Sat, 16 Nov 2019 14:17:54 +0100 Subject: [PATCH 099/224] update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81abe3fbb..b56462923 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Tools helper code +* `nf-core bump-version` now also bumps the version string of the exported conda environment in the Dockerfile * Updated Blacklist of synced pipelines ### Template From 1d22f332d3e82c326a7a2319cf64e792ac440e42 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Sun, 17 Nov 2019 12:26:55 +0100 Subject: [PATCH 100/224] Adapt linting to new patch branch check --- nf_core/lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index 5b65e5688..97c9d1243 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -455,7 +455,7 @@ def check_ci_config(self): with open(fn, 'r') as fh: ciconf = yaml.safe_load(fh) # Check that we have the master branch protection, but allow patch as well - travisMasterCheck = '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && ([ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ] || [ $TRAVIS_PULL_REQUEST_BRANCH = "patch" ]))' + travisMasterCheck = '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ]) || [ $TRAVIS_PULL_REQUEST_BRANCH = "patch" ]' try: assert(travisMasterCheck in ciconf.get('before_install', {})) except AssertionError: From bfb4421093fcccf4838b7a375c1dd0c6b3d69e4e Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Sun, 17 Nov 2019 12:28:34 +0100 Subject: [PATCH 101/224] Add changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81abe3fbb..d9e51de46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ * Updated Blacklist of synced pipelines +### Linting + +* Adjusted linting to enable `patch` branches from being tested + ### Template * Fixed incorrect paths in iGenomes config as described in issue [#418](https://github.com/nf-core/tools/issues/418) @@ -24,6 +28,7 @@ * Bump minimum Nextflow version to `19.10.0` - required to properly use `env` scope in `nextflow.config` * Added support for nf-tower in the travis tests, using public mailbox nf-core@mailinator.com * Add link to [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and [Semantic Versioning](http://semver.org/spec/v2.0.0.html) to CHANGELOG +* Adjusted `.travis.yml` checks to allow for `patch` branches to be tested ### Other From 5a7ec9549ae0d06c4a8471a2fbce2f5f4e31eea4 Mon Sep 17 00:00:00 2001 From: phue Date: Sun, 17 Nov 2019 12:36:08 +0100 Subject: [PATCH 102/224] add lint test for conda env export --- nf_core/lint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/lint.py b/nf_core/lint.py index 5b65e5688..730b003b2 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -759,6 +759,7 @@ def check_conda_dockerfile(self): "FROM nfcore/base:{}".format('dev' if 'dev' in nf_core.__version__ else nf_core.__version__), 'COPY environment.yml /', 'RUN conda env create -f /environment.yml && conda clean -a', + 'RUN conda env export --name {} > {}.yml'.format(self.conda_config['name'], self.conda_config['name']), 'ENV PATH /opt/conda/envs/{}/bin:$PATH'.format(self.conda_config['name']) ] From e5b3a204392ba11552f21e07090de77f8998b1aa Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Sun, 17 Nov 2019 12:36:52 +0100 Subject: [PATCH 103/224] Make minimal test working again --- tests/lint_examples/minimal_working_example/.travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lint_examples/minimal_working_example/.travis.yml b/tests/lint_examples/minimal_working_example/.travis.yml index dbdd8e191..4f7d5571b 100644 --- a/tests/lint_examples/minimal_working_example/.travis.yml +++ b/tests/lint_examples/minimal_working_example/.travis.yml @@ -11,7 +11,7 @@ matrix: before_install: # PRs to master are only ok if coming from dev branch - - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && ([ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ] || [ $TRAVIS_PULL_REQUEST_BRANCH = "patch" ]))' + - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ]) || [ $TRAVIS_PULL_REQUEST_BRANCH = "patch" ]' # Pull the docker image first so the test doesn't wait for this - docker pull nfcore/tools:dev # Fake the tag locally so that the pipeline runs properly From 5f08c4ce6a81eb0fa112c6094a58d520c19f661a Mon Sep 17 00:00:00 2001 From: phue Date: Sun, 17 Nov 2019 12:53:04 +0100 Subject: [PATCH 104/224] add conda env create to minimal_working_example --- tests/lint_examples/minimal_working_example/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/lint_examples/minimal_working_example/Dockerfile b/tests/lint_examples/minimal_working_example/Dockerfile index fc562dc1d..0b3f1d287 100644 --- a/tests/lint_examples/minimal_working_example/Dockerfile +++ b/tests/lint_examples/minimal_working_example/Dockerfile @@ -5,4 +5,5 @@ LABEL authors="phil.ewels@scilifelab.se" \ COPY environment.yml / RUN conda env create -f /environment.yml && conda clean -a +RUN conda env export --name nf-core-tools-0.4 > nf-core-tools-0.4.yml ENV PATH /opt/conda/envs/nf-core-tools-0.4/bin:$PATH From 98dd2c6e3eb28c7f434c15cfa803f05e2edec53d Mon Sep 17 00:00:00 2001 From: phue Date: Sun, 17 Nov 2019 13:02:44 +0100 Subject: [PATCH 105/224] expect 5 failures in test_conda_dockerfile_fail() --- tests/test_lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_lint.py b/tests/test_lint.py index 16366e5f5..151572e2b 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -316,7 +316,7 @@ def test_conda_dockerfile_fail(self): lint_obj.conda_config['name'] = 'nf-core-tools-0.4' lint_obj.dockerfile = ['fubar'] lint_obj.check_conda_dockerfile() - expectations = {"failed": 4, "warned": 0, "passed": 0} + expectations = {"failed": 5, "warned": 0, "passed": 0} self.assess_lint_status(lint_obj, **expectations) def test_conda_dockerfile_skip(self): From 384c764635d5ac1465f2b86712610692bbecb6ee Mon Sep 17 00:00:00 2001 From: phue Date: Sun, 17 Nov 2019 13:27:05 +0100 Subject: [PATCH 106/224] update lint_errors.md --- docs/lint_errors.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/lint_errors.md b/docs/lint_errors.md index afd986ae5..48aa89e79 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -201,11 +201,14 @@ to create the container. Such `Dockerfile`s can usually be very short, eg: ```Dockerfile FROM nfcore/base:1.7 +MAINTAINER Rocky Balboa LABEL authors="your@email.com" \ - description="Container image containing all requirements for nf-core/EXAMPLE pipeline" + description="Docker image containing all requirements for the nf-core mypipeline pipeline" COPY environment.yml / -RUN conda env update -n root -f /environment.yml && conda clean -a +RUN conda env create -f /environment.yml && conda clean -a +RUN conda env export --name nf-core-mypipeline-1.0 > nf-core-mypipeline-1.0.yml +ENV PATH /opt/conda/envs/nf-core-mypipeline-1.0/bin:$PATH ``` To enforce this minimal `Dockerfile` and check for common copy+paste errors, we require From e20f347545c35f8828b7a1f17b526389fdde8a38 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 19 Nov 2019 11:48:34 +0100 Subject: [PATCH 107/224] Template tweaks based on review of nf-core/methylseq#108 --- .../.github/CONTRIBUTING.md | 2 -- .../{{cookiecutter.name_noslash}}/.gitignore | 3 ++- .../docs/usage.md | 4 +++- .../{{cookiecutter.name_noslash}}/main.nf | 16 +++++++-------- .../nextflow.config | 20 +++++++++++-------- 5 files changed, 25 insertions(+), 20 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md index 7833bc580..6bef5a7ae 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md @@ -8,8 +8,6 @@ However, don't be put off by this template - other more general issues and sugge > If you need help using or modifying {{ cookiecutter.name }} then the best place to ask is on the pipeline channel on [Slack](https://nf-co.re/join/slack). - - ## Contribution workflow If you'd like to write some code for {{ cookiecutter.name }}, the standard workflow is as follows: diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.gitignore b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.gitignore index 5b54e3e6c..6354f3708 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.gitignore +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.gitignore @@ -3,5 +3,6 @@ work/ data/ results/ .DS_Store -tests/test_data +tests/ +testing/ *.pyc diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index c26833c8f..1c2c2c968 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -92,7 +92,9 @@ This version number will be logged in reports when you run the pipeline, so that ### `-profile` -Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. Note that multiple profiles can be loaded, for example: `-profile docker` - the order of arguments is important! +Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. +Note that multiple profiles can be loaded, for example: `-profile test,docker` - the order of arguments is important! +They are loaded in sequence, so later profiles can overwrite earlier profiles. If `-profile` is not specified at all the pipeline will be run locally and expects all software to be installed and available on the `PATH`. diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index 8ca801854..8b8e4fe5d 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -132,7 +132,7 @@ summary['Launch dir'] = workflow.launchDir summary['Working dir'] = workflow.workDir summary['Script dir'] = workflow.projectDir summary['User'] = workflow.userName -if (workflow.profile == 'awsbatch') { +if (workflow.profile.contains('awsbatch')) { summary['AWS Region'] = params.awsregion summary['AWS Queue'] = params.awsqueue } @@ -356,10 +356,10 @@ workflow.onComplete { def output_tf = new File(output_d, "pipeline_report.txt") output_tf.withWriter { w -> w << email_txt } - c_reset = params.monochrome_logs ? '' : "\033[0m"; - c_purple = params.monochrome_logs ? '' : "\033[0;35m"; c_green = params.monochrome_logs ? '' : "\033[0;32m"; + c_purple = params.monochrome_logs ? '' : "\033[0;35m"; c_red = params.monochrome_logs ? '' : "\033[0;31m"; + c_reset = params.monochrome_logs ? '' : "\033[0m"; if (workflow.stats.ignoredCount > 0 && workflow.success) { log.info "${c_purple}Warning, pipeline completed, but with errored process(es) ${c_reset}" @@ -379,15 +379,15 @@ workflow.onComplete { def nfcoreHeader() { // Log colors ANSI codes - c_reset = params.monochrome_logs ? '' : "\033[0m"; - c_dim = params.monochrome_logs ? '' : "\033[2m"; c_black = params.monochrome_logs ? '' : "\033[0;30m"; - c_green = params.monochrome_logs ? '' : "\033[0;32m"; - c_yellow = params.monochrome_logs ? '' : "\033[0;33m"; c_blue = params.monochrome_logs ? '' : "\033[0;34m"; - c_purple = params.monochrome_logs ? '' : "\033[0;35m"; c_cyan = params.monochrome_logs ? '' : "\033[0;36m"; + c_dim = params.monochrome_logs ? '' : "\033[2m"; + c_green = params.monochrome_logs ? '' : "\033[0;32m"; + c_purple = params.monochrome_logs ? '' : "\033[0;35m"; + c_reset = params.monochrome_logs ? '' : "\033[0m"; c_white = params.monochrome_logs ? '' : "\033[0;37m"; + c_yellow = params.monochrome_logs ? '' : "\033[0;33m"; return """ -${c_dim}--------------------------------------------------${c_reset}- ${c_green},--.${c_black}/${c_green},-.${c_reset} diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config index 6d84edee4..17fcd23e0 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config @@ -61,17 +61,21 @@ profiles { awsbatch { includeConfig 'conf/awsbatch.config' } conda { process.conda = "$baseDir/environment.yml" } debug { process.beforeScript = 'echo $HOSTNAME' } - docker { docker.enabled = true } - singularity { singularity.enabled = true - singularity.autoMounts = true } + docker { + docker.enabled = true + // Avoid this error: + // WARNING: Your kernel does not support swap limit capabilities or the cgroup is not mounted. Memory limited without swap. + // Testing this in nf-core after discussion here https://github.com/nf-core/tools/pull/351 + // once this is established and works well, nextflow might implement this behavior as new default. + docker.runOptions = '-u \$(id -u):\$(id -g)' + } + singularity { + singularity.enabled = true + singularity.autoMounts = true + } test { includeConfig 'conf/test.config' } } -// Avoid this error: -// WARNING: Your kernel does not support swap limit capabilities or the cgroup is not mounted. Memory limited without swap. -// Testing this in nf-core after discussion here https://github.com/nf-core/tools/pull/351, once this is established and works well, nextflow might implement this behavior as new default. -docker.runOptions = '-u \$(id -u):\$(id -g)' - // Load igenomes.config if required if (!params.igenomes_ignore) { includeConfig 'conf/igenomes.config' From 6b085fc9147a41f2bc2ebfcdf77a748a21caa0b0 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 19 Nov 2019 11:58:52 +0100 Subject: [PATCH 108/224] Changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81abe3fbb..57ab4205d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ * Use `env` scope to export `PYTHONNOUSERSITE` in `nextflow.config` to prevent conflicts with host Python environment. * Bump minimum Nextflow version to `19.10.0` - required to properly use `env` scope in `nextflow.config` * Added support for nf-tower in the travis tests, using public mailbox nf-core@mailinator.com -* Add link to [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and [Semantic Versioning](http://semver.org/spec/v2.0.0.html) to CHANGELOG +* Add link to [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and [Semantic Versioning](http://semver.org/spec/v2.0.0.html) to `CHANGELOG` ### Other From 311951a863b2a3d715fc50157c56b311982a2732 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 19 Nov 2019 13:43:26 +0100 Subject: [PATCH 109/224] Usage docs: Improve bit about -profile --- CHANGELOG.md | 1 + .../{{cookiecutter.name_noslash}}/docs/usage.md | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57ab4205d..7d4a06261 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ * Updated Base Dockerfile to Conda 4.7.10 * Entirely switched from Travis-Ci.org to Travis-Ci.com for template and tools +* Improved core documentation (`-profile`) ## v1.7 diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index 1c2c2c968..af120e321 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -93,10 +93,15 @@ This version number will be logged in reports when you run the pipeline, so that ### `-profile` Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. + +Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Conda) - see below. + +The pipeline also dynamically loads configurations from https://github.com/nf-core/configs when it runs, making multiple config profiles for various institutional clusters available at run time. For more information and to see if your system is available in these configs please see the [nf-core/configs documentaiton](https://github.com/nf-core/configs#documentation). + Note that multiple profiles can be loaded, for example: `-profile test,docker` - the order of arguments is important! They are loaded in sequence, so later profiles can overwrite earlier profiles. -If `-profile` is not specified at all the pipeline will be run locally and expects all software to be installed and available on the `PATH`. +If `-profile` is not specified, the pipeline will run locally and expect all software to be installed and available on the `PATH`. This is _not_ recommended. * `awsbatch` * A generic configuration profile to be used with AWS Batch. From 3187031b9b41b2bc98a1994cc555fddb8c546fd1 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 20 Nov 2019 10:24:19 +0100 Subject: [PATCH 110/224] nf-core list: ignore pre-releases --- CHANGELOG.md | 1 + nf_core/list.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81abe3fbb..0ed969800 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Tools helper code * Updated Blacklist of synced pipelines +* Ignore pre-releases in `nf-core list` ### Template diff --git a/nf_core/list.py b/nf_core/list.py index f77e159c5..609ad748b 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -238,8 +238,8 @@ def __init__(self, data): self.watchers_count = data.get('watchers_count') self.forks_count = data.get('forks_count') - # Placeholder vars for releases info - self.releases = data.get('releases') + # Placeholder vars for releases info (ignore pre-releases) + self.releases = [ r for r in data.get('releases', []) if r.get('published_at') is not None ] # Placeholder vars for local comparison self.local_wf = None From 3f38c11770866f5fa3c07ce88bd473a18546b81d Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 25 Nov 2019 10:54:06 +0100 Subject: [PATCH 111/224] Drop Singularity --- CHANGELOG.md | 1 + nf_core/lint.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ed969800..5a23a57f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ * Updated Blacklist of synced pipelines * Ignore pre-releases in `nf-core list` +* Lint for `Singularity` file [and remove it](https://github.com/nf-core/tools/issues/458) ### Template diff --git a/nf_core/lint.py b/nf_core/lint.py index 5b65e5688..eb22d0647 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -166,6 +166,7 @@ def lint_pipeline(self, release_mode=False): 'check_files_exist', 'check_licence', 'check_docker', + 'check_singularity', 'check_nextflow_config', 'check_ci_config', 'check_readme', @@ -277,6 +278,12 @@ def check_docker(self): self.failed.append((2, "Dockerfile check failed")) + def check_singularity(self): + """Checks whether a Singularity file exists and warns to remove that file then.""" + fn = os.path.join(self.path, "Singularity") + content = "" + self.failed.append((2, "Singularity file exists")) + def check_licence(self): """Checks licence file is MIT. From c41340c5ce475b96b55c54673bb36b9465f8592c Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 25 Nov 2019 10:55:43 +0100 Subject: [PATCH 112/224] Add Singularity file for failing tests --- tests/lint_examples/failing_example/Singularity | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/lint_examples/failing_example/Singularity diff --git a/tests/lint_examples/failing_example/Singularity b/tests/lint_examples/failing_example/Singularity new file mode 100644 index 000000000..02e88c804 --- /dev/null +++ b/tests/lint_examples/failing_example/Singularity @@ -0,0 +1 @@ +Nothing to be found here \ No newline at end of file From 31fa51467061f6849984ca4e4594bb11354f5e1e Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 25 Nov 2019 11:14:28 +0100 Subject: [PATCH 113/224] Fix tests and use an if --- nf_core/lint.py | 4 ++-- tests/test_lint.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index eb22d0647..9b0c1e9a5 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -281,8 +281,8 @@ def check_docker(self): def check_singularity(self): """Checks whether a Singularity file exists and warns to remove that file then.""" fn = os.path.join(self.path, "Singularity") - content = "" - self.failed.append((2, "Singularity file exists")) + if(path.exists(fn)): + self.failed.append((2, "Singularity file exists")) def check_licence(self): """Checks licence file is MIT. diff --git a/tests/test_lint.py b/tests/test_lint.py index 16366e5f5..2c22852cc 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -107,6 +107,13 @@ def test_mit_license_example_with_failed(self): bad_lint_obj.check_licence() expectations = {"failed": 1, "warned": 0, "passed": 0} self.assess_lint_status(bad_lint_obj, **expectations) + + def test_singularity_with_failed(self): + """Tests that Singularity file doesn't exist""" + bad_lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) + bad_lint_obj.check_singularity() + expectations = {"failed": 1, "warned": 0, "passed": 33} + self.assess_lint_status(bad_lint_obj, **expectations) def test_config_variable_example_pass(self): """Tests that config variable existence test works with good pipeline example""" From d5d2fd1773b9811aa994029a7a1561a4080c0ff9 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 25 Nov 2019 11:16:49 +0100 Subject: [PATCH 114/224] Nice error docs --- docs/lint_errors.md | 4 ++++ nf_core/lint.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/lint_errors.md b/docs/lint_errors.md index afd986ae5..84c02e227 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -227,3 +227,7 @@ The nf-core workflow template contains a number of comment lines with the follow ``` This lint test runs through all files in the pipeline and searches for these lines. + +## Error #11 - Singularity file found ##{#11} + +As we are relying on [Docker Hub](https://https://hub.docker.com/) instead of Singularity and all containers are automatically pulled from there, repositories should not have a `Singularity` file present. diff --git a/nf_core/lint.py b/nf_core/lint.py index 9b0c1e9a5..785c5b683 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -282,7 +282,7 @@ def check_singularity(self): """Checks whether a Singularity file exists and warns to remove that file then.""" fn = os.path.join(self.path, "Singularity") if(path.exists(fn)): - self.failed.append((2, "Singularity file exists")) + self.failed.append((11, "Singularity file exists")) def check_licence(self): """Checks licence file is MIT. From 8b4fc063068d16bcce5e7cc7cb82b90357a010cd Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 25 Nov 2019 11:44:49 +0100 Subject: [PATCH 115/224] Needs to be os --- nf_core/lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index 785c5b683..526d8be41 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -281,7 +281,7 @@ def check_docker(self): def check_singularity(self): """Checks whether a Singularity file exists and warns to remove that file then.""" fn = os.path.join(self.path, "Singularity") - if(path.exists(fn)): + if(os.path.exists(fn)): self.failed.append((11, "Singularity file exists")) def check_licence(self): From fe41b216376a8a4a6999e21d6875476df8f72eb6 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 25 Nov 2019 12:58:48 +0100 Subject: [PATCH 116/224] Its just testing the singularity --- tests/test_lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_lint.py b/tests/test_lint.py index 2c22852cc..6dd812934 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -112,7 +112,7 @@ def test_singularity_with_failed(self): """Tests that Singularity file doesn't exist""" bad_lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) bad_lint_obj.check_singularity() - expectations = {"failed": 1, "warned": 0, "passed": 33} + expectations = {"failed": 1, "warned": 0, "passed": 0} self.assess_lint_status(bad_lint_obj, **expectations) def test_config_variable_example_pass(self): From 2df5aff0d4a82a4e0c4db79409dce96e63daf0fa Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 26 Nov 2019 10:28:31 +0100 Subject: [PATCH 117/224] Fix markdown bare url --- .../{{cookiecutter.name_noslash}}/docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index af120e321..37d8d4a7a 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -96,7 +96,7 @@ Use this parameter to choose a configuration profile. Profiles can give configur Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Conda) - see below. -The pipeline also dynamically loads configurations from https://github.com/nf-core/configs when it runs, making multiple config profiles for various institutional clusters available at run time. For more information and to see if your system is available in these configs please see the [nf-core/configs documentaiton](https://github.com/nf-core/configs#documentation). +The pipeline also dynamically loads configurations from [https://github.com/nf-core/configs](https://github.com/nf-core/configs) when it runs, making multiple config profiles for various institutional clusters available at run time. For more information and to see if your system is available in these configs please see the [nf-core/configs documentaiton](https://github.com/nf-core/configs#documentation). Note that multiple profiles can be loaded, for example: `-profile test,docker` - the order of arguments is important! They are loaded in sequence, so later profiles can overwrite earlier profiles. From 5f37e3dde5762f39dde020ee59c590c67083b63e Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 26 Nov 2019 10:53:57 +0100 Subject: [PATCH 118/224] Conda - use Python 3 * Update base Dockerfile to use miniconda3, and update to conda 4.7.12 * Add Python 3.7 dependency to the template environment.yml file --- CHANGELOG.md | 6 ++++++ Dockerfile | 4 ++-- .../{{cookiecutter.name_noslash}}/environment.yml | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ed969800..3266c2548 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,12 @@ * Bump minimum Nextflow version to `19.10.0` - required to properly use `env` scope in `nextflow.config` * Added support for nf-tower in the travis tests, using public mailbox nf-core@mailinator.com * Add link to [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and [Semantic Versioning](http://semver.org/spec/v2.0.0.html) to CHANGELOG +* Add Python 3.7 dependency to the `environment.yml` file + +### Base Docker image + +* Use miniconda3 instead of miniconda for a Python 3k base environment +* Update conda version to 4.7.12 ### Other diff --git a/Dockerfile b/Dockerfile index 0a62abdf7..f8b138e9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -FROM continuumio/miniconda:4.7.10 +FROM continuumio/miniconda3:4.7.12 LABEL authors="phil.ewels@scilifelab.se,alexander.peltzer@qbic.uni-tuebingen.de" \ description="Docker image containing base requirements for the nfcore pipelines" # Install procps so that Nextflow can poll CPU usage -RUN apt-get update && apt-get install -y procps && apt-get clean -y \ No newline at end of file +RUN apt-get update && apt-get install -y procps && apt-get clean -y diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml index 537942ae7..3eb6a6ec1 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml @@ -6,6 +6,7 @@ channels: - bioconda - defaults dependencies: + - python=3.7 # TODO nf-core: Add required software dependencies here - bioconda::fastqc=0.11.8 - bioconda::multiqc=1.7 From 9b9a5af6cd4baa1e50537974478a8e8810af015a Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 26 Nov 2019 11:01:52 +0100 Subject: [PATCH 119/224] remove awsbatch profile from template --- .../conf/awsbatch.config | 18 ------------------ .../nextflow.config | 3 --- 2 files changed, 21 deletions(-) delete mode 100644 nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/awsbatch.config diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/awsbatch.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/awsbatch.config deleted file mode 100644 index 14af5866f..000000000 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/awsbatch.config +++ /dev/null @@ -1,18 +0,0 @@ -/* - * ------------------------------------------------- - * Nextflow config file for running on AWS batch - * ------------------------------------------------- - * Base config needed for running with -profile awsbatch - */ -params { - config_profile_name = 'AWSBATCH' - config_profile_description = 'AWSBATCH Cloud Profile' - config_profile_contact = 'Alexander Peltzer (@apeltzer)' - config_profile_url = 'https://aws.amazon.com/de/batch/' -} - -aws.region = params.awsregion -process.executor = 'awsbatch' -process.queue = params.awsqueue -executor.awscli = '/home/ec2-user/miniconda/bin/aws' -params.tracedir = './' diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config index 6d84edee4..a074b9f94 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config @@ -26,8 +26,6 @@ params { help = false igenomes_base = 's3://ngi-igenomes/igenomes/' tracedir = "${params.outdir}/pipeline_info" - awsqueue = false - awsregion = 'eu-west-1' igenomes_ignore = false custom_config_version = 'master' custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" @@ -58,7 +56,6 @@ try { } profiles { - awsbatch { includeConfig 'conf/awsbatch.config' } conda { process.conda = "$baseDir/environment.yml" } debug { process.beforeScript = 'echo $HOSTNAME' } docker { docker.enabled = true } From 8971342bcebda7ecef10fe83cfd894e3cc1ff84b Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 26 Nov 2019 11:03:38 +0100 Subject: [PATCH 120/224] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a9e0bae9..f33c8a004 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ * Added support for nf-tower in the travis tests, using public mailbox nf-core@mailinator.com * Add link to [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and [Semantic Versioning](http://semver.org/spec/v2.0.0.html) to CHANGELOG * Adjusted `.travis.yml` checks to allow for `patch` branches to be tested +* Remove awsbatch profile cf [#71 @ nf-core/configs](https://github.com/nf-core/configs/pull/71) ### Other From e64b234ebe44beae8c4a60d6c1028250d256cab8 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 26 Nov 2019 11:05:41 +0100 Subject: [PATCH 121/224] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f33c8a004..75769c8e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ * Added support for nf-tower in the travis tests, using public mailbox nf-core@mailinator.com * Add link to [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and [Semantic Versioning](http://semver.org/spec/v2.0.0.html) to CHANGELOG * Adjusted `.travis.yml` checks to allow for `patch` branches to be tested -* Remove awsbatch profile cf [#71 @ nf-core/configs](https://github.com/nf-core/configs/pull/71) +* Remove awsbatch profile cf [nf-core/configs#71](https://github.com/nf-core/configs/pull/71) ### Other From 3b2fbd13abdad626688ddfcd9096eab857c479b4 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 26 Nov 2019 15:24:41 +0100 Subject: [PATCH 122/224] Tests: Update expected test numbers --- tests/test_lint.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_lint.py b/tests/test_lint.py index 273f2232a..0402929f3 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -38,7 +38,7 @@ def pf(wd, path): pf(WD, 'lint_examples/license_incomplete_example')] # The maximum sum of passed tests currently possible -MAX_PASS_CHECKS = 59 +MAX_PASS_CHECKS = 61 # The additional tests passed for releases ADD_PASS_RELEASE = 1 @@ -112,14 +112,14 @@ def test_config_variable_example_pass(self): """Tests that config variable existence test works with good pipeline example""" good_lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) good_lint_obj.check_nextflow_config() - expectations = {"failed": 0, "warned": 0, "passed": 33} + expectations = {"failed": 0, "warned": 0, "passed": 35} self.assess_lint_status(good_lint_obj, **expectations) def test_config_variable_example_with_failed(self): """Tests that config variable existence test fails with bad pipeline example""" bad_lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) bad_lint_obj.check_nextflow_config() - expectations = {"failed": 19, "warned": 8, "passed": 6} + expectations = {"failed": 19, "warned": 8, "passed": 8} self.assess_lint_status(bad_lint_obj, **expectations) @pytest.mark.xfail(raises=AssertionError) From 859ac4f1c225427476d101f8cfbf004690ba9173 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 26 Nov 2019 15:56:40 +0100 Subject: [PATCH 123/224] nf-core download - update docs on README file --- README.md | 75 ++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index c6869ffde..3a8a8c813 100644 --- a/README.md +++ b/README.md @@ -209,7 +209,8 @@ Sometimes you may need to run an nf-core pipeline on a server or HPC system that To make this process easier and ensure accurate retrieval of correctly versioned code and software containers, we have written a download helper tool. Simply specify the name of the nf-core pipeline and it will be downloaded to your current working directory. -By default, the pipeline will just download the pipeline code. If you specify the flag `--singularity`, it will also download any singularity image files that are required. +By default, the pipeline will download the pipeline code and the [institutional nf-core/configs](https://github.com/nf-core/configs) files. +If you specify the flag `--singularity`, it will also download any singularity image files that are required. ```console $ nf-core download methylseq --singularity @@ -222,37 +223,83 @@ $ nf-core download methylseq --singularity INFO: Saving methylseq - Pipeline release: 1.0 + Pipeline release: 1.4 Pull singularity containers: Yes - Output directory: nf-core-methylseq-1.0 + Output file: nf-core-methylseq-1.4.tar.gz INFO: Downloading workflow files from GitHub +INFO: Downloading centralised configs from GitHub + INFO: Downloading 1 singularity container -nf-core-methylseq-1.0.simg [762.28MB] [####################################] 780573/780572 + +INFO: Building singularity image from dockerhub: docker://nfcore/methylseq:1.4 +INFO: Converting OCI blobs to SIF format +INFO: Starting build... +Getting image source signatures +.... +INFO: Creating SIF file... +INFO: Build complete: /domus/h1/phil/GitHub/nf-core/tools/testing/nf-core-methylseq-1.4/singularity-images/nf-core-methylseq-1.4.simg + +INFO: Compressing download.. + +INFO: Command to extract files: tar -xzf nf-core-methylseq-1.4.tar.gz + +INFO: MD5 checksum for nf-core-methylseq-1.4.tar.gz: f5c2b035619967bb227230bc3ec986c5 ``` -```console -$ tree -L 2 nf-core-methylseq-1.0/ +The tool automatically compresses all of the resulting file in to a `.tar.gz` archive. +You can choose other formats (`.tar.bz2`, `zip`) or to not compress (`none`) with the `-c`/`--compress` flag. +The console output provides the command you need to extract the files. + +Once uncompressed, you will see the following file structure for the downloaded pipeline: -nf-core-methylseq-1.0/ +```console +$ tree -L 2 nf-core-methylseq-1.4/ + +nf-core-methylseq-1.4 +├── configs +│   ├── bin +│   ├── conf +│   ├── configtest.nf +│   ├── docs +│   ├── LICENSE +│   ├── nextflow.config +│   ├── nfcore_custom.config +│   └── README.md ├── singularity-images -│   └── nf-core-methylseq-1.0.simg +│   └── nf-core-methylseq-1.4.simg └── workflow - ├── CHANGELOG.md - ├── Dockerfile - ├── LICENCE.md - ├── README.md ├── assets ├── bin + ├── CHANGELOG.md + ├── CODE_OF_CONDUCT.md ├── conf + ├── Dockerfile ├── docs ├── environment.yml + ├── LICENSE ├── main.nf ├── nextflow.config - └── tests + ├── parameters.settings.json + └── README.md + +10 directories, 15 files +``` + +The pipeline files are automatically updated so that the local copy of institutional configs are available when running the pipeline. +So using `-profile ` should work if available within [nf-core/configs](https://github.com/nf-core/configs). -7 directories, 8 files +You can run the pipeline by simply providing the directory path for the `workflow` folder. +Note that if using Singularity, you will also need to provide the path to the Singularity image. +For example: + +```bash +nextflow run /path/to/nf-core-methylseq-1.4/workflow/ \ + -profile singularity \ + -with-singularity /path/to/nf-core-methylseq-1.4/singularity-images/nf-core-methylseq-1.4.simg \ + # .. other normal pipeline parameters from here on.. + --reads '*_R{1,2}.fastq.gz' --genome GRCh38 ``` ## Pipeline software licences From 59798c3803390c33452d703d1e5a605a2d67f305 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 26 Nov 2019 15:57:18 +0100 Subject: [PATCH 124/224] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75769c8e6..d33960d5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * Updated Blacklist of synced pipelines * Ignore pre-releases in `nf-core list` * Lint for `Singularity` file [and remove it](https://github.com/nf-core/tools/issues/458) +* Updated documentation for `nf-core download` ### Linting From f1bdd87b39210e354feca1479f8bb1f30d037cd7 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 26 Nov 2019 16:07:18 +0100 Subject: [PATCH 125/224] Added mention of '-r' for download --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3a8a8c813..90432143f 100644 --- a/README.md +++ b/README.md @@ -212,8 +212,10 @@ To make this process easier and ensure accurate retrieval of correctly versioned By default, the pipeline will download the pipeline code and the [institutional nf-core/configs](https://github.com/nf-core/configs) files. If you specify the flag `--singularity`, it will also download any singularity image files that are required. +Use `-r`/`--release` to download a specific release of the pipeline. If not specified, the tool will automatically fetch the latest release. + ```console -$ nf-core download methylseq --singularity +$ nf-core download methylseq -r 1.4 --singularity ,--./,-. ___ __ __ __ ___ /,-._.--~\ From 233b5cdb64bed75c55b08876f6f3fa2e461fb3e3 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Wed, 27 Nov 2019 12:01:58 +0100 Subject: [PATCH 126/224] Update nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml Co-Authored-By: Phil Ewels --- .../{{cookiecutter.name_noslash}}/.github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml index 8fd361870..4100e809e 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml @@ -12,9 +12,9 @@ jobs: - uses: actions/checkout@v1 - name: Install Nextflow run: | - export NXF_VER=${{ matrix.nxf_ver }} + {% raw %}export NXF_VER=${{ matrix.nxf_ver }}{% endraw %} wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - name: Run test run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker \ No newline at end of file + nextflow run ${GITHUB_WORKSPACE} -profile test,docker From a75a4bc971e34c8581ded04524a8b8d59ffcf17c Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 27 Nov 2019 16:23:11 +0100 Subject: [PATCH 127/224] fix ubuntu to 18.04 and update Nextflow version to 19.10.0 --- .../.github/workflows/branch.yml | 4 ++-- .../{{cookiecutter.name_noslash}}/.github/workflows/ci.yml | 4 ++-- .../.github/workflows/linting.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml index e3c557ce1..523011501 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml @@ -7,9 +7,9 @@ on: jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 steps: # PRs are only ok if coming from an nf-core dev branch - name: Check PRs run: | - [ ${GITHUB_ACTOR} = "nf-core" ] && [ ${GITHUB_HEAD_REF} = "dev" ] \ No newline at end of file + { [[ $(git remote get-url origin) == *nf-core/sarek ]] && [[ ${GITHUB_BASE_REF} = "master" ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == patch ]] \ No newline at end of file diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml index 4100e809e..1a1b0dfae 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml @@ -4,10 +4,10 @@ on: [push, pull_request] jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 strategy: matrix: - nxf_ver: ['19.04.0', ''] + nxf_ver: ['19.10.0', ''] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml index 8c1e97c07..f125de38c 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: Markdown: - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v1 - uses: actions/setup-node@v1 From 064e4e9dd015ef35e9e879e1a7b558479294d3bd Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 27 Nov 2019 16:55:43 +0100 Subject: [PATCH 128/224] update docs --- .../.github/CONTRIBUTING.md | 44 +++++++++++-------- .../.github/ISSUE_TEMPLATE/bug_report.md | 43 +++++++++++------- .../.github/ISSUE_TEMPLATE/feature_request.md | 18 +++++--- .../.github/PULL_REQUEST_TEMPLATE.md | 26 ++++++----- .../.github/workflows/branch.yml | 2 +- 5 files changed, 81 insertions(+), 52 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md index 7833bc580..bce45ed48 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md @@ -1,47 +1,53 @@ # {{ cookiecutter.name }}: Contributing Guidelines -Hi there! Many thanks for taking an interest in improving {{ cookiecutter.name }}. +Hi there! +Many thanks for taking an interest in improving {{ cookiecutter.name }}. -We try to manage the required tasks for {{ cookiecutter.name }} using GitHub issues, you probably came to this page when creating one. Please use the pre-filled template to save time. +We try to manage the required tasks for {{ cookiecutter.name }} using GitHub issues, you probably came to this page when creating one. +Please use the pre-filled template to save time. -However, don't be put off by this template - other more general issues and suggestions are welcome! Contributions to the code are even more welcome ;) +However, don't be put off by this template - other more general issues and suggestions are welcome! +Contributions to the code are even more welcome ;) > If you need help using or modifying {{ cookiecutter.name }} then the best place to ask is on the pipeline channel on [Slack](https://nf-co.re/join/slack). - - ## Contribution workflow -If you'd like to write some code for {{ cookiecutter.name }}, the standard workflow -is as follows: -1. Check that there isn't already an issue about your idea in the - [{{ cookiecutter.name }} issues](https://github.com/{{ cookiecutter.name }}/issues) to avoid - duplicating work. +If you'd like to write some code for {{ cookiecutter.name }}, the standard workflow is as follows: + +1. Check that there isn't already an issue about your idea in the [{{ cookiecutter.name }} issues](https://github.com/{{ cookiecutter.name }}/issues) to avoid duplicating work * If there isn't one already, please create one so that others know you're working on this -2. Fork the [{{ cookiecutter.name }} repository](https://github.com/{{ cookiecutter.name }}) to your GitHub account +2. [Fork](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) the [{{ cookiecutter.name }} repository](https://github.com/{{ cookiecutter.name }}) to your GitHub account 3. Make the necessary changes / additions within your forked repository -4. Submit a Pull Request against the `dev` branch and wait for the code to be reviewed and merged. - -If you're not used to this workflow with git, you can start with some [basic docs from GitHub](https://help.github.com/articles/fork-a-repo/) or even their [excellent interactive tutorial](https://try.github.io/). +4. Submit a Pull Request against the `dev` branch and wait for the code to be reviewed and merged +If you're not used to this workflow with git, you can start with some [docs from GitHub](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests) or even their [excellent `git` resources](https://try.github.io/). ## Tests -When you create a pull request with changes, [Travis CI](https://travis-ci.com/) will run automatic tests. + +When you create a pull request with changes, [GitHub Actions](https://github.com/features/actions) will run automatic tests. Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then. There are typically two types of tests that run: ### Lint Tests -The nf-core has a [set of guidelines](https://nf-co.re/developers/guidelines) which all pipelines must adhere to. + +`nf-core` has a [set of guidelines](https://nf-co.re/developers/guidelines) which all pipelines must adhere to. To enforce these and ensure that all pipelines stay in sync, we have developed a helper tool which runs checks on the pipeline code. This is in the [nf-core/tools repository](https://github.com/nf-core/tools) and once installed can be run locally with the `nf-core lint ` command. If any failures or warnings are encountered, please follow the listed URL for more documentation. ### Pipeline Tests -Each nf-core pipeline should be set up with a minimal set of test-data. -Travis CI then runs the pipeline on this data to ensure that it exists successfully. + +Each `nf-core` pipeline should be set up with a minimal set of test-data. +`GitHub Actions` then runs the pipeline on this data to ensure that it exits successfully. If there are any failures then the automated tests fail. -These tests are run both with the latest available version of Nextflow and also the minimum required version that is stated in the pipeline code. +These tests are run both with the latest available version of `Nextflow` and also the minimum required version that is stated in the pipeline code. + +## Patch + +When patching a release, please work on your fork on a new branch named `patch` ## Getting help + For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.name }}#documentation) and don't hesitate to get in touch on the [{{ cookiecutter.name }} pipeline channel](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) on [Slack](https://nf-co.re/join/slack). diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/bug_report.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/bug_report.md index e1aa587d9..c3852f1be 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/bug_report.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,31 +1,42 @@ +# {{ cookiecutter.name }} bug report + Hi there! -Thanks for telling us about a problem with the pipeline. Please delete this text and anything that's not relevant from the template below: +Thanks for telling us about a problem with the pipeline. +Please delete this text and anything that's not relevant from the template below: + +## Describe the bug -#### Describe the bug A clear and concise description of what the bug is. -#### Steps to reproduce +## Steps to reproduce + Steps to reproduce the behaviour: + 1. Command line: `nextflow run ...` 2. See error: _Please provide your error message_ -#### Expected behaviour +## Expected behaviour + A clear and concise description of what you expected to happen. -#### System: - - Hardware: [e.g. HPC, Desktop, Cloud...] - - Executor: [e.g. slurm, local, awsbatch...] - - OS: [e.g. CentOS Linux, macOS, Linux Mint...] - - Version [e.g. 7, 10.13.6, 18.3...] +## System + +- Hardware: [e.g. HPC, Desktop, Cloud...] +- Executor: [e.g. slurm, local, awsbatch...] +- OS: [e.g. CentOS Linux, macOS, Linux Mint...] +- Version [e.g. 7, 10.13.6, 18.3...] + +## Nextflow Installation + +- Version: [e.g. 0.31.0] + +## Container engine -#### Nextflow Installation: - - Version: [e.g. 0.31.0] +- Engine: [e.g. Conda, Docker or Singularity] +- version: [e.g. 1.0.0] +- Image tag: [e.g. {{ cookiecutter.name_docker }}:1.0.0] -#### Container engine: - - Engine: [e.g. Conda, Docker or Singularity] - - version: [e.g. 1.0.0] - - Image tag: [e.g. {{ cookiecutter.name_docker }}:1.0.0] +## Additional context -#### Additional context Add any other context about the problem here. diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/feature_request.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/feature_request.md index 1f025b779..148df5999 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/feature_request.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,16 +1,24 @@ +# {{ cookiecutter.name }} feature request + Hi there! -Thanks for suggesting a new feature for the pipeline! Please delete this text and anything that's not relevant from the template below: +Thanks for suggesting a new feature for the pipeline! +Please delete this text and anything that's not relevant from the template below: + +## Is your feature request related to a problem? Please describe -#### Is your feature request related to a problem? Please describe. A clear and concise description of what the problem is. + Ex. I'm always frustrated when [...] -#### Describe the solution you'd like +## Describe the solution you'd like + A clear and concise description of what you want to happen. -#### Describe alternatives you've considered +## Describe alternatives you've considered + A clear and concise description of any alternative solutions or features you've considered. -#### Additional context +## Additional context + Add any other context about the feature request here. diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/PULL_REQUEST_TEMPLATE.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/PULL_REQUEST_TEMPLATE.md index 3600b8f1d..3143db960 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/PULL_REQUEST_TEMPLATE.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/PULL_REQUEST_TEMPLATE.md @@ -1,15 +1,19 @@ +# {{ cookiecutter.name }} pull request + Many thanks for contributing to {{ cookiecutter.name }}! -Please fill in the appropriate checklist below (delete whatever is not relevant). These are the most common things requested on pull requests (PRs). +Please fill in the appropriate checklist below (delete whatever is not relevant). +These are the most common things requested on pull requests (PRs). ## PR checklist - - [ ] This comment contains a description of changes (with reason) - - [ ] If you've fixed a bug or added code that should be tested, add tests! - - [ ] If necessary, also make a PR on the [{{ cookiecutter.name }} branch on the nf-core/test-datasets repo]( https://github.com/nf-core/test-datasets/pull/new/{{ cookiecutter.name }}) - - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker`). - - [ ] Make sure your code lints (`nf-core lint .`). - - [ ] Documentation in `docs` is updated - - [ ] `CHANGELOG.md` is updated - - [ ] `README.md` is updated - -**Learn more about contributing:** https://github.com/{{ cookiecutter.name }}/tree/master/.github/CONTRIBUTING.md + +- [ ] This comment contains a description of changes (with reason) +- [ ] If you've fixed a bug or added code that should be tested, add tests! +- [ ] If necessary, also make a PR on the [{{ cookiecutter.name }} branch on the nf-core/test-datasets repo](https://github.com/nf-core/test-datasets/pull/new/{{ cookiecutter.name }}) +- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker`). +- [ ] Make sure your code lints (`nf-core lint .`). +- [ ] Documentation in `docs` is updated +- [ ] `CHANGELOG.md` is updated +- [ ] `README.md` is updated + +**Learn more about contributing:** [CONTRIBUTING.md](https://github.com/{{ cookiecutter.name }}/tree/master/.github/CONTRIBUTING.md) \ No newline at end of file diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml index 523011501..26ed8339f 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml @@ -12,4 +12,4 @@ jobs: # PRs are only ok if coming from an nf-core dev branch - name: Check PRs run: | - { [[ $(git remote get-url origin) == *nf-core/sarek ]] && [[ ${GITHUB_BASE_REF} = "master" ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == patch ]] \ No newline at end of file + { [[ $(git remote get-url origin) == *{{cookiecutter.name_noslash}} ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]] \ No newline at end of file From 6d13a76b0bfc7c8e8e4eee0a67bf13f6e3301840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=BCther?= Date: Thu, 28 Nov 2019 10:37:55 +0100 Subject: [PATCH 129/224] Apply suggestions from code review removed some empty lines. thanks @MaxUlysse Co-Authored-By: Maxime Garcia --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index ccd411a8d..b7f8d58d8 100644 --- a/README.md +++ b/README.md @@ -400,27 +400,22 @@ INFO: Changing version number: Current version number is '1.0dev' New version number will be '1.0' - INFO: Updating version in nextflow.config - version = '1.0dev' + version = '1.0' - INFO: Updating version in nextflow.config - process.container = 'nfcore/mypipeline:1.0dev' + process.container = 'nfcore/mypipeline:1.0' - INFO: Updating version in .travis.yml - - docker tag nfcore/mypipeline:dev nfcore/mypipeline:1.0dev + - docker tag nfcore/mypipeline:dev nfcore/mypipeline:1.0 - INFO: Updating version in environment.yml - name: nf-core-mypipeline-1.0dev + name: nf-core-mypipeline-1.0 - INFO: Updating version in Dockerfile - RUN conda env export --name nf-core-mypipeline-1.0dev > nf-core-mypipeline-1.0dev.yml - ENV PATH /opt/conda/envs/nf-core-mypipeline-1.0dev/bin:$PATH From 659ac5d007648317221e4835dd2f5df75b943a27 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Thu, 28 Nov 2019 16:53:38 +0100 Subject: [PATCH 130/224] better link to slack channel --- .../{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md index bce45ed48..c885f86bd 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md @@ -9,7 +9,7 @@ Please use the pre-filled template to save time. However, don't be put off by this template - other more general issues and suggestions are welcome! Contributions to the code are even more welcome ;) -> If you need help using or modifying {{ cookiecutter.name }} then the best place to ask is on the pipeline channel on [Slack](https://nf-co.re/join/slack). +> If you need help using or modifying {{ cookiecutter.name }} then the best place to ask is on the nf-core Slack [#{{ cookiecutter.cookiecutter.short_name }}](https://nfcore.slack.com/channels/{{ cookiecutter.cookiecutter.short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). ## Contribution workflow @@ -50,4 +50,4 @@ When patching a release, please work on your fork on a new branch named `patch` ## Getting help -For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.name }}#documentation) and don't hesitate to get in touch on the [{{ cookiecutter.name }} pipeline channel](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) on [Slack](https://nf-co.re/join/slack). +For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.name }}#documentation) and don't hesitate to get in touch on the nf-core Slack [#{{ cookiecutter.cookiecutter.short_name }}](https://nfcore.slack.com/channels/{{ cookiecutter.cookiecutter.short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). From 59771d70b8f2bdaa8c7b37b8950189fc1cf2b8f0 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 28 Nov 2019 17:04:28 +0100 Subject: [PATCH 131/224] Apply suggestions from code review Co-Authored-By: Maxime Garcia --- .../pipeline-template/{{cookiecutter.name_noslash}}/.gitignore | 3 +-- .../{{cookiecutter.name_noslash}}/docs/usage.md | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.gitignore b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.gitignore index 6354f3708..0189a4444 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.gitignore +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.gitignore @@ -3,6 +3,5 @@ work/ data/ results/ .DS_Store -tests/ -testing/ +test* *.pyc diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index 37d8d4a7a..83886c7db 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -96,7 +96,7 @@ Use this parameter to choose a configuration profile. Profiles can give configur Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Conda) - see below. -The pipeline also dynamically loads configurations from [https://github.com/nf-core/configs](https://github.com/nf-core/configs) when it runs, making multiple config profiles for various institutional clusters available at run time. For more information and to see if your system is available in these configs please see the [nf-core/configs documentaiton](https://github.com/nf-core/configs#documentation). +The pipeline also dynamically loads configurations from [https://github.com/nf-core/configs](https://github.com/nf-core/configs) when it runs, making multiple config profiles for various institutional clusters available at run time. For more information and to see if your system is available in these configs please see the [nf-core/configs documentation](https://github.com/nf-core/configs#documentation). Note that multiple profiles can be loaded, for example: `-profile test,docker` - the order of arguments is important! They are loaded in sequence, so later profiles can overwrite earlier profiles. From d4b6b47ee66834e49b35449a7ac3817d3069bd1d Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Thu, 28 Nov 2019 17:14:23 +0100 Subject: [PATCH 132/224] Apply suggestions from code review Co-Authored-By: Phil Ewels --- .../{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md index c885f86bd..000f2fe05 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md @@ -9,7 +9,7 @@ Please use the pre-filled template to save time. However, don't be put off by this template - other more general issues and suggestions are welcome! Contributions to the code are even more welcome ;) -> If you need help using or modifying {{ cookiecutter.name }} then the best place to ask is on the nf-core Slack [#{{ cookiecutter.cookiecutter.short_name }}](https://nfcore.slack.com/channels/{{ cookiecutter.cookiecutter.short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). +> If you need help using or modifying {{ cookiecutter.name }} then the best place to ask is on the nf-core Slack [#{{ cookiecutter.short_name }}](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). ## Contribution workflow @@ -50,4 +50,4 @@ When patching a release, please work on your fork on a new branch named `patch` ## Getting help -For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.name }}#documentation) and don't hesitate to get in touch on the nf-core Slack [#{{ cookiecutter.cookiecutter.short_name }}](https://nfcore.slack.com/channels/{{ cookiecutter.cookiecutter.short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). +For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.name }}#documentation) and don't hesitate to get in touch on the nf-core Slack [#{{ cookiecutter.short_name }}](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). From e736abb52eaf78335670521a0066a80cb5459cc5 Mon Sep 17 00:00:00 2001 From: phue Date: Thu, 28 Nov 2019 20:34:12 +0100 Subject: [PATCH 133/224] update README.md toy example for nf-core bump-version had some errors, thanks @ewels for spotting this --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b7f8d58d8..3d7b9928b 100644 --- a/README.md +++ b/README.md @@ -405,11 +405,11 @@ INFO: Updating version in nextflow.config + version = '1.0' INFO: Updating version in nextflow.config - - process.container = 'nfcore/mypipeline:1.0dev' + - process.container = 'nfcore/mypipeline:dev' + process.container = 'nfcore/mypipeline:1.0' INFO: Updating version in .travis.yml - - - docker tag nfcore/mypipeline:dev nfcore/mypipeline:1.0dev + - - docker tag nfcore/mypipeline:dev nfcore/mypipeline:dev + - docker tag nfcore/mypipeline:dev nfcore/mypipeline:1.0 INFO: Updating version in environment.yml From 061b8bc2e308f62c20a7ff72666711e3c0e31803 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 28 Nov 2019 21:59:09 +0100 Subject: [PATCH 134/224] Launch: fix double hyphen typo in final command example --- CHANGELOG.md | 1 + README.md | 2 +- nf_core/launch.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72f929ef2..cb91dc1e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * Updated Blacklist of synced pipelines * Ignore pre-releases in `nf-core list` * Lint for `Singularity` file [and remove it](https://github.com/nf-core/tools/issues/458) +* Fixed typo in `nf-core launch` final command ### Linting diff --git a/README.md b/README.md index c6869ffde..4a8f95a65 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,7 @@ Specify the location of your input FastQ files. [..truncated..] Nextflow command: - nextflow run nf-core/rnaseq -profile "docker" -name "test_run" -r "1.3" --params-file "/Users/ewels/testing/nfx-params.json" + nextflow run nf-core/rnaseq -profile "docker" -name "test_run" -r "1.3" -params-file "/Users/ewels/testing/nfx-params.json" Do you want to run this command now? [y/N]: y diff --git a/nf_core/launch.py b/nf_core/launch.py index 37b91b25a..ff297c7b0 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -370,7 +370,7 @@ def build_command(self): if self.use_params_file: path = self.create_nfx_params_file() if path is not None: - self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, "--params-file", path) + self.nextflow_cmd = '{} {} "{}"'.format(self.nextflow_cmd, "-params-file", path) self.write_params_as_full_json() # Call nextflow with a list of command line flags From c8bdebc4c470baaa7b8c8fd70a144f584d2413bf Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 29 Nov 2019 12:43:48 +0100 Subject: [PATCH 135/224] generalise path for download in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 90432143f..126c200af 100644 --- a/README.md +++ b/README.md @@ -241,7 +241,7 @@ INFO: Starting build... Getting image source signatures .... INFO: Creating SIF file... -INFO: Build complete: /domus/h1/phil/GitHub/nf-core/tools/testing/nf-core-methylseq-1.4/singularity-images/nf-core-methylseq-1.4.simg +INFO: Build complete: /my-pipelines/nf-core-methylseq-1.4/singularity-images/nf-core-methylseq-1.4.simg INFO: Compressing download.. From c722bf66622fde2ac7b139027a1df2a029b9e25e Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Fri, 29 Nov 2019 13:25:06 +0100 Subject: [PATCH 136/224] Apply suggestions from code review Thanks to @ewels for useful suggestions Co-Authored-By: Phil Ewels --- .../.github/CONTRIBUTING.md | 2 +- .../.github/ISSUE_TEMPLATE/bug_report.md | 16 ++++++++-------- .../.github/workflows/branch.yml | 5 +++-- .../.github/workflows/ci.yml | 2 ++ .../.github/workflows/linting.yml | 3 ++- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md index 000f2fe05..f4f8240f7 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md @@ -50,4 +50,4 @@ When patching a release, please work on your fork on a new branch named `patch` ## Getting help -For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.name }}#documentation) and don't hesitate to get in touch on the nf-core Slack [#{{ cookiecutter.short_name }}](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). +For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://nf-co.re/{{ cookiecutter.name }}/docs) and don't hesitate to get in touch on the nf-core Slack [#{{ cookiecutter.short_name }}](https://nfcore.slack.com/channels/{{ cookiecutter.short_name }}) channel ([join our Slack here](https://nf-co.re/join/slack)). diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/bug_report.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/bug_report.md index c3852f1be..a73273430 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/bug_report.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/ISSUE_TEMPLATE/bug_report.md @@ -22,20 +22,20 @@ A clear and concise description of what you expected to happen. ## System -- Hardware: [e.g. HPC, Desktop, Cloud...] -- Executor: [e.g. slurm, local, awsbatch...] -- OS: [e.g. CentOS Linux, macOS, Linux Mint...] -- Version [e.g. 7, 10.13.6, 18.3...] +- Hardware: +- Executor: +- OS: +- Version ## Nextflow Installation -- Version: [e.g. 0.31.0] +- Version: ## Container engine -- Engine: [e.g. Conda, Docker or Singularity] -- version: [e.g. 1.0.0] -- Image tag: [e.g. {{ cookiecutter.name_docker }}:1.0.0] +- Engine: +- version: +- Image tag: ## Additional context diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml index 26ed8339f..f7c906c29 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml @@ -1,5 +1,6 @@ name: nf-core branch protection # This workflow is triggered on PRs to master branch on the repository +# It fails when someone tries to make a PR against the nf-core `master` branch instead of `dev` on: pull_request: branches: @@ -9,7 +10,7 @@ jobs: test: runs-on: ubuntu-18.04 steps: - # PRs are only ok if coming from an nf-core dev branch + # PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch - name: Check PRs run: | - { [[ $(git remote get-url origin) == *{{cookiecutter.name_noslash}} ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]] \ No newline at end of file + { [[ $(git remote get-url origin) == *{{cookiecutter.name_noslash}} ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]] diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml index 1a1b0dfae..2e4eab1ab 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml @@ -1,5 +1,6 @@ name: nf-core CI # This workflow is triggered on pushes and PRs to the repository. +# It runs the pipeline with the minimal test dataset to check that it completes without any syntax errors on: [push, pull_request] jobs: @@ -7,6 +8,7 @@ jobs: runs-on: ubuntu-18.04 strategy: matrix: + # Nextflow versions: check pipeline minimum and current latest nxf_ver: ['19.10.0', ''] steps: - uses: actions/checkout@v1 diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml index f125de38c..7354dc743 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml @@ -1,5 +1,6 @@ name: nf-core linting # This workflow is triggered on pushes and PRs to the repository. +# It runs the `nf-core lint` and markdown lint tests to ensure that the code meets the nf-core guidelines on: [push, pull_request] jobs: @@ -37,4 +38,4 @@ jobs: pip install nf-core - name: Run nf-core lint run: | - nf-core lint ${GITHUB_WORKSPACE} \ No newline at end of file + nf-core lint ${GITHUB_WORKSPACE} From cf3f2640d3ab319365b13b3344c10cc4b06a2743 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Fri, 29 Nov 2019 14:12:49 +0100 Subject: [PATCH 137/224] Update nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md --- .../{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md index f4f8240f7..348686384 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md @@ -46,7 +46,11 @@ These tests are run both with the latest available version of `Nextflow` and als ## Patch -When patching a release, please work on your fork on a new branch named `patch` +: warning: Only in the unlikely and regretful event of a release happening with a bug. + +* On your own fork, make a new branch `patch` based on `upstream/master`. +* Fix the bug, and bump version (X.Y.Z+1). +* A PR should be made on `master` from patch to directly this particular bug. ## Getting help From e04530f71f42c42de65ca6b639e1e6afa231a54e Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 4 Dec 2019 14:26:36 +0100 Subject: [PATCH 138/224] List: Handle missing description when searching --- nf_core/list.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/list.py b/nf_core/list.py index 609ad748b..204147e06 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -145,8 +145,8 @@ def filtered_workflows(self): filtered_workflows = [] for wf in self.remote_workflows: for k in self.keyword_filters: - in_name = k in wf.name - in_desc = k in wf.description + in_name = k in wf.name if wf.name else False + in_desc = k in wf.description if wf.description else False in_topics = any([ k in t for t in wf.topics]) if not in_name and not in_desc and not in_topics: break From a2694f2c80c237dab1d50aba75ec5153f896d130 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 4 Dec 2019 14:29:26 +0100 Subject: [PATCH 139/224] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 294170d6a..74eb52cee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ * Lint for `Singularity` file [and remove it](https://github.com/nf-core/tools/issues/458) * Updated documentation for `nf-core download` * Fixed typo in `nf-core launch` final command +* Handle missing pipeline descriptions in `nf-core list` ### Linting From 1fbb82d23ebb4aa79ed72fd55d36b3bbe19f0fce Mon Sep 17 00:00:00 2001 From: ggabernet Date: Thu, 5 Dec 2019 11:55:36 +0100 Subject: [PATCH 140/224] updated pipelinelint file check list --- CHANGELOG.md | 1 + nf_core/lint.py | 56 ++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74eb52cee..1846ba98c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ ### Linting * Adjusted linting to enable `patch` branches from being tested +* `.travis.yml` now optional, `.circle.yml` and `Singularity` must not be present ### Template diff --git a/nf_core/lint.py b/nf_core/lint.py index 124bd4881..a37210b46 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -195,7 +195,6 @@ def check_files_exist(self): 'nextflow.config', 'Dockerfile', - ['.travis.yml', '.circle.yml'], ['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling 'README.md', 'CHANGELOG.md', @@ -207,7 +206,19 @@ def check_files_exist(self): 'main.nf', 'environment.yml', - 'conf/base.config' + 'conf/base.config', + '.github/workflows/branch.yml', + '.github/workflows/ci.yml', + '.github/workfows/linting.yml' + + Files that *must not* be present:: + + '.circle.yml', + 'Singularity' + + Files that *should not* be present:: + + '.travis.yml' Raises: An AssertionError if neither `nextflow.config` or `main.nf` found. @@ -218,7 +229,6 @@ def check_files_exist(self): files_fail = [ 'nextflow.config', 'Dockerfile', - ['.travis.yml', '.circle.yml'], ['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling 'README.md', 'CHANGELOG.md', @@ -229,7 +239,17 @@ def check_files_exist(self): files_warn = [ 'main.nf', 'environment.yml', - 'conf/base.config' + 'conf/base.config', + '.github/workflows/branch.yml', + '.github/workflows/ci.yml', + '.github/workflows/linting.yml' + ] + files_fail_ifexists = [ + 'Singularity', + '.circle.yml' + ] + files_warn_ifexists = [ + '.travis.yml' ] def pf(file_path): @@ -239,7 +259,7 @@ def pf(file_path): if not os.path.isfile(pf('nextflow.config')) and not os.path.isfile(pf('main.nf')): raise AssertionError('Neither nextflow.config or main.nf found! Is this a Nextflow pipeline?') - # Files that cause an error + # Files that cause an error if they don't exist for files in files_fail: if not isinstance(files, list): files = [files] @@ -249,7 +269,7 @@ def pf(file_path): else: self.failed.append((1, "File not found: {}".format(files))) - # Files that cause a warning + # Files that cause a warning if they don't exist for files in files_warn: if not isinstance(files, list): files = [files] @@ -259,6 +279,24 @@ def pf(file_path): else: self.warned.append((1, "File not found: {}".format(files))) + # Files that cause an error if they exist + for files in files_fail_ifexists: + if not isinstance(files, list): + files = [files] + if any([os.path.isfile(pf(f)) for f in files]): + self.failed.append((1, "File must be removed: {}".format(files))) + else: + self.passed.append((1, "File not found check: {}".format(files))) + + # Files that cause a warning if they exist + for files in files_warn_ifexists: + if not isinstance(files, list): + files = [files] + if any ([os.path.isfile(pf(f)) for f in files]): + self.warned.append((1, "File should be removed: {}".format(files))) + else: + self.passed.append((1, "File not found check: {}".format(files))) + # Load and parse files for later if 'environment.yml' in self.files: with open(os.path.join(self.path, 'environment.yml'), 'r') as fh: @@ -278,12 +316,6 @@ def check_docker(self): self.failed.append((2, "Dockerfile check failed")) - def check_singularity(self): - """Checks whether a Singularity file exists and warns to remove that file then.""" - fn = os.path.join(self.path, "Singularity") - if(os.path.exists(fn)): - self.failed.append((11, "Singularity file exists")) - def check_licence(self): """Checks licence file is MIT. From dfdfb42343a532b5eefa14c0b09750d7d74f35a5 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 5 Dec 2019 12:30:11 +0100 Subject: [PATCH 141/224] Sync: Allow zero pipeline directories on the cli --- scripts/nf-core | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/nf-core b/scripts/nf-core index 101446fb3..32a714803 100755 --- a/scripts/nf-core +++ b/scripts/nf-core @@ -280,6 +280,7 @@ def bump_version(pipeline_dir, new_version, nextflow): @click.argument( 'pipeline_dir', type = click.Path(exists=True), + nargs = -1, metavar = "" ) @click.option( @@ -328,9 +329,11 @@ def sync(pipeline_dir, make_template_branch, from_branch, pull_request, username nf_core.sync.sync_all_pipelines() else: # Manually check for the required parameter - if not pipeline_dir: - logging.error(" is required") + if not pipeline_dir or len(pipeline_dir) != 1: + logging.error("Either use --all or specify one ") sys.exit(1) + else: + pipeline_dir = pipeline_dir[0] # Sync the given pipeline dir sync_obj = nf_core.sync.PipelineSync(pipeline_dir, make_template_branch, from_branch, pull_request) From a3fba3dede77e631bbce2566cb8e967bf6f6bcfb Mon Sep 17 00:00:00 2001 From: ggabernet Date: Thu, 5 Dec 2019 12:58:05 +0100 Subject: [PATCH 142/224] added linting check for branch.yml --- CHANGELOG.md | 1 + nf_core/lint.py | 23 +++++++++++++++++++ .../.github/workflows/branch.yml | 1 + 3 files changed, 25 insertions(+) create mode 100644 tests/lint_examples/github_actions_failing_example/.github/workflows/branch.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 1846ba98c..9ba3600a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ * Adjusted linting to enable `patch` branches from being tested * `.travis.yml` now optional, `.circle.yml` and `Singularity` must not be present +* warn if github actions workfkows do not exist ### Template diff --git a/nf_core/lint.py b/nf_core/lint.py index a37210b46..743317756 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -235,6 +235,10 @@ def check_files_exist(self): 'docs/README.md', 'docs/output.md', 'docs/usage.md', + ['.travis.yml', '.github/workflows/branch.yml'], + ['.travis.yml', '.github/workflows/ci.yml'], + ['.travis.yml', '.github/workflows/linting.yml'] + ] files_warn = [ 'main.nf', @@ -483,6 +487,25 @@ def check_nextflow_config(self): else: self.passed.append((4, "Config variable process.container looks correct: '{}'".format(container_name))) + def check_actions_branch_protection(self): + """Checks that the github actions workflows are valid. + + Makes sure that ``nf-core lint``runs in github actions workflows and that + tests run with the required nextflow version. + """ + for wf in ['.github/workflows/branch.yml']: + fn = os.path.join(self.path, wf) + if os.path.isfile(fn): + with open(fn, 'r') as fh: + branchwf = yaml.safe_load(fh) + # Check that we have the master branch protection, but allow patch as well + branchMasterCheck = 'master' + try: + assert(branchMasterCheck in branchwf.get('on', {'pull_request', {'branches', {}}})) + except AssertionError: + self.failed.append((5, "Github actions branch workflow must check for master branch PRs: '{}'".format(fn))) + else: + self.passed.append((5, "Github actions branch workflow checks for master branch PRs: '{}'".format(fn))) def check_ci_config(self): """Checks that the Travis or Circle CI YAML config is valid. diff --git a/tests/lint_examples/github_actions_failing_example/.github/workflows/branch.yml b/tests/lint_examples/github_actions_failing_example/.github/workflows/branch.yml new file mode 100644 index 000000000..d89973d9e --- /dev/null +++ b/tests/lint_examples/github_actions_failing_example/.github/workflows/branch.yml @@ -0,0 +1 @@ +name: nf-core branch protection From f271c155a841292b310ae8844282ce53444b3b18 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Thu, 5 Dec 2019 14:19:02 +0100 Subject: [PATCH 143/224] removed chech_singularity function --- nf_core/lint.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index 743317756..150799248 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -166,7 +166,6 @@ def lint_pipeline(self, release_mode=False): 'check_files_exist', 'check_licence', 'check_docker', - 'check_singularity', 'check_nextflow_config', 'check_ci_config', 'check_readme', From 67027cb261b0be54751d43606eac41dc41dd9178 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Thu, 5 Dec 2019 14:26:59 +0100 Subject: [PATCH 144/224] added actions workflows to minimal working example --- .../.github/workflows/branch.yml | 16 ++++++++ .../.github/workflows/ci.yml | 22 ++++++++++ .../.github/workflows/linting.yml | 41 +++++++++++++++++++ .../minimal_working_example/.travis.yml | 40 ------------------ tests/test_lint.py | 2 +- 5 files changed, 80 insertions(+), 41 deletions(-) create mode 100644 tests/lint_examples/minimal_working_example/.github/workflows/branch.yml create mode 100644 tests/lint_examples/minimal_working_example/.github/workflows/ci.yml create mode 100644 tests/lint_examples/minimal_working_example/.github/workflows/linting.yml delete mode 100644 tests/lint_examples/minimal_working_example/.travis.yml diff --git a/tests/lint_examples/minimal_working_example/.github/workflows/branch.yml b/tests/lint_examples/minimal_working_example/.github/workflows/branch.yml new file mode 100644 index 000000000..f7c906c29 --- /dev/null +++ b/tests/lint_examples/minimal_working_example/.github/workflows/branch.yml @@ -0,0 +1,16 @@ +name: nf-core branch protection +# This workflow is triggered on PRs to master branch on the repository +# It fails when someone tries to make a PR against the nf-core `master` branch instead of `dev` +on: + pull_request: + branches: + - master + +jobs: + test: + runs-on: ubuntu-18.04 + steps: + # PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch + - name: Check PRs + run: | + { [[ $(git remote get-url origin) == *{{cookiecutter.name_noslash}} ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]] diff --git a/tests/lint_examples/minimal_working_example/.github/workflows/ci.yml b/tests/lint_examples/minimal_working_example/.github/workflows/ci.yml new file mode 100644 index 000000000..2e4eab1ab --- /dev/null +++ b/tests/lint_examples/minimal_working_example/.github/workflows/ci.yml @@ -0,0 +1,22 @@ +name: nf-core CI +# This workflow is triggered on pushes and PRs to the repository. +# It runs the pipeline with the minimal test dataset to check that it completes without any syntax errors +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-18.04 + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + steps: + - uses: actions/checkout@v1 + - name: Install Nextflow + run: | + {% raw %}export NXF_VER=${{ matrix.nxf_ver }}{% endraw %} + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + - name: Run test + run: | + nextflow run ${GITHUB_WORKSPACE} -profile test,docker diff --git a/tests/lint_examples/minimal_working_example/.github/workflows/linting.yml b/tests/lint_examples/minimal_working_example/.github/workflows/linting.yml new file mode 100644 index 000000000..7354dc743 --- /dev/null +++ b/tests/lint_examples/minimal_working_example/.github/workflows/linting.yml @@ -0,0 +1,41 @@ +name: nf-core linting +# This workflow is triggered on pushes and PRs to the repository. +# It runs the `nf-core lint` and markdown lint tests to ensure that the code meets the nf-core guidelines +on: [push, pull_request] + +jobs: + Markdown: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-node@v1 + with: + node-version: '10' + - name: Install markdownlint + run: | + npm install -g markdownlint-cli + - name: Run Markdownlint + run: | + markdownlint ${GITHUB_WORKSPACE} -c ${GITHUB_WORKSPACE}/.github/markdownlint.yml + nf-core: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Install Nextflow + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + - uses: actions/setup-python@v1 + with: + python-version: '3.6' + architecture: 'x64' + - name: Install pip + run: | + sudo apt install python3-pip + pip install --upgrade pip + - name: Install nf-core tools + run: | + pip install nf-core + - name: Run nf-core lint + run: | + nf-core lint ${GITHUB_WORKSPACE} diff --git a/tests/lint_examples/minimal_working_example/.travis.yml b/tests/lint_examples/minimal_working_example/.travis.yml deleted file mode 100644 index 4f7d5571b..000000000 --- a/tests/lint_examples/minimal_working_example/.travis.yml +++ /dev/null @@ -1,40 +0,0 @@ -sudo: required -language: java -jdk: openjdk8 -services: - - docker -python: - - "2.7" -cache: pip -matrix: - fast_finish: true - -before_install: - # PRs to master are only ok if coming from dev branch - - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ]) || [ $TRAVIS_PULL_REQUEST_BRANCH = "patch" ]' - # Pull the docker image first so the test doesn't wait for this - - docker pull nfcore/tools:dev - # Fake the tag locally so that the pipeline runs properly - - docker tag nfcore/tools:dev nfcore/tools:0.4 - -install: - # Install Nextflow - - mkdir /tmp/nextflow - - cd /tmp/nextflow - - wget -qO- get.nextflow.io | bash - - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow - # Install nf-core/tools - - git clone https://github.com/nf-core/tools.git /tmp/nf-core-tools - - cd /tmp/nf-core-tools - - pip install --user -e . - # Reset - - cd ${TRAVIS_BUILD_DIR} - -env: - matrix: - - NXF_VER=19.10.0 - - NXF_VER='' - -script: - - "nf-core lint ${TRAVIS_BUILD_DIR}" - - "cd ${TRAVIS_BUILD_DIR}/tests && ./run_test.sh" diff --git a/tests/test_lint.py b/tests/test_lint.py index 7f07ae939..fbd9cc195 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -91,7 +91,7 @@ def test_failing_missingfiles_example(self): """Tests for missing files like Dockerfile or LICENSE""" lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) lint_obj.check_files_exist() - expectations = {"failed": 4, "warned": 1, "passed": len(listfiles(PATH_WORKING_EXAMPLE)) - 5 - 1} + expectations = {"failed": 5, "warned": 1, "passed": len(listfiles(PATH_WORKING_EXAMPLE)) - 5 - 1} self.assess_lint_status(lint_obj, **expectations) def test_mit_licence_example_pass(self): From 3bf0a5b34bef1905e16b45df7a671c432da6784a Mon Sep 17 00:00:00 2001 From: Gisela Gabernet Date: Thu, 5 Dec 2019 14:45:55 +0100 Subject: [PATCH 145/224] Update CHANGELOG.md Co-Authored-By: Alexander Peltzer --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ba3600a0..b94d1b8fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ * Adjusted linting to enable `patch` branches from being tested * `.travis.yml` now optional, `.circle.yml` and `Singularity` must not be present -* warn if github actions workfkows do not exist +* Warn if GitHub actions workflows do not exist ### Template From e93be963ad4835dd39ef7ac4c55cd50178471496 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Thu, 5 Dec 2019 16:04:45 +0100 Subject: [PATCH 146/224] removed singularity test lint --- tests/test_lint.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/test_lint.py b/tests/test_lint.py index fbd9cc195..145913961 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -107,13 +107,6 @@ def test_mit_license_example_with_failed(self): bad_lint_obj.check_licence() expectations = {"failed": 1, "warned": 0, "passed": 0} self.assess_lint_status(bad_lint_obj, **expectations) - - def test_singularity_with_failed(self): - """Tests that Singularity file doesn't exist""" - bad_lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) - bad_lint_obj.check_singularity() - expectations = {"failed": 1, "warned": 0, "passed": 0} - self.assess_lint_status(bad_lint_obj, **expectations) def test_config_variable_example_pass(self): """Tests that config variable existence test works with good pipeline example""" From dee3b56abca1f4c8796fcc71ec799abb09d6953b Mon Sep 17 00:00:00 2001 From: ggabernet Date: Thu, 5 Dec 2019 16:16:49 +0100 Subject: [PATCH 147/224] added again .travis.yml in working example --- .../minimal_working_example/.travis.yml | 40 +++++++++++++++++++ tests/test_lint.py | 2 +- 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 tests/lint_examples/minimal_working_example/.travis.yml diff --git a/tests/lint_examples/minimal_working_example/.travis.yml b/tests/lint_examples/minimal_working_example/.travis.yml new file mode 100644 index 000000000..bee246bfa --- /dev/null +++ b/tests/lint_examples/minimal_working_example/.travis.yml @@ -0,0 +1,40 @@ +sudo: required +language: java +jdk: openjdk8 +services: + - docker +python: + - "2.7" +cache: pip +matrix: + fast_finish: true + +before_install: + # PRs to master are only ok if coming from dev branch + - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && ([ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ] || [ $TRAVIS_PULL_REQUEST_BRANCH = "patch" ]))' + # Pull the docker image first so the test doesn't wait for this + - docker pull nfcore/tools:dev + # Fake the tag locally so that the pipeline runs properly + - docker tag nfcore/tools:dev nfcore/tools:0.4 + +install: + # Install Nextflow + - mkdir /tmp/nextflow + - cd /tmp/nextflow + - wget -qO- get.nextflow.io | bash + - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow + # Install nf-core/tools + - git clone https://github.com/nf-core/tools.git /tmp/nf-core-tools + - cd /tmp/nf-core-tools + - pip install --user -e . + # Reset + - cd ${TRAVIS_BUILD_DIR} + +env: + matrix: + - NXF_VER=0.32.0 + - NXF_VER='' + +script: + - "nf-core lint ${TRAVIS_BUILD_DIR}" + - "cd ${TRAVIS_BUILD_DIR}/tests && ./run_test.sh" \ No newline at end of file diff --git a/tests/test_lint.py b/tests/test_lint.py index 145913961..f7d9c1809 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -91,7 +91,7 @@ def test_failing_missingfiles_example(self): """Tests for missing files like Dockerfile or LICENSE""" lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) lint_obj.check_files_exist() - expectations = {"failed": 5, "warned": 1, "passed": len(listfiles(PATH_WORKING_EXAMPLE)) - 5 - 1} + expectations = {"failed": 5, "warned": 5, "passed": len(listfiles(PATH_WORKING_EXAMPLE)) - 5 - 1} self.assess_lint_status(lint_obj, **expectations) def test_mit_licence_example_pass(self): From 57cba7a52718f4f6c71a8ce337f4b44051bf7a2a Mon Sep 17 00:00:00 2001 From: ggabernet Date: Thu, 5 Dec 2019 17:06:52 +0100 Subject: [PATCH 148/224] added new warnings and errors to expectations --- nf_core/lint.py | 1 + tests/lint_examples/minimal_working_example/.travis.yml | 4 ++-- tests/test_lint.py | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index 150799248..d80cd9e6a 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -519,6 +519,7 @@ def check_ci_config(self): ciconf = yaml.safe_load(fh) # Check that we have the master branch protection, but allow patch as well travisMasterCheck = '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ]) || [ $TRAVIS_PULL_REQUEST_BRANCH = "patch" ]' + try: assert(travisMasterCheck in ciconf.get('before_install', {})) except AssertionError: diff --git a/tests/lint_examples/minimal_working_example/.travis.yml b/tests/lint_examples/minimal_working_example/.travis.yml index bee246bfa..8c140c785 100644 --- a/tests/lint_examples/minimal_working_example/.travis.yml +++ b/tests/lint_examples/minimal_working_example/.travis.yml @@ -11,7 +11,7 @@ matrix: before_install: # PRs to master are only ok if coming from dev branch - - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && ([ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ] || [ $TRAVIS_PULL_REQUEST_BRANCH = "patch" ]))' + - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ]) || [ $TRAVIS_PULL_REQUEST_BRANCH = "patch" ]' # Pull the docker image first so the test doesn't wait for this - docker pull nfcore/tools:dev # Fake the tag locally so that the pipeline runs properly @@ -32,7 +32,7 @@ install: env: matrix: - - NXF_VER=0.32.0 + - NXF_VER=19.10.0 - NXF_VER='' script: diff --git a/tests/test_lint.py b/tests/test_lint.py index f7d9c1809..b71e4fb47 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -38,7 +38,7 @@ def pf(wd, path): pf(WD, 'lint_examples/license_incomplete_example')] # The maximum sum of passed tests currently possible -MAX_PASS_CHECKS = 61 +MAX_PASS_CHECKS = 68 # The additional tests passed for releases ADD_PASS_RELEASE = 1 @@ -58,7 +58,7 @@ def test_call_lint_pipeline_pass(self): This should not result in any exception for the minimal working example""" lint_obj = nf_core.lint.run_linting(PATH_WORKING_EXAMPLE, False) - expectations = {"failed": 0, "warned": 3, "passed": MAX_PASS_CHECKS} + expectations = {"failed": 0, "warned": 4, "passed": MAX_PASS_CHECKS} self.assess_lint_status(lint_obj, **expectations) @pytest.mark.xfail(raises=AssertionError) @@ -73,7 +73,7 @@ def test_call_lint_pipeline_release(self): """Test the main execution function of PipelineLint when running with --release""" lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) lint_obj.lint_pipeline(release_mode=True) - expectations = {"failed": 0, "warned": 3, "passed": MAX_PASS_CHECKS + ADD_PASS_RELEASE} + expectations = {"failed": 0, "warned": 4, "passed": MAX_PASS_CHECKS + ADD_PASS_RELEASE} self.assess_lint_status(lint_obj, **expectations) def test_failing_dockerfile_example(self): From 8ff5d5286f609d08ccf13359d7eb2f2745957fdc Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Fri, 6 Dec 2019 09:44:48 +0100 Subject: [PATCH 149/224] Added function to lint pipeline naming --- nf_core/lint.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index 124bd4881..558079ad7 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -172,7 +172,8 @@ def lint_pipeline(self, release_mode=False): 'check_readme', 'check_conda_env_yaml', 'check_conda_dockerfile', - 'check_pipeline_todos' + 'check_pipeline_todos', + 'check_pipeline_naming' ] if release_mode: self.release_mode = True @@ -802,6 +803,18 @@ def check_pipeline_todos(self): l = '{}..'.format(l[:50-len(fname)]) self.warned.append((10, "TODO string found in '{}': {}".format(fname,l))) + def check_pipeline_naming(self): + """Check whether pipeline name adheres to lower case/no hyphen naming convention""" + + #Extract last part of given path, a.k.a. the workflow name + workflow_name = os.path.basename(os.path.normpath(self.path)) + if not workflow_name.islower(): + self.warned.append((10, "Naming does not adhere to nf-core conventions: Contains uppercase letters")) + if not workflow_name.isalpha(): + self.warned.append((10, "Naming does not adhere to nf-core conventions: Contains non alphabetical characters")) + + + def print_results(self): # Print results rl = "\n Using --release mode linting tests" if self.release_mode else '' From 56801b54398a64eb5a1aaa7296fd05041a72f06b Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Fri, 6 Dec 2019 09:49:30 +0100 Subject: [PATCH 150/224] Changed error code from 10 to 11 --- nf_core/lint.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index 558079ad7..e8cacda89 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -809,9 +809,9 @@ def check_pipeline_naming(self): #Extract last part of given path, a.k.a. the workflow name workflow_name = os.path.basename(os.path.normpath(self.path)) if not workflow_name.islower(): - self.warned.append((10, "Naming does not adhere to nf-core conventions: Contains uppercase letters")) + self.warned.append((11, "Naming does not adhere to nf-core conventions: Contains uppercase letters")) if not workflow_name.isalpha(): - self.warned.append((10, "Naming does not adhere to nf-core conventions: Contains non alphabetical characters")) + self.warned.append((11, "Naming does not adhere to nf-core conventions: Contains non alphabetical characters")) From 7a4f9d6dedd61d2badcddc5156f1e5f5a6308702 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Fri, 6 Dec 2019 10:06:18 +0100 Subject: [PATCH 151/224] Add some basic test --- nf_core/lint.py | 4 ++-- tests/test_lint.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index e8cacda89..73ba0f8aa 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -173,7 +173,7 @@ def lint_pipeline(self, release_mode=False): 'check_conda_env_yaml', 'check_conda_dockerfile', 'check_pipeline_todos', - 'check_pipeline_naming' + 'check_pipeline_name' ] if release_mode: self.release_mode = True @@ -803,7 +803,7 @@ def check_pipeline_todos(self): l = '{}..'.format(l[:50-len(fname)]) self.warned.append((10, "TODO string found in '{}': {}".format(fname,l))) - def check_pipeline_naming(self): + def check_pipeline_name(self): """Check whether pipeline name adheres to lower case/no hyphen naming convention""" #Extract last part of given path, a.k.a. the workflow name diff --git a/tests/test_lint.py b/tests/test_lint.py index 7f07ae939..bb03a03e7 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -423,3 +423,18 @@ def test_pip_dependency_fails(self): lint_obj.check_conda_env_yaml() expectations = {"failed": 1, "warned": 0, "passed": 2} self.assess_lint_status(lint_obj, **expectations) + + def test_pipeline_name_pass(self): + """Tests pipeline name good pipeline example: lower case, no punctuation""" + good_lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) + good_lint_obj.check_pipeline_name() + expectations = {"failed": 0, "warned": 0, "passed": 1} + self.assess_lint_status(good_lint_obj, **expectations) + print("something happened") + + def test_pipeline_name_critical(self): + """Tests that warning is returned for pipeline not adhering to naming convention""" + good_lint_obj = nf_core.lint.PipelineLint(PATH_CRITICAL_EXAMPLE) + good_lint_obj.check_pipeline_name() + expectations = {"failed": 0, "warned": 1, "passed": 0} + self.assess_lint_status(good_lint_obj, **expectations) \ No newline at end of file From d77721ea5ad8ade125717e886ae01adc5faa6a4d Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Fri, 6 Dec 2019 11:05:01 +0100 Subject: [PATCH 152/224] Increase num of passed tests --- nf_core/lint.py | 3 +++ tests/test_lint.py | 5 ++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index 73ba0f8aa..32b84790a 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -808,6 +808,9 @@ def check_pipeline_name(self): #Extract last part of given path, a.k.a. the workflow name workflow_name = os.path.basename(os.path.normpath(self.path)) + + if workflow_name.islower() and workflow_name.isalpha(): + self.passed.append((11, "Name adheres to nf-core convention")) if not workflow_name.islower(): self.warned.append((11, "Naming does not adhere to nf-core conventions: Contains uppercase letters")) if not workflow_name.isalpha(): diff --git a/tests/test_lint.py b/tests/test_lint.py index bb03a03e7..073998ba6 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -32,13 +32,13 @@ def pf(wd, path): WD = os.path.dirname(__file__) PATH_CRITICAL_EXAMPLE = pf(WD, 'lint_examples/critical_example') PATH_FAILING_EXAMPLE = pf(WD, 'lint_examples/failing_example') -PATH_WORKING_EXAMPLE = pf(WD, 'lint_examples/minimal_working_example') +PATH_WORKING_EXAMPLE = pf(WD, 'lint_examples/minimalworkingexample') PATH_MISSING_LICENSE_EXAMPLE = pf(WD, 'lint_examples/missing_license_example') PATHS_WRONG_LICENSE_EXAMPLE = [pf(WD, 'lint_examples/wrong_license_example'), pf(WD, 'lint_examples/license_incomplete_example')] # The maximum sum of passed tests currently possible -MAX_PASS_CHECKS = 61 +MAX_PASS_CHECKS = 62 # The additional tests passed for releases ADD_PASS_RELEASE = 1 @@ -430,7 +430,6 @@ def test_pipeline_name_pass(self): good_lint_obj.check_pipeline_name() expectations = {"failed": 0, "warned": 0, "passed": 1} self.assess_lint_status(good_lint_obj, **expectations) - print("something happened") def test_pipeline_name_critical(self): """Tests that warning is returned for pipeline not adhering to naming convention""" From d5fb63ea537d94ed5abd5f922342f03697e35870 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Fri, 6 Dec 2019 11:06:29 +0100 Subject: [PATCH 153/224] Change example name to adhere to naming --- .../minimalworkingexample/.travis.yml | 40 ++++++++++++++++++ .../minimalworkingexample/CHANGELOG.md | 0 .../minimalworkingexample/Dockerfile | 9 ++++ .../minimalworkingexample/LICENSE | 7 ++++ .../minimalworkingexample/README.md | 5 +++ .../minimalworkingexample/conf/base.config | 0 .../minimalworkingexample/docs/README.md | 0 .../minimalworkingexample/docs/output.md | 0 .../minimalworkingexample/docs/usage.md | 0 .../minimalworkingexample/environment.yml | 12 ++++++ .../minimalworkingexample/main.nf | 0 .../minimalworkingexample/nextflow.config | 42 +++++++++++++++++++ .../minimalworkingexample/tests/run_test.sh | 0 13 files changed, 115 insertions(+) create mode 100644 tests/lint_examples/minimalworkingexample/.travis.yml create mode 100644 tests/lint_examples/minimalworkingexample/CHANGELOG.md create mode 100644 tests/lint_examples/minimalworkingexample/Dockerfile create mode 100644 tests/lint_examples/minimalworkingexample/LICENSE create mode 100644 tests/lint_examples/minimalworkingexample/README.md create mode 100644 tests/lint_examples/minimalworkingexample/conf/base.config create mode 100644 tests/lint_examples/minimalworkingexample/docs/README.md create mode 100644 tests/lint_examples/minimalworkingexample/docs/output.md create mode 100644 tests/lint_examples/minimalworkingexample/docs/usage.md create mode 100644 tests/lint_examples/minimalworkingexample/environment.yml create mode 100644 tests/lint_examples/minimalworkingexample/main.nf create mode 100644 tests/lint_examples/minimalworkingexample/nextflow.config create mode 100644 tests/lint_examples/minimalworkingexample/tests/run_test.sh diff --git a/tests/lint_examples/minimalworkingexample/.travis.yml b/tests/lint_examples/minimalworkingexample/.travis.yml new file mode 100644 index 000000000..4f7d5571b --- /dev/null +++ b/tests/lint_examples/minimalworkingexample/.travis.yml @@ -0,0 +1,40 @@ +sudo: required +language: java +jdk: openjdk8 +services: + - docker +python: + - "2.7" +cache: pip +matrix: + fast_finish: true + +before_install: + # PRs to master are only ok if coming from dev branch + - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ]) || [ $TRAVIS_PULL_REQUEST_BRANCH = "patch" ]' + # Pull the docker image first so the test doesn't wait for this + - docker pull nfcore/tools:dev + # Fake the tag locally so that the pipeline runs properly + - docker tag nfcore/tools:dev nfcore/tools:0.4 + +install: + # Install Nextflow + - mkdir /tmp/nextflow + - cd /tmp/nextflow + - wget -qO- get.nextflow.io | bash + - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow + # Install nf-core/tools + - git clone https://github.com/nf-core/tools.git /tmp/nf-core-tools + - cd /tmp/nf-core-tools + - pip install --user -e . + # Reset + - cd ${TRAVIS_BUILD_DIR} + +env: + matrix: + - NXF_VER=19.10.0 + - NXF_VER='' + +script: + - "nf-core lint ${TRAVIS_BUILD_DIR}" + - "cd ${TRAVIS_BUILD_DIR}/tests && ./run_test.sh" diff --git a/tests/lint_examples/minimalworkingexample/CHANGELOG.md b/tests/lint_examples/minimalworkingexample/CHANGELOG.md new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lint_examples/minimalworkingexample/Dockerfile b/tests/lint_examples/minimalworkingexample/Dockerfile new file mode 100644 index 000000000..0b3f1d287 --- /dev/null +++ b/tests/lint_examples/minimalworkingexample/Dockerfile @@ -0,0 +1,9 @@ +FROM nfcore/base:dev +MAINTAINER Phil Ewels +LABEL authors="phil.ewels@scilifelab.se" \ + description="Docker image containing all requirements for the nf-core tools pipeline" + +COPY environment.yml / +RUN conda env create -f /environment.yml && conda clean -a +RUN conda env export --name nf-core-tools-0.4 > nf-core-tools-0.4.yml +ENV PATH /opt/conda/envs/nf-core-tools-0.4/bin:$PATH diff --git a/tests/lint_examples/minimalworkingexample/LICENSE b/tests/lint_examples/minimalworkingexample/LICENSE new file mode 100644 index 000000000..ba37e5dbb --- /dev/null +++ b/tests/lint_examples/minimalworkingexample/LICENSE @@ -0,0 +1,7 @@ +Copyright 1984 me-myself-and-I + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/tests/lint_examples/minimalworkingexample/README.md b/tests/lint_examples/minimalworkingexample/README.md new file mode 100644 index 000000000..838a6faef --- /dev/null +++ b/tests/lint_examples/minimalworkingexample/README.md @@ -0,0 +1,5 @@ +# The pipeline readme file + +[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A519.10.0-brightgreen.svg)](https://www.nextflow.io/) + +[![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](http://bioconda.github.io/) diff --git a/tests/lint_examples/minimalworkingexample/conf/base.config b/tests/lint_examples/minimalworkingexample/conf/base.config new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lint_examples/minimalworkingexample/docs/README.md b/tests/lint_examples/minimalworkingexample/docs/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lint_examples/minimalworkingexample/docs/output.md b/tests/lint_examples/minimalworkingexample/docs/output.md new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lint_examples/minimalworkingexample/docs/usage.md b/tests/lint_examples/minimalworkingexample/docs/usage.md new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lint_examples/minimalworkingexample/environment.yml b/tests/lint_examples/minimalworkingexample/environment.yml new file mode 100644 index 000000000..75bd6b291 --- /dev/null +++ b/tests/lint_examples/minimalworkingexample/environment.yml @@ -0,0 +1,12 @@ +# You can use this file to create a conda environment for this pipeline: +# conda env create -f environment.yml +name: nf-core-tools-0.4 +channels: + - defaults + - conda-forge + - bioconda +dependencies: + - conda-forge::openjdk=8.0.144 + - fastqc=0.11.7 + - pip: + - multiqc==1.4 diff --git a/tests/lint_examples/minimalworkingexample/main.nf b/tests/lint_examples/minimalworkingexample/main.nf new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lint_examples/minimalworkingexample/nextflow.config b/tests/lint_examples/minimalworkingexample/nextflow.config new file mode 100644 index 000000000..303d675d2 --- /dev/null +++ b/tests/lint_examples/minimalworkingexample/nextflow.config @@ -0,0 +1,42 @@ + +params { + outdir = './results' + reads = "data/*.fastq" + single_end = false + custom_config_version = 'master' + custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" +} + +process { + container = 'nfcore/tools:0.4' + cpus = 1 + memory = 2.GB + time = 14.h +} + +timeline { + enabled = true + file = "timeline.html" +} +report { + enabled = true + file = "report.html" +} +trace { + enabled = true + file = "trace.txt" +} +dag { + enabled = true + file = "dag.svg" +} + +manifest { + name = 'nf-core/tools' + author = 'Phil Ewels' + homePage = 'https://github.com/nf-core/tools' + description = 'Minimal working example pipeline' + mainScript = 'main.nf' + nextflowVersion = '>=19.10.0' + version = '0.4' +} diff --git a/tests/lint_examples/minimalworkingexample/tests/run_test.sh b/tests/lint_examples/minimalworkingexample/tests/run_test.sh new file mode 100644 index 000000000..e69de29bb From 5cafadc68fe9ff14ac2a855a0f8cb8929d0e9484 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Fri, 6 Dec 2019 11:07:02 +0100 Subject: [PATCH 154/224] Change example name to adhere to naming --- .../minimal_working_example/.travis.yml | 40 ------------------ .../minimal_working_example/CHANGELOG.md | 0 .../minimal_working_example/Dockerfile | 9 ---- .../minimal_working_example/LICENSE | 7 ---- .../minimal_working_example/README.md | 5 --- .../minimal_working_example/conf/base.config | 0 .../minimal_working_example/docs/README.md | 0 .../minimal_working_example/docs/output.md | 0 .../minimal_working_example/docs/usage.md | 0 .../minimal_working_example/environment.yml | 12 ------ .../minimal_working_example/main.nf | 0 .../minimal_working_example/nextflow.config | 42 ------------------- .../minimal_working_example/tests/run_test.sh | 0 13 files changed, 115 deletions(-) delete mode 100644 tests/lint_examples/minimal_working_example/.travis.yml delete mode 100644 tests/lint_examples/minimal_working_example/CHANGELOG.md delete mode 100644 tests/lint_examples/minimal_working_example/Dockerfile delete mode 100644 tests/lint_examples/minimal_working_example/LICENSE delete mode 100644 tests/lint_examples/minimal_working_example/README.md delete mode 100644 tests/lint_examples/minimal_working_example/conf/base.config delete mode 100644 tests/lint_examples/minimal_working_example/docs/README.md delete mode 100644 tests/lint_examples/minimal_working_example/docs/output.md delete mode 100644 tests/lint_examples/minimal_working_example/docs/usage.md delete mode 100644 tests/lint_examples/minimal_working_example/environment.yml delete mode 100644 tests/lint_examples/minimal_working_example/main.nf delete mode 100644 tests/lint_examples/minimal_working_example/nextflow.config delete mode 100644 tests/lint_examples/minimal_working_example/tests/run_test.sh diff --git a/tests/lint_examples/minimal_working_example/.travis.yml b/tests/lint_examples/minimal_working_example/.travis.yml deleted file mode 100644 index 4f7d5571b..000000000 --- a/tests/lint_examples/minimal_working_example/.travis.yml +++ /dev/null @@ -1,40 +0,0 @@ -sudo: required -language: java -jdk: openjdk8 -services: - - docker -python: - - "2.7" -cache: pip -matrix: - fast_finish: true - -before_install: - # PRs to master are only ok if coming from dev branch - - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ]) || [ $TRAVIS_PULL_REQUEST_BRANCH = "patch" ]' - # Pull the docker image first so the test doesn't wait for this - - docker pull nfcore/tools:dev - # Fake the tag locally so that the pipeline runs properly - - docker tag nfcore/tools:dev nfcore/tools:0.4 - -install: - # Install Nextflow - - mkdir /tmp/nextflow - - cd /tmp/nextflow - - wget -qO- get.nextflow.io | bash - - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow - # Install nf-core/tools - - git clone https://github.com/nf-core/tools.git /tmp/nf-core-tools - - cd /tmp/nf-core-tools - - pip install --user -e . - # Reset - - cd ${TRAVIS_BUILD_DIR} - -env: - matrix: - - NXF_VER=19.10.0 - - NXF_VER='' - -script: - - "nf-core lint ${TRAVIS_BUILD_DIR}" - - "cd ${TRAVIS_BUILD_DIR}/tests && ./run_test.sh" diff --git a/tests/lint_examples/minimal_working_example/CHANGELOG.md b/tests/lint_examples/minimal_working_example/CHANGELOG.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/lint_examples/minimal_working_example/Dockerfile b/tests/lint_examples/minimal_working_example/Dockerfile deleted file mode 100644 index 0b3f1d287..000000000 --- a/tests/lint_examples/minimal_working_example/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM nfcore/base:dev -MAINTAINER Phil Ewels -LABEL authors="phil.ewels@scilifelab.se" \ - description="Docker image containing all requirements for the nf-core tools pipeline" - -COPY environment.yml / -RUN conda env create -f /environment.yml && conda clean -a -RUN conda env export --name nf-core-tools-0.4 > nf-core-tools-0.4.yml -ENV PATH /opt/conda/envs/nf-core-tools-0.4/bin:$PATH diff --git a/tests/lint_examples/minimal_working_example/LICENSE b/tests/lint_examples/minimal_working_example/LICENSE deleted file mode 100644 index ba37e5dbb..000000000 --- a/tests/lint_examples/minimal_working_example/LICENSE +++ /dev/null @@ -1,7 +0,0 @@ -Copyright 1984 me-myself-and-I - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/tests/lint_examples/minimal_working_example/README.md b/tests/lint_examples/minimal_working_example/README.md deleted file mode 100644 index 838a6faef..000000000 --- a/tests/lint_examples/minimal_working_example/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# The pipeline readme file - -[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A519.10.0-brightgreen.svg)](https://www.nextflow.io/) - -[![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](http://bioconda.github.io/) diff --git a/tests/lint_examples/minimal_working_example/conf/base.config b/tests/lint_examples/minimal_working_example/conf/base.config deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/lint_examples/minimal_working_example/docs/README.md b/tests/lint_examples/minimal_working_example/docs/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/lint_examples/minimal_working_example/docs/output.md b/tests/lint_examples/minimal_working_example/docs/output.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/lint_examples/minimal_working_example/docs/usage.md b/tests/lint_examples/minimal_working_example/docs/usage.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/lint_examples/minimal_working_example/environment.yml b/tests/lint_examples/minimal_working_example/environment.yml deleted file mode 100644 index 75bd6b291..000000000 --- a/tests/lint_examples/minimal_working_example/environment.yml +++ /dev/null @@ -1,12 +0,0 @@ -# You can use this file to create a conda environment for this pipeline: -# conda env create -f environment.yml -name: nf-core-tools-0.4 -channels: - - defaults - - conda-forge - - bioconda -dependencies: - - conda-forge::openjdk=8.0.144 - - fastqc=0.11.7 - - pip: - - multiqc==1.4 diff --git a/tests/lint_examples/minimal_working_example/main.nf b/tests/lint_examples/minimal_working_example/main.nf deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/lint_examples/minimal_working_example/nextflow.config b/tests/lint_examples/minimal_working_example/nextflow.config deleted file mode 100644 index 303d675d2..000000000 --- a/tests/lint_examples/minimal_working_example/nextflow.config +++ /dev/null @@ -1,42 +0,0 @@ - -params { - outdir = './results' - reads = "data/*.fastq" - single_end = false - custom_config_version = 'master' - custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" -} - -process { - container = 'nfcore/tools:0.4' - cpus = 1 - memory = 2.GB - time = 14.h -} - -timeline { - enabled = true - file = "timeline.html" -} -report { - enabled = true - file = "report.html" -} -trace { - enabled = true - file = "trace.txt" -} -dag { - enabled = true - file = "dag.svg" -} - -manifest { - name = 'nf-core/tools' - author = 'Phil Ewels' - homePage = 'https://github.com/nf-core/tools' - description = 'Minimal working example pipeline' - mainScript = 'main.nf' - nextflowVersion = '>=19.10.0' - version = '0.4' -} diff --git a/tests/lint_examples/minimal_working_example/tests/run_test.sh b/tests/lint_examples/minimal_working_example/tests/run_test.sh deleted file mode 100644 index e69de29bb..000000000 From 5643b9d1ccbc33131ed298a37711b3168b3340b0 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Fri, 6 Dec 2019 11:09:58 +0100 Subject: [PATCH 155/224] Add name linting to changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74eb52cee..a873570f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ ### Linting * Adjusted linting to enable `patch` branches from being tested +* Warn if pipeline name contains upper case letters or non alphabetical characters [#85](https://github.com/nf-core/tools/issues/85) ### Template From 7ac9b8251af2c152d24208b216af0c6f41f1f6ad Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Fri, 6 Dec 2019 11:49:40 +0100 Subject: [PATCH 156/224] Update website --- docs/lint_errors.md | 4 ++++ nf_core/lint.py | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/lint_errors.md b/docs/lint_errors.md index df07ce5c6..d0c73704d 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -236,3 +236,7 @@ This lint test runs through all files in the pipeline and searches for these lin ## Error #11 - Singularity file found ##{#11} As we are relying on [Docker Hub](https://https://hub.docker.com/) instead of Singularity and all containers are automatically pulled from there, repositories should not have a `Singularity` file present. + +## Error #12 - Pipeline name ## {#12} + +In order to ensure consistent naming, pipeline names should contain only lower case, alphabetical characters. Otherwise a warning is displayed. diff --git a/nf_core/lint.py b/nf_core/lint.py index 32b84790a..ebd10901d 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -808,13 +808,13 @@ def check_pipeline_name(self): #Extract last part of given path, a.k.a. the workflow name workflow_name = os.path.basename(os.path.normpath(self.path)) - + if workflow_name.islower() and workflow_name.isalpha(): - self.passed.append((11, "Name adheres to nf-core convention")) + self.passed.append((12, "Name adheres to nf-core convention")) if not workflow_name.islower(): - self.warned.append((11, "Naming does not adhere to nf-core conventions: Contains uppercase letters")) + self.warned.append((12, "Naming does not adhere to nf-core conventions: Contains uppercase letters")) if not workflow_name.isalpha(): - self.warned.append((11, "Naming does not adhere to nf-core conventions: Contains non alphabetical characters")) + self.warned.append((12, "Naming does not adhere to nf-core conventions: Contains non alphabetical characters")) From a33300af9e7e1456da904c66273bb1059c523ad3 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Fri, 6 Dec 2019 12:41:06 +0100 Subject: [PATCH 157/224] added check github actions branch protection --- nf_core/lint.py | 26 ++++++++++++------- .../.github/workflows/branch.yml | 0 tests/test_lint.py | 17 ++++++++++++ 3 files changed, 33 insertions(+), 10 deletions(-) rename tests/lint_examples/{github_actions_failing_example => actions_failing_example}/.github/workflows/branch.yml (100%) diff --git a/nf_core/lint.py b/nf_core/lint.py index d80cd9e6a..9d2a73492 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -489,20 +489,26 @@ def check_nextflow_config(self): def check_actions_branch_protection(self): """Checks that the github actions workflows are valid. - Makes sure that ``nf-core lint``runs in github actions workflows and that + Makes sure that ``nf-core lint`` runs in github actions workflows and that tests run with the required nextflow version. """ - for wf in ['.github/workflows/branch.yml']: - fn = os.path.join(self.path, wf) - if os.path.isfile(fn): - with open(fn, 'r') as fh: - branchwf = yaml.safe_load(fh) - # Check that we have the master branch protection, but allow patch as well - branchMasterCheck = 'master' + wf = '.github/workflows/branch.yml' + fn = os.path.join(self.path, wf) + if os.path.isfile(fn): + with open(fn, 'r') as fh: + branchwf = yaml.safe_load(fh) + # Check that we have the master branch protection, but allow patch as well + branchMasterCheck = 'master' + + try: + branchwf[True]['pull_request']['branches'] + except KeyError: + self.failed.append((5, "Github actions branch workflow must check for master branch PRs (KeyError): '{}'".format(fn))) + else: try: - assert(branchMasterCheck in branchwf.get('on', {'pull_request', {'branches', {}}})) + assert(branchMasterCheck in branchwf[True]['pull_request']['branches']) except AssertionError: - self.failed.append((5, "Github actions branch workflow must check for master branch PRs: '{}'".format(fn))) + self.failed.append((5, "Github actions branch workflow must check for master branch PRs: '{}'".format(fn))) else: self.passed.append((5, "Github actions branch workflow checks for master branch PRs: '{}'".format(fn))) diff --git a/tests/lint_examples/github_actions_failing_example/.github/workflows/branch.yml b/tests/lint_examples/actions_failing_example/.github/workflows/branch.yml similarity index 100% rename from tests/lint_examples/github_actions_failing_example/.github/workflows/branch.yml rename to tests/lint_examples/actions_failing_example/.github/workflows/branch.yml diff --git a/tests/test_lint.py b/tests/test_lint.py index b71e4fb47..272b0e6f2 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -36,6 +36,7 @@ def pf(wd, path): PATH_MISSING_LICENSE_EXAMPLE = pf(WD, 'lint_examples/missing_license_example') PATHS_WRONG_LICENSE_EXAMPLE = [pf(WD, 'lint_examples/wrong_license_example'), pf(WD, 'lint_examples/license_incomplete_example')] +PATH_ACTIONS_FAILING_EXAMPLE = pf(WD, 'lint_examples/actions_failing_example') # The maximum sum of passed tests currently possible MAX_PASS_CHECKS = 68 @@ -127,6 +128,22 @@ def test_config_variable_error(self): """Tests that config variable existence test falls over nicely with nextflow can't run""" bad_lint_obj = nf_core.lint.PipelineLint('/non/existant/path') bad_lint_obj.check_nextflow_config() + + def test_actions_wf_branch_pass(self): + """Tests that linting for GitHub actions workflow for branch protection works for a good example""" + lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) + lint_obj.minNextflowVersion = '19.10.0' + lint_obj.check_actions_branch_protection() + expectations = {"failed": 0, "warned": 0, "passed": 1} + self.assess_lint_status(lint_obj, **expectations) + + def test_actions_wf_branch_fail(self): + """Tests that linting for Github actions workflow for branch protection fails for a bad example""" + lint_obj = nf_core.lint.PipelineLint(PATH_ACTIONS_FAILING_EXAMPLE) + lint_obj.minNextflowVersion = '19.10.0' + lint_obj.check_actions_branch_protection() + expectations = {"failed": 1, "warned":0, "passed": 0} + self.assess_lint_status(lint_obj, **expectations) def test_ci_conf_pass(self): """Tests that the continous integration config checks work with a good example""" From c95c289bfef70cc1bcaed6908b3a0c3b537e07d8 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Fri, 6 Dec 2019 12:45:32 +0100 Subject: [PATCH 158/224] linting actions branch protection --- nf_core/lint.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index 9d2a73492..7a2272987 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -502,15 +502,10 @@ def check_actions_branch_protection(self): try: branchwf[True]['pull_request']['branches'] - except KeyError: - self.failed.append((5, "Github actions branch workflow must check for master branch PRs (KeyError): '{}'".format(fn))) + except (AssertionError, KeyError): + self.failed.append((5, "Github actions branch workflow must check for master branch PRs: '{}'".format(fn))) else: - try: - assert(branchMasterCheck in branchwf[True]['pull_request']['branches']) - except AssertionError: - self.failed.append((5, "Github actions branch workflow must check for master branch PRs: '{}'".format(fn))) - else: - self.passed.append((5, "Github actions branch workflow checks for master branch PRs: '{}'".format(fn))) + self.passed.append((5, "Github actions branch workflow checks for master branch PRs: '{}'".format(fn))) def check_ci_config(self): """Checks that the Travis or Circle CI YAML config is valid. From f321004243c0b5bdf2ee65f79ef05f120f900354 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 6 Dec 2019 13:56:32 +0100 Subject: [PATCH 159/224] Python3 compatible scrape_software_versions --- .../bin/scrape_software_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/bin/scrape_software_versions.py b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/bin/scrape_software_versions.py index c9d236122..9f5a6a64a 100755 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/bin/scrape_software_versions.py +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/bin/scrape_software_versions.py @@ -28,7 +28,7 @@ results[k] = False # Remove software set to false in results -for k in results: +for k in list(results): if not results[k]: del(results[k]) From 495614f6e2013d56be6afc438387f6ccb4de8292 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 6 Dec 2019 13:57:40 +0100 Subject: [PATCH 160/224] Add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 074d7fbdc..a21a709ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ * Add link to [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and [Semantic Versioning](http://semver.org/spec/v2.0.0.html) to CHANGELOG * Adjusted `.travis.yml` checks to allow for `patch` branches to be tested * Remove awsbatch profile cf [nf-core/configs#71](https://github.com/nf-core/configs/pull/71) +* Make `scrape_software_versions.py` compatible with Python3 to enable miniconda3 in [base image PR](https://github.com/nf-core/tools/pull/462) ### Other From 251b6f0fed39ee4c67e4271cdeee73f3ff073507 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Fri, 6 Dec 2019 14:46:04 +0100 Subject: [PATCH 161/224] Change working example name in pther test files --- tests/test_bump_version.py | 2 +- tests/test_download.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_bump_version.py b/tests/test_bump_version.py index ae9dbe1b5..aa81e8520 100644 --- a/tests/test_bump_version.py +++ b/tests/test_bump_version.py @@ -8,7 +8,7 @@ import nf_core.lint, nf_core.bump_version WD = os.path.dirname(__file__) -PATH_WORKING_EXAMPLE = os.path.join(WD, 'lint_examples/minimal_working_example') +PATH_WORKING_EXAMPLE = os.path.join(WD, 'lint_examples/minimalworkingexample') @pytest.mark.datafiles(PATH_WORKING_EXAMPLE) diff --git a/tests/test_download.py b/tests/test_download.py index 75b891cef..18bf26c55 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -16,7 +16,7 @@ import tempfile import unittest -PATH_WORKING_EXAMPLE = os.path.join(os.path.dirname(__file__), 'lint_examples/minimal_working_example') +PATH_WORKING_EXAMPLE = os.path.join(os.path.dirname(__file__), 'lint_examples/minimalworkingexample') class DownloadTest(unittest.TestCase): From 734b8d3b2e61c824f0a1fb1234d1c03ed435b20c Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Fri, 6 Dec 2019 15:11:29 +0100 Subject: [PATCH 162/224] Changed workflow_name to self.pipeline_name --- nf_core/lint.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index ebd10901d..1d62d38b1 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -805,15 +805,12 @@ def check_pipeline_todos(self): def check_pipeline_name(self): """Check whether pipeline name adheres to lower case/no hyphen naming convention""" - - #Extract last part of given path, a.k.a. the workflow name - workflow_name = os.path.basename(os.path.normpath(self.path)) - - if workflow_name.islower() and workflow_name.isalpha(): + + if self.pipeline_name.islower() and self.pipeline_name.isalpha(): self.passed.append((12, "Name adheres to nf-core convention")) - if not workflow_name.islower(): + if not self.pipeline_name.islower(): self.warned.append((12, "Naming does not adhere to nf-core conventions: Contains uppercase letters")) - if not workflow_name.isalpha(): + if not self.pipeline_name.isalpha(): self.warned.append((12, "Naming does not adhere to nf-core conventions: Contains non alphabetical characters")) From 9bc6e42d914502418588e4a07ce51c084591fe78 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 6 Dec 2019 15:23:22 +0100 Subject: [PATCH 163/224] Add mention of how to keep py2k in the changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bca62c3fe..3529d9bce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ ### Base Docker image * Use miniconda3 instead of miniconda for a Python 3k base environment + * If you still need Python 2 for your pipeline, add `conda-forge:python=2.7.4` to the dependencies in your `environment.yml` * Update conda version to 4.7.12 From 849f7f2b9af65af2398e117f9f420275a6c0fb5c Mon Sep 17 00:00:00 2001 From: ggabernet Date: Fri, 6 Dec 2019 15:45:54 +0100 Subject: [PATCH 164/224] finished actions branch linting --- nf_core/lint.py | 40 ++++++++++++++----- .../.github/workflows/branch.yml | 9 +++++ tests/test_lint.py | 4 +- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index 7a2272987..8479846d2 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -487,25 +487,47 @@ def check_nextflow_config(self): self.passed.append((4, "Config variable process.container looks correct: '{}'".format(container_name))) def check_actions_branch_protection(self): - """Checks that the github actions workflows are valid. + """Checks that the GitHub actions branch protection workflow is valid. - Makes sure that ``nf-core lint`` runs in github actions workflows and that + Makes sure that ``nf-core lint`` runs in GitHub actions workflows and that tests run with the required nextflow version. """ - wf = '.github/workflows/branch.yml' - fn = os.path.join(self.path, wf) + fn = os.path.join(self.path, '.github', 'workflows', 'branch.yml') if os.path.isfile(fn): with open(fn, 'r') as fh: branchwf = yaml.safe_load(fh) - # Check that we have the master branch protection, but allow patch as well - branchMasterCheck = 'master' + + # Check that the action is turned on for PRs to master + try: + assert('master' in branchwf[True]['pull_request']['branches']) + except (AssertionError, KeyError): + self.failed.append((5, "GitHub actions branch workflow must check for master branch PRs: '{}'".format(fn))) + else: + self.passed.append((5, "GitHub actions branch workflow checks for master branch PRs: '{}'".format(fn))) + # Check that PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch + PRMasterCheck = '{ [[ $(git remote get-url origin) == *{{cookiecutter.name_noslash}} ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]]' + print(branchwf) try: - branchwf[True]['pull_request']['branches'] + steps = branchwf['jobs']['test']['steps'] + assert(any([PRMasterCheck in step['run'] for step in steps])) except (AssertionError, KeyError): - self.failed.append((5, "Github actions branch workflow must check for master branch PRs: '{}'".format(fn))) + self.failed.append((5, "GitHub actions branch workflow checks for master branch PRs: '{}'".format(fn))) else: - self.passed.append((5, "Github actions branch workflow checks for master branch PRs: '{}'".format(fn))) + self.passed.append((5, "GitHub actions branch workflow checks for master branch PRs: '{}'".format(fn))) + + def check_actions_ci(self): + """Checks that the GitHub actions ci workflow is valid + + Makes sure that ``nf-core lint``runs in GitHub actions workflows and that + tests run with the required nextflow version. + """ + wf = '.github/workflows/ci.yml' + fn = os.path.join(self.path, wf) + if os.path.isfile(fn): + with open(fn, 'r') as fh: + ciwf = yaml.safe_load(fh) + def check_ci_config(self): """Checks that the Travis or Circle CI YAML config is valid. diff --git a/tests/lint_examples/actions_failing_example/.github/workflows/branch.yml b/tests/lint_examples/actions_failing_example/.github/workflows/branch.yml index d89973d9e..4f730ec2f 100644 --- a/tests/lint_examples/actions_failing_example/.github/workflows/branch.yml +++ b/tests/lint_examples/actions_failing_example/.github/workflows/branch.yml @@ -1 +1,10 @@ name: nf-core branch protection +jobs: + test: + runs-on: ubuntu-18.04 + steps: + # PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch + - name: Check PRs + run: bad example + - name: Check sth + run: still bad example diff --git a/tests/test_lint.py b/tests/test_lint.py index 272b0e6f2..43527925e 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -134,7 +134,7 @@ def test_actions_wf_branch_pass(self): lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) lint_obj.minNextflowVersion = '19.10.0' lint_obj.check_actions_branch_protection() - expectations = {"failed": 0, "warned": 0, "passed": 1} + expectations = {"failed": 0, "warned": 0, "passed": 2} self.assess_lint_status(lint_obj, **expectations) def test_actions_wf_branch_fail(self): @@ -142,7 +142,7 @@ def test_actions_wf_branch_fail(self): lint_obj = nf_core.lint.PipelineLint(PATH_ACTIONS_FAILING_EXAMPLE) lint_obj.minNextflowVersion = '19.10.0' lint_obj.check_actions_branch_protection() - expectations = {"failed": 1, "warned":0, "passed": 0} + expectations = {"failed": 2, "warned":0, "passed": 0} self.assess_lint_status(lint_obj, **expectations) def test_ci_conf_pass(self): From e6db858533c43f140620a180122aa3f0575d2859 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Mon, 9 Dec 2019 14:48:00 +0100 Subject: [PATCH 165/224] Set pipeline name in test Pipeline name is not set in PipelineLint object, but initialized with 'None'. It is set in function check_nextflow_config. To test the function check_pipeline_name by itself, I set the name manually. --- nf_core/lint.py | 3 ++- tests/test_lint.py | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index 1d62d38b1..bfd054d3d 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -334,6 +334,7 @@ def check_nextflow_config(self): and print all config variables. NB: Does NOT parse contents of main.nf / nextflow script """ + # Fail tests if these are missing config_fail = [ 'manifest.name', @@ -805,7 +806,7 @@ def check_pipeline_todos(self): def check_pipeline_name(self): """Check whether pipeline name adheres to lower case/no hyphen naming convention""" - + if self.pipeline_name.islower() and self.pipeline_name.isalpha(): self.passed.append((12, "Name adheres to nf-core convention")) if not self.pipeline_name.islower(): diff --git a/tests/test_lint.py b/tests/test_lint.py index 073998ba6..5d87c3de9 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -426,14 +426,17 @@ def test_pip_dependency_fails(self): def test_pipeline_name_pass(self): """Tests pipeline name good pipeline example: lower case, no punctuation""" + #good_lint_obj = nf_core.lint.run_linting(PATH_WORKING_EXAMPLE) good_lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) + good_lint_obj.pipeline_name = 'tools' good_lint_obj.check_pipeline_name() expectations = {"failed": 0, "warned": 0, "passed": 1} self.assess_lint_status(good_lint_obj, **expectations) def test_pipeline_name_critical(self): """Tests that warning is returned for pipeline not adhering to naming convention""" - good_lint_obj = nf_core.lint.PipelineLint(PATH_CRITICAL_EXAMPLE) - good_lint_obj.check_pipeline_name() + critical_lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) + critical_lint_obj.pipeline_name = 'Tools123' + critical_lint_obj.check_pipeline_name() expectations = {"failed": 0, "warned": 1, "passed": 0} - self.assess_lint_status(good_lint_obj, **expectations) \ No newline at end of file + self.assess_lint_status(critical_lint_obj, **expectations) \ No newline at end of file From 60d1359564a5c172259c430169238b648241ecca Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Mon, 9 Dec 2019 14:55:04 +0100 Subject: [PATCH 166/224] Corrected expected num of warnings --- tests/test_lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_lint.py b/tests/test_lint.py index 5d87c3de9..82c4dfa7d 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -438,5 +438,5 @@ def test_pipeline_name_critical(self): critical_lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) critical_lint_obj.pipeline_name = 'Tools123' critical_lint_obj.check_pipeline_name() - expectations = {"failed": 0, "warned": 1, "passed": 0} + expectations = {"failed": 0, "warned": 2, "passed": 0} self.assess_lint_status(critical_lint_obj, **expectations) \ No newline at end of file From 6f16532bd9f86cc673f3ebb17121c9e5d7ad7713 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Mon, 9 Dec 2019 15:05:08 +0100 Subject: [PATCH 167/224] Remove spaces --- docs/lint_errors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/lint_errors.md b/docs/lint_errors.md index d0c73704d..8dc6b4b8f 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -239,4 +239,4 @@ As we are relying on [Docker Hub](https://https://hub.docker.com/) instead of Si ## Error #12 - Pipeline name ## {#12} -In order to ensure consistent naming, pipeline names should contain only lower case, alphabetical characters. Otherwise a warning is displayed. +In order to ensure consistent naming, pipeline names should contain only lower case, alphabetical characters. Otherwise a warning is displayed. \ No newline at end of file From 03f4d066b702cc2a667fd97936bbd619dfd9f15b Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Mon, 9 Dec 2019 15:11:28 +0100 Subject: [PATCH 168/224] Add newline for linting --- docs/lint_errors.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/lint_errors.md b/docs/lint_errors.md index 8dc6b4b8f..00bfa3c46 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -239,4 +239,5 @@ As we are relying on [Docker Hub](https://https://hub.docker.com/) instead of Si ## Error #12 - Pipeline name ## {#12} -In order to ensure consistent naming, pipeline names should contain only lower case, alphabetical characters. Otherwise a warning is displayed. \ No newline at end of file + In order to ensure consistent naming, pipeline names should contain only lower case, alphabetical characters. Otherwise a warning is displayed. + \ No newline at end of file From f82d8d131c9b294f5fa7d8d628d87ee069bab0cc Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Mon, 9 Dec 2019 15:15:26 +0100 Subject: [PATCH 169/224] Remove in trailing space in line 242 and 243 --- docs/lint_errors.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/lint_errors.md b/docs/lint_errors.md index 00bfa3c46..896f16b6f 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -239,5 +239,4 @@ As we are relying on [Docker Hub](https://https://hub.docker.com/) instead of Si ## Error #12 - Pipeline name ## {#12} - In order to ensure consistent naming, pipeline names should contain only lower case, alphabetical characters. Otherwise a warning is displayed. - \ No newline at end of file +In order to ensure consistent naming, pipeline names should contain only lower case, alphabetical characters. Otherwise a warning is displayed. From a21077ce5297c20370148024e480c10ee84cc0cf Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 10 Dec 2019 15:46:29 +0100 Subject: [PATCH 170/224] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Patrick Hüther --- CHANGELOG.md | 2 +- .../{{cookiecutter.name_noslash}}/environment.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3529d9bce..af66f5f97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,7 @@ ### Base Docker image * Use miniconda3 instead of miniconda for a Python 3k base environment - * If you still need Python 2 for your pipeline, add `conda-forge:python=2.7.4` to the dependencies in your `environment.yml` + * If you still need Python 2 for your pipeline, add `conda-forge::python=2.7.4` to the dependencies in your `environment.yml` * Update conda version to 4.7.12 diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml index 3eb6a6ec1..11a4cc08e 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml @@ -6,7 +6,7 @@ channels: - bioconda - defaults dependencies: - - python=3.7 + - conda-forge::python=3.7.4 # TODO nf-core: Add required software dependencies here - bioconda::fastqc=0.11.8 - bioconda::multiqc=1.7 From ef2994c98eda905f83c93f026f279a1bd2258785 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 10 Dec 2019 15:46:49 +0100 Subject: [PATCH 171/224] Markdown linting error - double line break --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af66f5f97..1fa92abdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,7 +46,6 @@ * If you still need Python 2 for your pipeline, add `conda-forge::python=2.7.4` to the dependencies in your `environment.yml` * Update conda version to 4.7.12 - ### Other * Updated Base Dockerfile to Conda 4.7.10 From f9cc10ef7440768f9223ff30a86a2d7e7df68780 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 10 Dec 2019 17:08:57 +0100 Subject: [PATCH 172/224] Markdown linting fixes --- CHANGELOG.md | 2 +- .../pipeline-template/{{cookiecutter.name_noslash}}/README.md | 1 + .../{{cookiecutter.name_noslash}}/docs/usage.md | 4 +--- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fa92abdd..89e29c7ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,7 @@ ### Base Docker image * Use miniconda3 instead of miniconda for a Python 3k base environment - * If you still need Python 2 for your pipeline, add `conda-forge::python=2.7.4` to the dependencies in your `environment.yml` + * If you still need Python 2 for your pipeline, add `conda-forge::python=2.7.4` to the dependencies in your `environment.yml` * Update conda version to 4.7.12 ### Other diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md index d94a12561..3fac6d7ea 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md @@ -31,6 +31,7 @@ nextflow run {{ cookiecutter.name }} -profile test, + ```bash nextflow run {{ cookiecutter.name }} -profile --reads '*_R{1,2}.fastq.gz' --genome GRCh37 ``` diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index 83886c7db..c4b0d579f 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -2,8 +2,6 @@ ## Table of contents - - * [Table of contents](#table-of-contents) * [Introduction](#introduction) * [Running the pipeline](#running-the-pipeline) @@ -39,7 +37,6 @@ * [`--plaintext_email`](#--plaintext_email) * [`--monochrome_logs`](#--monochrome_logs) * [`--multiqc_config`](#--multiqc_config) - ## Introduction @@ -185,6 +182,7 @@ params { ``` + ### `--fasta` If you prefer, you can specify the full path to your reference genome when you run the pipeline: From 400ebcfbd3e12f53b0ccf965f56e19aba8754953 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Tue, 10 Dec 2019 18:16:04 +0100 Subject: [PATCH 173/224] github actions branch linting test --- nf_core/lint.py | 14 ++++++++------ .../.github/workflows/branch.yml | 2 +- tests/test_lint.py | 4 ++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index 8479846d2..4f25452b7 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -489,8 +489,7 @@ def check_nextflow_config(self): def check_actions_branch_protection(self): """Checks that the GitHub actions branch protection workflow is valid. - Makes sure that ``nf-core lint`` runs in GitHub actions workflows and that - tests run with the required nextflow version. + Makes sure PRs can only come from nf-core dev or 'patch' of a fork. """ fn = os.path.join(self.path, '.github', 'workflows', 'branch.yml') if os.path.isfile(fn): @@ -506,8 +505,9 @@ def check_actions_branch_protection(self): self.passed.append((5, "GitHub actions branch workflow checks for master branch PRs: '{}'".format(fn))) # Check that PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch - PRMasterCheck = '{ [[ $(git remote get-url origin) == *{{cookiecutter.name_noslash}} ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]]' - print(branchwf) + pipeline_version = self.config.get('manifest.version', '').strip(' \'"') + PRMasterCheck = "{{ [[ $(git remote get-url origin) == *nf-core/{} ]] && [[ ${{GITHUB_HEAD_REF}} = \"dev\" ]]; }} || [[ ${{GITHUB_HEAD_REF}} == \"patch\" ]]".format(self.pipeline_name.lower()) + steps = branchwf['jobs']['test']['steps'] try: steps = branchwf['jobs']['test']['steps'] assert(any([PRMasterCheck in step['run'] for step in steps])) @@ -522,11 +522,13 @@ def check_actions_ci(self): Makes sure that ``nf-core lint``runs in GitHub actions workflows and that tests run with the required nextflow version. """ - wf = '.github/workflows/ci.yml' - fn = os.path.join(self.path, wf) + fn = os.path.join(self.path, '.github', 'workflows', 'ci.yml') if os.path.isfile(fn): with open(fn, 'r') as fh: ciwf = yaml.safe_load(fh) + + + def check_ci_config(self): diff --git a/tests/lint_examples/minimal_working_example/.github/workflows/branch.yml b/tests/lint_examples/minimal_working_example/.github/workflows/branch.yml index f7c906c29..f1e5aef52 100644 --- a/tests/lint_examples/minimal_working_example/.github/workflows/branch.yml +++ b/tests/lint_examples/minimal_working_example/.github/workflows/branch.yml @@ -13,4 +13,4 @@ jobs: # PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch - name: Check PRs run: | - { [[ $(git remote get-url origin) == *{{cookiecutter.name_noslash}} ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]] + { [[ $(git remote get-url origin) == *nf-core/tools ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]] diff --git a/tests/test_lint.py b/tests/test_lint.py index 43527925e..e94eb4541 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -133,6 +133,8 @@ def test_actions_wf_branch_pass(self): """Tests that linting for GitHub actions workflow for branch protection works for a good example""" lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) lint_obj.minNextflowVersion = '19.10.0' + lint_obj.pipeline_name = 'tools' + lint_obj.config['manifest.version'] = '0.4' lint_obj.check_actions_branch_protection() expectations = {"failed": 0, "warned": 0, "passed": 2} self.assess_lint_status(lint_obj, **expectations) @@ -141,6 +143,8 @@ def test_actions_wf_branch_fail(self): """Tests that linting for Github actions workflow for branch protection fails for a bad example""" lint_obj = nf_core.lint.PipelineLint(PATH_ACTIONS_FAILING_EXAMPLE) lint_obj.minNextflowVersion = '19.10.0' + lint_obj.pipeline_name = 'tools' + lint_obj.config['manifest.version'] = '0.4' lint_obj.check_actions_branch_protection() expectations = {"failed": 2, "warned":0, "passed": 0} self.assess_lint_status(lint_obj, **expectations) From 6a971d510921f5f30e2639446cfae15bd275cc13 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 10 Dec 2019 21:50:04 +0100 Subject: [PATCH 174/224] Fix pinned python 3.7.3 version for conda --- .../{{cookiecutter.name_noslash}}/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml index 11a4cc08e..df25d32c6 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml @@ -6,7 +6,7 @@ channels: - bioconda - defaults dependencies: - - conda-forge::python=3.7.4 + - conda-forge::python=3.7.3 # TODO nf-core: Add required software dependencies here - bioconda::fastqc=0.11.8 - bioconda::multiqc=1.7 From 5913bbdb8a04088c12494cac90eda7e69b203804 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Tue, 10 Dec 2019 22:24:03 +0100 Subject: [PATCH 175/224] updated ci github actions workflow template --- .../{{cookiecutter.name_noslash}}/.github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml index 2e4eab1ab..a9eb162dd 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml @@ -17,6 +17,9 @@ jobs: {% raw %}export NXF_VER=${{ matrix.nxf_ver }}{% endraw %} wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ + - name: Pull image + run: | + docker pull {{ cookiecutter.name_docker }}:dev && docker tag {{ cookiecutter.name_docker }}:dev {{ cookiecutter.name_docker }}:dev - name: Run test run: | nextflow run ${GITHUB_WORKSPACE} -profile test,docker From a07f4b26172aa7cbd4c517eacd92fc42de1a49a9 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Wed, 11 Dec 2019 00:37:08 +0100 Subject: [PATCH 176/224] test actions ci docker container --- nf_core/lint.py | 29 +++++++++++++++-- .../.github/workflows/branch.yml | 10 ++++++ .../failing_example/.github/workflows/ci.yml | 20 ++++++++++++ .../.github/workflows/ci.yml | 3 ++ tests/test_lint.py | 32 ++++++++++++++----- 5 files changed, 84 insertions(+), 10 deletions(-) create mode 100644 tests/lint_examples/failing_example/.github/workflows/branch.yml create mode 100644 tests/lint_examples/failing_example/.github/workflows/ci.yml diff --git a/nf_core/lint.py b/nf_core/lint.py index 4f25452b7..0a4170b8c 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -167,6 +167,8 @@ def lint_pipeline(self, release_mode=False): 'check_licence', 'check_docker', 'check_nextflow_config', + 'check_actions_branch_protection', + 'check_actions_ci', 'check_ci_config', 'check_readme', 'check_conda_env_yaml', @@ -519,14 +521,36 @@ def check_actions_branch_protection(self): def check_actions_ci(self): """Checks that the GitHub actions ci workflow is valid - Makes sure that ``nf-core lint``runs in GitHub actions workflows and that - tests run with the required nextflow version. + Makes sure tests run with the required nextflow version. """ fn = os.path.join(self.path, '.github', 'workflows', 'ci.yml') if os.path.isfile(fn): with open(fn, 'r') as fh: ciwf = yaml.safe_load(fh) + # Check that the action is turned on for push and pull requests + try: + assert('push' in ciwf[True]) + assert('pull_request' in ciwf[True]) + except (AssertionError, KeyError, TypeError): + self.failed.append((5, "GitHub actions ci workflow must be triggered on PR and push: '{}'".format(fn))) + else: + self.passed.append((5, "GitHub actions ci workflow is triggered on PR and push: '{}'".format(fn))) + + # Check that we're pulling the right docker image and tagging it properly + print(self.config) + if self.config.get('process.container', ''): + docker_notag = re.sub(r':(?:[\.\d]+|dev)$', '', self.config.get('process.container', '').strip('"\'')) + docker_withtag = self.config.get('process.container', '').strip('"\'') + docker_pull_cmd = 'docker pull {}:dev && docker tag {}:dev {}\n'.format(docker_notag, docker_notag, docker_withtag) + try: + steps = ciwf['jobs']['test']['steps'] + assert(any([docker_pull_cmd in step['run'] for step in steps[1:]])) + except AssertionError: + self.failed.append((5, "CI is not pulling and tagging the correct docker image. Should be:\n '{}'".format(docker_pull_cmd))) + else: + self.passed.append((5, "CI is pulling and tagging the correct docker image: {}".format(docker_pull_cmd))) + @@ -560,6 +584,7 @@ def check_ci_config(self): self.passed.append((5, "Continuous integration runs nf-core lint Tests: '{}'".format(fn))) # Check that we're pulling the right docker image + print(self.config) if self.config.get('process.container', ''): docker_notag = re.sub(r':(?:[\.\d]+|dev)$', '', self.config.get('process.container', '').strip('"\'')) docker_pull_cmd = 'docker pull {}:dev'.format(docker_notag) diff --git a/tests/lint_examples/failing_example/.github/workflows/branch.yml b/tests/lint_examples/failing_example/.github/workflows/branch.yml new file mode 100644 index 000000000..05e345fd2 --- /dev/null +++ b/tests/lint_examples/failing_example/.github/workflows/branch.yml @@ -0,0 +1,10 @@ +name: nf-core branch protection +jobs: + test: + runs-on: ubuntu-18.04 + steps: + # PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch + - name: Check PRs + run: bad example + - name: Check sth + run: still bad example \ No newline at end of file diff --git a/tests/lint_examples/failing_example/.github/workflows/ci.yml b/tests/lint_examples/failing_example/.github/workflows/ci.yml new file mode 100644 index 000000000..d0d42bc1d --- /dev/null +++ b/tests/lint_examples/failing_example/.github/workflows/ci.yml @@ -0,0 +1,20 @@ +name: nf-core CI +# This workflow is triggered on pushes and PRs to the repository. +# It runs the pipeline with the minimal test dataset to check that it completes without any syntax errors +on: + +jobs: + test: + runs-on: ubuntu-18.04 + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + steps: + - uses: actions/checkout@v1 + - name: Install Nextflow + run: | + - name: Pull container + run: | + - name: Run test + run: | diff --git a/tests/lint_examples/minimal_working_example/.github/workflows/ci.yml b/tests/lint_examples/minimal_working_example/.github/workflows/ci.yml index 2e4eab1ab..a1b5c3cf2 100644 --- a/tests/lint_examples/minimal_working_example/.github/workflows/ci.yml +++ b/tests/lint_examples/minimal_working_example/.github/workflows/ci.yml @@ -17,6 +17,9 @@ jobs: {% raw %}export NXF_VER=${{ matrix.nxf_ver }}{% endraw %} wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ + - name: Pull container + run: | + docker pull nfcore/tools:dev && docker tag nfcore/tools:dev nfcore/tools:0.4 - name: Run test run: | nextflow run ${GITHUB_WORKSPACE} -profile test,docker diff --git a/tests/test_lint.py b/tests/test_lint.py index e94eb4541..e182f1d65 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -39,7 +39,7 @@ def pf(wd, path): PATH_ACTIONS_FAILING_EXAMPLE = pf(WD, 'lint_examples/actions_failing_example') # The maximum sum of passed tests currently possible -MAX_PASS_CHECKS = 68 +MAX_PASS_CHECKS = 72 # The additional tests passed for releases ADD_PASS_RELEASE = 1 @@ -92,7 +92,7 @@ def test_failing_missingfiles_example(self): """Tests for missing files like Dockerfile or LICENSE""" lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) lint_obj.check_files_exist() - expectations = {"failed": 5, "warned": 5, "passed": len(listfiles(PATH_WORKING_EXAMPLE)) - 5 - 1} + expectations = {"failed": 5, "warned": 3, "passed": len(listfiles(PATH_WORKING_EXAMPLE)) - 3 - 1} self.assess_lint_status(lint_obj, **expectations) def test_mit_licence_example_pass(self): @@ -132,21 +132,37 @@ def test_config_variable_error(self): def test_actions_wf_branch_pass(self): """Tests that linting for GitHub actions workflow for branch protection works for a good example""" lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) - lint_obj.minNextflowVersion = '19.10.0' lint_obj.pipeline_name = 'tools' - lint_obj.config['manifest.version'] = '0.4' lint_obj.check_actions_branch_protection() expectations = {"failed": 0, "warned": 0, "passed": 2} self.assess_lint_status(lint_obj, **expectations) def test_actions_wf_branch_fail(self): """Tests that linting for Github actions workflow for branch protection fails for a bad example""" - lint_obj = nf_core.lint.PipelineLint(PATH_ACTIONS_FAILING_EXAMPLE) - lint_obj.minNextflowVersion = '19.10.0' + lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) lint_obj.pipeline_name = 'tools' - lint_obj.config['manifest.version'] = '0.4' lint_obj.check_actions_branch_protection() - expectations = {"failed": 2, "warned":0, "passed": 0} + expectations = {"failed": 2, "warned": 0, "passed": 0} + self.assess_lint_status(lint_obj, **expectations) + + def test_actions_wf_ci_pass(self): + """Tests that linting for GitHub actions CI workflow works for a good example""" + lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) + lint_obj.minNextflowVersion = '19.10.0' + lint_obj.pipeline_name = 'tools' + lint_obj.config['process.container'] = "'nfcore/tools:0.4'" + lint_obj.check_actions_ci() + expectations = {"failed": 0, "warned": 0, "passed": 1} + self.assess_lint_status(lint_obj, **expectations) + + def test_actions_wf_ci_fail(self): + """Tests that linting for GitHub actions CI workflow works for a bad example""" + lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) + lint_obj.minNextflowVersion = '19.10.0' + lint_obj.pipeline_name = 'tools' + lint_obj.config['process.container'] = "'nfcore/tools:0.4'" + lint_obj.check_actions_ci() + expectations = {"failed": 2, "warned": 0, "passed": 0} self.assess_lint_status(lint_obj, **expectations) def test_ci_conf_pass(self): From 971f36398a8ce4813753c7a0bbc0adab0a1d7ed0 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Wed, 11 Dec 2019 10:56:38 +0100 Subject: [PATCH 177/224] added linting GH actions ci NXF version --- nf_core/lint.py | 39 +++++++++++-------- .../.github/workflows/branch.yml | 10 ----- .../failing_example/.github/workflows/ci.yml | 4 -- tests/test_lint.py | 16 ++++++-- 4 files changed, 35 insertions(+), 34 deletions(-) delete mode 100644 tests/lint_examples/actions_failing_example/.github/workflows/branch.yml diff --git a/nf_core/lint.py b/nf_core/lint.py index 0a4170b8c..c5361fa16 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -527,7 +527,7 @@ def check_actions_ci(self): if os.path.isfile(fn): with open(fn, 'r') as fh: ciwf = yaml.safe_load(fh) - + # Check that the action is turned on for push and pull requests try: assert('push' in ciwf[True]) @@ -538,23 +538,29 @@ def check_actions_ci(self): self.passed.append((5, "GitHub actions ci workflow is triggered on PR and push: '{}'".format(fn))) # Check that we're pulling the right docker image and tagging it properly - print(self.config) if self.config.get('process.container', ''): - docker_notag = re.sub(r':(?:[\.\d]+|dev)$', '', self.config.get('process.container', '').strip('"\'')) - docker_withtag = self.config.get('process.container', '').strip('"\'') - docker_pull_cmd = 'docker pull {}:dev && docker tag {}:dev {}\n'.format(docker_notag, docker_notag, docker_withtag) - try: - steps = ciwf['jobs']['test']['steps'] - assert(any([docker_pull_cmd in step['run'] for step in steps[1:]])) - except AssertionError: - self.failed.append((5, "CI is not pulling and tagging the correct docker image. Should be:\n '{}'".format(docker_pull_cmd))) - else: - self.passed.append((5, "CI is pulling and tagging the correct docker image: {}".format(docker_pull_cmd))) - - - + docker_notag = re.sub(r':(?:[\.\d]+|dev)$', '', self.config.get('process.container', '').strip('"\'')) + docker_withtag = self.config.get('process.container', '').strip('"\'') + docker_pull_cmd = 'docker pull {}:dev && docker tag {}:dev {}\n'.format(docker_notag, docker_notag, docker_withtag) + try: + steps = ciwf['jobs']['test']['steps'] + assert(any([docker_pull_cmd in step['run'] for step in steps[1:]])) + except AssertionError: + self.failed.append((5, "CI is not pulling and tagging the correct docker image. Should be:\n '{}'".format(docker_pull_cmd))) + else: + self.passed.append((5, "CI is pulling and tagging the correct docker image: {}".format(docker_pull_cmd))) - + # Check that we are testing the minimum nextflow version + try: + matrix = ciwf['jobs']['test']['strategy']['matrix']['nxf_ver'] + assert(any([self.minNextflowVersion in matrix])) + except (KeyError, TypeError): + self.failed.append((5, "Continuous integration does not check minimum NF version: '{}'".format(fn))) + except AssertionError: + self.failed.append((5, "Minimum NF version differed from CI and what was set in the pipelines manifest: {}".format(fn))) + else: + self.passed.append((5, "Continuous integration checks minimum NF version: '{}'".format(fn))) + def check_ci_config(self): """Checks that the Travis or Circle CI YAML config is valid. @@ -584,7 +590,6 @@ def check_ci_config(self): self.passed.append((5, "Continuous integration runs nf-core lint Tests: '{}'".format(fn))) # Check that we're pulling the right docker image - print(self.config) if self.config.get('process.container', ''): docker_notag = re.sub(r':(?:[\.\d]+|dev)$', '', self.config.get('process.container', '').strip('"\'')) docker_pull_cmd = 'docker pull {}:dev'.format(docker_notag) diff --git a/tests/lint_examples/actions_failing_example/.github/workflows/branch.yml b/tests/lint_examples/actions_failing_example/.github/workflows/branch.yml deleted file mode 100644 index 4f730ec2f..000000000 --- a/tests/lint_examples/actions_failing_example/.github/workflows/branch.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: nf-core branch protection -jobs: - test: - runs-on: ubuntu-18.04 - steps: - # PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch - - name: Check PRs - run: bad example - - name: Check sth - run: still bad example diff --git a/tests/lint_examples/failing_example/.github/workflows/ci.yml b/tests/lint_examples/failing_example/.github/workflows/ci.yml index d0d42bc1d..eab6f8351 100644 --- a/tests/lint_examples/failing_example/.github/workflows/ci.yml +++ b/tests/lint_examples/failing_example/.github/workflows/ci.yml @@ -6,10 +6,6 @@ on: jobs: test: runs-on: ubuntu-18.04 - strategy: - matrix: - # Nextflow versions: check pipeline minimum and current latest - nxf_ver: ['19.10.0', ''] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/tests/test_lint.py b/tests/test_lint.py index e182f1d65..fa434fc26 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -152,17 +152,27 @@ def test_actions_wf_ci_pass(self): lint_obj.pipeline_name = 'tools' lint_obj.config['process.container'] = "'nfcore/tools:0.4'" lint_obj.check_actions_ci() - expectations = {"failed": 0, "warned": 0, "passed": 1} + expectations = {"failed": 0, "warned": 0, "passed": 3} self.assess_lint_status(lint_obj, **expectations) def test_actions_wf_ci_fail(self): - """Tests that linting for GitHub actions CI workflow works for a bad example""" + """Tests that linting for GitHub actions CI workflow fails for a bad example""" lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) lint_obj.minNextflowVersion = '19.10.0' lint_obj.pipeline_name = 'tools' lint_obj.config['process.container'] = "'nfcore/tools:0.4'" lint_obj.check_actions_ci() - expectations = {"failed": 2, "warned": 0, "passed": 0} + expectations = {"failed": 3, "warned": 0, "passed": 0} + self.assess_lint_status(lint_obj, **expectations) + + def test_actions_wf_ci_fail_wrong_NF_version(self): + """Tests that linting for GitHub actions CI workflow fails for a bad NXF version""" + lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) + lint_obj.minNextflowVersion = '0.28.0' + lint_obj.pipeline_name = 'tools' + lint_obj.config['process.container'] = "'nfcore/tools:0.4'" + lint_obj.check_actions_ci() + expectations = {"failed": 1, "warned": 0, "passed": 2} self.assess_lint_status(lint_obj, **expectations) def test_ci_conf_pass(self): From d3271c08a7262b1bcd412edccb7242e7a90bd414 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Wed, 11 Dec 2019 12:59:35 +0100 Subject: [PATCH 178/224] added lint func and tests actions lint workflow --- nf_core/lint.py | 47 +++++++++++++++++-- .../.github/workflows/linting.yml | 40 ++++++++++++++++ tests/test_lint.py | 18 ++++++- 3 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 tests/lint_examples/failing_example/.github/workflows/linting.yml diff --git a/nf_core/lint.py b/nf_core/lint.py index c5361fa16..0e305ecb0 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -169,6 +169,7 @@ def lint_pipeline(self, release_mode=False): 'check_nextflow_config', 'check_actions_branch_protection', 'check_actions_ci', + 'check_actions_lint', 'check_ci_config', 'check_readme', 'check_conda_env_yaml', @@ -527,7 +528,7 @@ def check_actions_ci(self): if os.path.isfile(fn): with open(fn, 'r') as fh: ciwf = yaml.safe_load(fh) - + # Check that the action is turned on for push and pull requests try: assert('push' in ciwf[True]) @@ -544,7 +545,7 @@ def check_actions_ci(self): docker_pull_cmd = 'docker pull {}:dev && docker tag {}:dev {}\n'.format(docker_notag, docker_notag, docker_withtag) try: steps = ciwf['jobs']['test']['steps'] - assert(any([docker_pull_cmd in step['run'] for step in steps[1:]])) + assert(any([docker_pull_cmd in step['run'] for step in steps if 'run' in step.keys()])) except AssertionError: self.failed.append((5, "CI is not pulling and tagging the correct docker image. Should be:\n '{}'".format(docker_pull_cmd))) else: @@ -559,8 +560,48 @@ def check_actions_ci(self): except AssertionError: self.failed.append((5, "Minimum NF version differed from CI and what was set in the pipelines manifest: {}".format(fn))) else: - self.passed.append((5, "Continuous integration checks minimum NF version: '{}'".format(fn))) + self.passed.append((5, "Continuous integration checks minimum NF version: '{}'".format(fn))) + def check_actions_lint(self): + """Checks that the GitHub actions lint workflow is valid + + Makes sure ``nf-core lint`` and ``markdownlint`` runs. + """ + fn = os.path.join(self.path, '.github', 'workflows', 'linting.yml') + if os.path.isfile(fn): + with open(fn, 'r') as fh: + lintwf = yaml.safe_load(fh) + + # Check that the action is turned on for push and pull requests + try: + assert('push' in lintwf[True]) + assert('pull_request' in lintwf[True]) + except (AssertionError, KeyError, TypeError): + self.failed.append((5, "GitHub actions linting workflow must be triggered on PR and push: '{}'".format(fn))) + else: + self.passed.append((5, "GitHub actions linting workflow is triggered on PR and push: '{}'".format(fn))) + + # Check that the Markdown linting runs + Markdownlint_cmd = 'markdownlint ${GITHUB_WORKSPACE} -c ${GITHUB_WORKSPACE}/.github/markdownlint.yml' + try: + steps = lintwf['jobs']['Markdown']['steps'] + assert(any([Markdownlint_cmd in step['run'] for step in steps if 'run' in step.keys()])) + except (AssertionError, KeyError, TypeError): + self.failed.append((5, "Continuous integration must run Markdown lint Tests: '{}'".format(fn))) + else: + self.passed.append((5, "Continuous integration runs Markdown lint Tests: '{}'".format(fn))) + + + # Check that the nf-core linting runs + nfcore_lint_cmd = 'nf-core lint ${GITHUB_WORKSPACE}' + try: + steps = lintwf['jobs']['nf-core']['steps'] + assert(any([ nfcore_lint_cmd in step['run'] for step in steps if 'run' in step.keys()])) + except (AssertionError, KeyError, TypeError): + self.failed.append((5, "Continuous integration must run nf-core lint Tests: '{}'".format(fn))) + else: + self.passed.append((5, "Continuous integration runs nf-core lint Tests: '{}'".format(fn))) + def check_ci_config(self): """Checks that the Travis or Circle CI YAML config is valid. diff --git a/tests/lint_examples/failing_example/.github/workflows/linting.yml b/tests/lint_examples/failing_example/.github/workflows/linting.yml new file mode 100644 index 000000000..0c774d0fe --- /dev/null +++ b/tests/lint_examples/failing_example/.github/workflows/linting.yml @@ -0,0 +1,40 @@ +name: nf-core linting +# This workflow is triggered on pushes and PRs to the repository. +# It runs the `nf-core lint` and markdown lint tests to ensure that the code meets the nf-core guidelines +on: + +jobs: + Markdown: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-node@v1 + with: + node-version: '10' + - name: Install markdownlint + run: | + npm install -g markdownlint-cli + - name: Run Markdownlint + run: | + nf-core: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Install Nextflow + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + - uses: actions/setup-python@v1 + with: + python-version: '3.6' + architecture: 'x64' + - name: Install pip + run: | + sudo apt install python3-pip + pip install --upgrade pip + - name: Install nf-core tools + run: | + pip install nf-core + - name: Run nf-core lint + run: | + \ No newline at end of file diff --git a/tests/test_lint.py b/tests/test_lint.py index fa434fc26..492b5d971 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -39,7 +39,7 @@ def pf(wd, path): PATH_ACTIONS_FAILING_EXAMPLE = pf(WD, 'lint_examples/actions_failing_example') # The maximum sum of passed tests currently possible -MAX_PASS_CHECKS = 72 +MAX_PASS_CHECKS = 76 # The additional tests passed for releases ADD_PASS_RELEASE = 1 @@ -92,7 +92,7 @@ def test_failing_missingfiles_example(self): """Tests for missing files like Dockerfile or LICENSE""" lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) lint_obj.check_files_exist() - expectations = {"failed": 5, "warned": 3, "passed": len(listfiles(PATH_WORKING_EXAMPLE)) - 3 - 1} + expectations = {"failed": 5, "warned": 2, "passed": len(listfiles(PATH_WORKING_EXAMPLE)) - 2 - 1} self.assess_lint_status(lint_obj, **expectations) def test_mit_licence_example_pass(self): @@ -175,6 +175,20 @@ def test_actions_wf_ci_fail_wrong_NF_version(self): expectations = {"failed": 1, "warned": 0, "passed": 2} self.assess_lint_status(lint_obj, **expectations) + def test_actions_wf_lint_pass(self): + """Tests that linting for GitHub actions linting wf works for a good example""" + lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) + lint_obj.check_actions_lint() + expectations = {"failed": 0, "warned": 0, "passed": 3} + self.assess_lint_status(lint_obj, **expectations) + + def test_actions_wf_lint_fail(self): + """Tests that linting for GitHub actions linting wf fails for a bad example""" + lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) + lint_obj.check_actions_lint() + expectations = {"failed": 3, "warned": 0, "passed": 0} + self.assess_lint_status(lint_obj, **expectations) + def test_ci_conf_pass(self): """Tests that the continous integration config checks work with a good example""" lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) From b8c9b63795bd50d4947d8bbdd15d1ac7be5e8c05 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Wed, 11 Dec 2019 13:15:26 +0100 Subject: [PATCH 179/224] updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cdbd4655..eef2d975e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ * Adjusted linting to enable `patch` branches from being tested * `.travis.yml` now optional, `.circle.yml` and `Singularity` must not be present * Warn if GitHub actions workflows do not exist +* Added linting of GitHub actions workflows `linting.yml`, `ci.yml` and `branch.yml` * Warn if pipeline name contains upper case letters or non alphabetical characters [#85](https://github.com/nf-core/tools/issues/85) ### Template From 19dd02035e812411d4babd41eaedba4d8e312ff2 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Wed, 11 Dec 2019 13:21:08 +0100 Subject: [PATCH 180/224] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eef2d975e..91797c3cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ * Add Python 3.7 dependency to the `environment.yml` file * Remove awsbatch profile cf [nf-core/configs#71](https://github.com/nf-core/configs/pull/71) * Make `scrape_software_versions.py` compatible with Python3 to enable miniconda3 in [base image PR](https://github.com/nf-core/tools/pull/462) +* Add `docker pull` command to GitHub actions CI workflow ### Base Docker image From c28555a14227a0bffdb28e4c9615b8d9586bd544 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Wed, 11 Dec 2019 13:26:54 +0100 Subject: [PATCH 181/224] fixed missing error type check actions --- nf_core/lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index daa846f09..68d259e01 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -548,7 +548,7 @@ def check_actions_ci(self): try: steps = ciwf['jobs']['test']['steps'] assert(any([docker_pull_cmd in step['run'] for step in steps if 'run' in step.keys()])) - except AssertionError: + except (AssertionError, KeyError, TypeError): self.failed.append((5, "CI is not pulling and tagging the correct docker image. Should be:\n '{}'".format(docker_pull_cmd))) else: self.passed.append((5, "CI is pulling and tagging the correct docker image: {}".format(docker_pull_cmd))) From f42f46a5b1266ee5072c8899848efae73e87d285 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Wed, 11 Dec 2019 13:32:08 +0100 Subject: [PATCH 182/224] rm path actions failing example --- tests/test_lint.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_lint.py b/tests/test_lint.py index 71b1b749d..840d320a0 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -36,7 +36,6 @@ def pf(wd, path): PATH_MISSING_LICENSE_EXAMPLE = pf(WD, 'lint_examples/missing_license_example') PATHS_WRONG_LICENSE_EXAMPLE = [pf(WD, 'lint_examples/wrong_license_example'), pf(WD, 'lint_examples/license_incomplete_example')] -PATH_ACTIONS_FAILING_EXAMPLE = pf(WD, 'lint_examples/actions_failing_example') # The maximum sum of passed tests currently possible MAX_PASS_CHECKS = 76 From 47d2a899baf9f05037b42765ee4d5633b6d38c12 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Wed, 11 Dec 2019 14:11:42 +0100 Subject: [PATCH 183/224] fixed issue 2 minimal working examples --- .../.github/workflows/branch.yml | 0 .../.github/workflows/ci.yml | 0 .../.github/workflows/linting.yml | 0 tests/test_lint.py | 4 ++-- 4 files changed, 2 insertions(+), 2 deletions(-) rename tests/lint_examples/{minimal_working_example => minimalworkingexample}/.github/workflows/branch.yml (100%) rename tests/lint_examples/{minimal_working_example => minimalworkingexample}/.github/workflows/ci.yml (100%) rename tests/lint_examples/{minimal_working_example => minimalworkingexample}/.github/workflows/linting.yml (100%) diff --git a/tests/lint_examples/minimal_working_example/.github/workflows/branch.yml b/tests/lint_examples/minimalworkingexample/.github/workflows/branch.yml similarity index 100% rename from tests/lint_examples/minimal_working_example/.github/workflows/branch.yml rename to tests/lint_examples/minimalworkingexample/.github/workflows/branch.yml diff --git a/tests/lint_examples/minimal_working_example/.github/workflows/ci.yml b/tests/lint_examples/minimalworkingexample/.github/workflows/ci.yml similarity index 100% rename from tests/lint_examples/minimal_working_example/.github/workflows/ci.yml rename to tests/lint_examples/minimalworkingexample/.github/workflows/ci.yml diff --git a/tests/lint_examples/minimal_working_example/.github/workflows/linting.yml b/tests/lint_examples/minimalworkingexample/.github/workflows/linting.yml similarity index 100% rename from tests/lint_examples/minimal_working_example/.github/workflows/linting.yml rename to tests/lint_examples/minimalworkingexample/.github/workflows/linting.yml diff --git a/tests/test_lint.py b/tests/test_lint.py index 840d320a0..4fd8decae 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -38,7 +38,7 @@ def pf(wd, path): pf(WD, 'lint_examples/license_incomplete_example')] # The maximum sum of passed tests currently possible -MAX_PASS_CHECKS = 76 +MAX_PASS_CHECKS = 77 # The additional tests passed for releases ADD_PASS_RELEASE = 1 @@ -91,7 +91,7 @@ def test_failing_missingfiles_example(self): """Tests for missing files like Dockerfile or LICENSE""" lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) lint_obj.check_files_exist() - expectations = {"failed": 5, "warned": 2, "passed": len(listfiles(PATH_WORKING_EXAMPLE)) - 2 - 1} + expectations = {"failed": 5, "warned": 2, "passed": 13} self.assess_lint_status(lint_obj, **expectations) def test_mit_licence_example_pass(self): From 27ae6d9770902773743fdbd2c25120dd66fd37f4 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Wed, 11 Dec 2019 14:24:41 +0100 Subject: [PATCH 184/224] fixed issue in template GH actions branch.yml --- .../{{cookiecutter.name_noslash}}/.github/workflows/branch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml index f7c906c29..1018f6d25 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml @@ -13,4 +13,4 @@ jobs: # PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch - name: Check PRs run: | - { [[ $(git remote get-url origin) == *{{cookiecutter.name_noslash}} ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]] + { [[ $(git remote get-url origin) == *{{cookiecutter.name}} ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]] From ada1681296d9a49f3bc9422a0d82209da5657dbb Mon Sep 17 00:00:00 2001 From: ggabernet Date: Wed, 11 Dec 2019 14:29:09 +0100 Subject: [PATCH 185/224] remove .circleci/config.yml from blacklist --- nf_core/lint.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index 68d259e01..fc14957d8 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -216,7 +216,6 @@ def check_files_exist(self): Files that *must not* be present:: - '.circle.yml', 'Singularity' Files that *should not* be present:: @@ -253,7 +252,6 @@ def check_files_exist(self): ] files_fail_ifexists = [ 'Singularity', - '.circle.yml' ] files_warn_ifexists = [ '.travis.yml' @@ -610,7 +608,7 @@ def check_ci_config(self): Makes sure that ``nf-core lint`` runs in travis tests and that tests run with the required nextflow version. """ - for cf in ['.travis.yml', 'circle.yml']: + for cf in ['.travis.yml', os.path.join('.circleci','config.yml')]: fn = os.path.join(self.path, cf) if os.path.isfile(fn): with open(fn, 'r') as fh: From fb2abdcf99ef8e4c3d8c62f12a228d0f57242b9d Mon Sep 17 00:00:00 2001 From: ggabernet Date: Wed, 11 Dec 2019 14:31:47 +0100 Subject: [PATCH 186/224] fixed n passing tests --- tests/test_lint.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_lint.py b/tests/test_lint.py index 4fd8decae..c9b361c3e 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -38,7 +38,7 @@ def pf(wd, path): pf(WD, 'lint_examples/license_incomplete_example')] # The maximum sum of passed tests currently possible -MAX_PASS_CHECKS = 77 +MAX_PASS_CHECKS = 76 # The additional tests passed for releases ADD_PASS_RELEASE = 1 @@ -91,7 +91,7 @@ def test_failing_missingfiles_example(self): """Tests for missing files like Dockerfile or LICENSE""" lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) lint_obj.check_files_exist() - expectations = {"failed": 5, "warned": 2, "passed": 13} + expectations = {"failed": 5, "warned": 2, "passed": 12} self.assess_lint_status(lint_obj, **expectations) def test_mit_licence_example_pass(self): From 76b3763170780df4953dc4c8535775c8e7638ad3 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Wed, 11 Dec 2019 14:34:01 +0100 Subject: [PATCH 187/224] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91797c3cd..541bb4b07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ ### Linting * Adjusted linting to enable `patch` branches from being tested -* `.travis.yml` now optional, `.circle.yml` and `Singularity` must not be present +* `.travis.yml` and circleCI now optional, `Singularity` must not be present * Warn if GitHub actions workflows do not exist * Added linting of GitHub actions workflows `linting.yml`, `ci.yml` and `branch.yml` * Warn if pipeline name contains upper case letters or non alphabetical characters [#85](https://github.com/nf-core/tools/issues/85) From 54d19a6cc611b04d2b8f5361562b25e7c7e8aeb4 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Thu, 12 Dec 2019 12:26:38 +0000 Subject: [PATCH 188/224] Add awscli parameter --- .../{{cookiecutter.name_noslash}}/main.nf | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index 8b8e4fe5d..e8341c60d 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -21,27 +21,28 @@ def helpMessage() { nextflow run {{ cookiecutter.name }} --reads '*_R{1,2}.fastq.gz' -profile docker Mandatory arguments: - --reads Path to input data (must be surrounded with quotes) - -profile Configuration profile to use. Can use multiple (comma separated) - Available: conda, docker, singularity, awsbatch, test and more + --reads [file] Path to input data (must be surrounded with quotes) + -profile [str] Configuration profile to use. Can use multiple (comma separated) + Available: conda, docker, singularity, test, awsbatch and more Options: - --genome Name of iGenomes reference - --single_end Specifies that the input is single-end reads + --genome [str] Name of iGenomes reference + --single_end [bool] Specifies that the input is single-end reads - References If not specified in the configuration file or you wish to overwrite any of the references - --fasta Path to fasta reference + References If not specified in the configuration file or you wish to overwrite any of the references + --fasta [file] Path to fasta reference Other options: - --outdir The output directory where the results will be saved - --email Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits - --email_on_fail Same as --email, except only send mail if the workflow is not successful - --max_multiqc_email_size Theshold size for MultiQC report to be attached in notification email. If file generated by pipeline exceeds the threshold, it will not be attached (Default: 25MB) - -name Name for the pipeline run. If not specified, Nextflow will automatically generate a random mnemonic + --outdir [file] The output directory where the results will be saved + --email [email] Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits + --email_on_fail [email] Same as --email, except only send mail if the workflow is not successful + --max_multiqc_email_size [str] Theshold size for MultiQC report to be attached in notification email. If file generated by pipeline exceeds the threshold, it will not be attached (Default: 25MB) + -name [str] Name for the pipeline run. If not specified, Nextflow will automatically generate a random mnemonic AWSBatch options: - --awsqueue The AWSBatch JobQueue that needs to be set when running on AWSBatch - --awsregion The AWS Region for your AWS Batch job to run on + --awsqueue [str] The AWSBatch JobQueue that needs to be set when running on AWSBatch + --awsregion [str] The AWS Region for your AWS Batch job to run on + --awscli [str] Path to the AWS CLI tool """.stripIndent() } From 795cf0920064742e16c02e5d3c961199b0ed82ab Mon Sep 17 00:00:00 2001 From: drpatelh Date: Thu, 12 Dec 2019 12:26:52 +0000 Subject: [PATCH 189/224] Add aws parameters back in --- .../{{cookiecutter.name_noslash}}/nextflow.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config index 7e2d548de..cb0b77c50 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config @@ -34,6 +34,11 @@ params { config_profile_contact = false config_profile_url = false + // AWSBatch + awsqueue = false + awsregion = 'eu-west-1' + awscli = '/home/ec2-user/miniconda/bin/aws' + // Defaults only, expecting to be overwritten max_memory = 128.GB max_cpus = 16 From e50a15d50e2cc8b43ebb78541ee9af7093c1c5a0 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Thu, 12 Dec 2019 12:33:37 +0000 Subject: [PATCH 190/224] Update usage docs --- .../{{cookiecutter.name_noslash}}/docs/usage.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index c4b0d579f..727f8d566 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -21,6 +21,7 @@ * [AWS Batch specific parameters](#aws-batch-specific-parameters) * [`--awsqueue`](#--awsqueue) * [`--awsregion`](#--awsregion) + * [`--awscli`](#--awscli) * [Other command line parameters](#other-command-line-parameters) * [`--outdir`](#--outdir) * [`--email`](#--email) @@ -100,8 +101,6 @@ They are loaded in sequence, so later profiles can overwrite earlier profiles. If `-profile` is not specified, the pipeline will run locally and expect all software to be installed and available on the `PATH`. This is _not_ recommended. -* `awsbatch` - * A generic configuration profile to be used with AWS Batch. * `conda` * A generic configuration profile to be used with [conda](https://conda.io/docs/) * Pulls most software from [Bioconda](https://bioconda.github.io/) @@ -211,7 +210,7 @@ If you have any questions or issues please send us a message on [Slack](https:// ## AWS Batch specific parameters -Running the pipeline on AWS Batch requires a couple of specific parameters to be set according to your AWS Batch configuration. Please use the `-awsbatch` profile and then specify all of the following parameters. +Running the pipeline on AWS Batch requires a couple of specific parameters to be set according to your AWS Batch configuration. Please use [`-profile awsbatch`](https://github.com/nf-core/configs/blob/master/conf/awsbatch.config) and then specify all of the following parameters. ### `--awsqueue` @@ -219,7 +218,11 @@ The JobQueue that you intend to use on AWS Batch. ### `--awsregion` -The AWS region to run your job in. Default is set to `eu-west-1` but can be adjusted to your needs. +The AWS region in which to run your job. Default is set to `eu-west-1` but can be adjusted to your needs. + +### `--awscli` + +The [AWS CLI](https://www.nextflow.io/docs/latest/awscloud.html#aws-cli-installation) path in your custom AMI. Default: `/home/ec2-user/miniconda/bin/aws`. Please make sure to also set the `-w/--work-dir` and `--outdir` parameters to a S3 storage bucket of your choice - you'll get an error message notifying you if you didn't. @@ -269,7 +272,7 @@ Note - you can use this to override pipeline defaults. ### `--custom_config_version` -Provide git commit id for custom Institutional configs hosted at `nf-core/configs`. This was implemented for reproducibility purposes. Default is set to `master`. +Provide git commit id for custom Institutional configs hosted at `nf-core/configs`. This was implemented for reproducibility purposes. Default: `master`. ```bash ## Download and use config file with following git commid id From e5dd9d367e7aa3d3b81db86ce31ce7733d59acf6 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Thu, 12 Dec 2019 12:33:55 +0000 Subject: [PATCH 191/224] Add input type definitions --- nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index e8341c60d..ab07fa604 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -136,6 +136,7 @@ summary['User'] = workflow.userName if (workflow.profile.contains('awsbatch')) { summary['AWS Region'] = params.awsregion summary['AWS Queue'] = params.awsqueue + summary['AWS CLI'] = params.awscli } summary['Config Profile'] = workflow.profile if (params.config_profile_description) summary['Config Description'] = params.config_profile_description From 7abbe1a0239bf61deddac6bbdb250acc323d62c4 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Thu, 12 Dec 2019 12:40:14 +0000 Subject: [PATCH 192/224] Update CHANGELOG --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9ed4b5ad..6bc8c509a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ * Adjusted linting to enable `patch` branches from being tested * Warn if pipeline name contains upper case letters or non alphabetical characters [#85](https://github.com/nf-core/tools/issues/85) -### Template +### Template pipeline * Fixed incorrect paths in iGenomes config as described in issue [#418](https://github.com/nf-core/tools/issues/418) * Fixed [incorrect usage of non-existant parameter](https://github.com/nf-core/tools/issues/446) in the template @@ -40,7 +40,8 @@ * Adjusted `.travis.yml` checks to allow for `patch` branches to be tested * Add Python 3.7 dependency to the `environment.yml` file * Remove awsbatch profile cf [nf-core/configs#71](https://github.com/nf-core/configs/pull/71) -* Make `scrape_software_versions.py` compatible with Python3 to enable miniconda3 in [base image PR](https://github.com/nf-core/tools/pull/462) +* Make `scrape_software_versions.py` compatible with Python3 to enable miniconda3 in [base image PR](https://github.com/nf-core/tools/pull/462) +* Add `--awscli` parameter ### Base Docker image @@ -80,7 +81,7 @@ * If the container slug does not contain the nf-core organisation (for example during development on a fork), linting will raise a warning, and an error with release mode on -### Template +### Template pipeline * Add new code for Travis CI to allow PRs from patch branches too * Fix small typo in central readme of tools for future releases From 75828e733486ae4069862bffe0ee46e1a06e78d2 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Thu, 12 Dec 2019 13:47:56 +0000 Subject: [PATCH 193/224] Remove awsbatch params --- .../{{cookiecutter.name_noslash}}/nextflow.config | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config index cb0b77c50..7e2d548de 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config @@ -34,11 +34,6 @@ params { config_profile_contact = false config_profile_url = false - // AWSBatch - awsqueue = false - awsregion = 'eu-west-1' - awscli = '/home/ec2-user/miniconda/bin/aws' - // Defaults only, expecting to be overwritten max_memory = 128.GB max_cpus = 16 From a8dcaad2df2979f22817a92d88efc7eb425ce961 Mon Sep 17 00:00:00 2001 From: Gisela Gabernet Date: Mon, 16 Dec 2019 21:14:01 +0100 Subject: [PATCH 194/224] Update nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml Co-Authored-By: Harshil Patel --- .../{{cookiecutter.name_noslash}}/.github/workflows/branch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml index 1018f6d25..2d75670cf 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml @@ -13,4 +13,4 @@ jobs: # PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch - name: Check PRs run: | - { [[ $(git remote get-url origin) == *{{cookiecutter.name}} ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]] + { [[ $(git remote get-url origin) == *{{cookiecutter.short_name}} ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]] From 6cd48b2cd19fbe02d56329885ab9d75dddc2b896 Mon Sep 17 00:00:00 2001 From: Gisela Gabernet Date: Mon, 16 Dec 2019 21:44:11 +0100 Subject: [PATCH 195/224] Update nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml Co-Authored-By: Harshil Patel --- .../{{cookiecutter.name_noslash}}/.github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml index a9eb162dd..1fead64e7 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml @@ -22,4 +22,4 @@ jobs: docker pull {{ cookiecutter.name_docker }}:dev && docker tag {{ cookiecutter.name_docker }}:dev {{ cookiecutter.name_docker }}:dev - name: Run test run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker + nextflow run ${GITHUB_WORKSPACE} -ansi-log false -profile test,docker From ab9c66dd9104e42f889b565e2c659ba16250546a Mon Sep 17 00:00:00 2001 From: ggabernet Date: Mon, 16 Dec 2019 22:41:56 +0100 Subject: [PATCH 196/224] added docs for lint errors --- docs/lint_errors.md | 74 ++++++++++++++++--- nf_core/lint.py | 23 +++--- .../.github/workflows/branch.yml | 2 +- 3 files changed, 77 insertions(+), 22 deletions(-) diff --git a/docs/lint_errors.md b/docs/lint_errors.md index 896f16b6f..464ff334c 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -2,16 +2,19 @@ This page contains detailed descriptions of the tests done by the [nf-core/tools](https://github.com/nf-core/tools) package. Linting errors should show URLs next to any failures that link to the relevant heading below. -## Error #1 - File not found ## {#1} +## Error #1 - File not found / must be removed ## {#1} -nf-core pipelines should adhere to a common file structure for consistency. The lint test looks for the following required files: +nf-core pipelines should adhere to a common file structure for consistency. + +The lint test looks for the following required files: * `nextflow.config` * The main nextflow config file * `Dockerfile` * A docker build script to generate a docker image with the required software -* `.travis.yml` or `circle.yml` - * A config file for automated continuous testing with either [Travis CI](https://travis-ci.com/) or [Circle CI](https://circleci.com/) +* Continuous integration tests with either [GitHub actions](https://github.com/features/actions), [Travis CI](https://travis-ci.com/) or [Circle CI](https://circleci.com/) + * GitHub actions workflows for CI (`.github/workflows/ci.yml`), branch protection (`.github/workflows/branch.yml`) and linting (`.github/workflows/linting.yml`) + * Alternatively, `.travis.yml` or `.circleci/config.yml` continuous testing are still allowed but might be deprecated * `LICENSE`, `LICENSE.md`, `LICENCE.md` or `LICENCE.md` * The MIT licence. Copy from [here](https://raw.githubusercontent.com/nf-core/tools/master/LICENSE). * `README.md` @@ -28,6 +31,10 @@ The following files are suggested but not a hard requirement. If they are missin * `conf/base.config` * A `conf` directory with at least one config called `base.config` +Additionally, the following files must not be present: + +* `Singularity` + ## Error #2 - Docker file check failed ## {#2} Pipelines should have a files called `Dockerfile` in their root directory. @@ -110,14 +117,61 @@ The following variables are depreciated and fail the test if they are still pres ## Error #5 - Continuous Integration configuration ## {#5} -nf-core pipelines must have CI testing with Travis or Circle CI. +nf-core pipelines must have CI testing with GitHub actions, Travis or Circle CI. + +This test fails if the following requirements are not met: + +For GitHub actions CI workflow: + +* `.github/workflows/ci.yml` must be turned on for `push` and `pull_request`` +* `.github/workflows/ci.yml` must test with the minimum Nextflow version specified in the pipline as `manifest.nextflowVersion` under `jobs`, `test`, `strategy`, `matrix`, `nxf_ver`. E.g.: + + ```yaml + jobs: + test: + runs-on: ubuntu-18.04 + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + +* `.github/workflows/ci.yml` must pull the container with the command `docker pull :dev && docker tag :dev :tag` under `jobs`,`test`,`steps`. E.g. for nfcore/tools container: + + ```yaml + jobs: + test: + runs-on: ubuntu-18.04 + steps: + - name: Pull image + run: | + docker pull nfcore/tools:dev && docker tag nfcore/tools:dev nfcore/tools:1.0.0 + ``` + +* `.github/workflows/branch.yml` must be turned on for pull requests to `master` +* `.github/workflows/branch.yml` must check that PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch. E.g. for nf-core/tools: + + ```yaml + jobs: + test: + runs-on: ubuntu-18.04 + steps: + # PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch + - name: Check PRs + run: | + { [[ $(git remote get-url origin) == *nf-core/tools ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]] + ``` + +* `.github/workflows/linting.yml` must be turned on for push and pull requests +* `.github/workflows/linting.yml` must perform markdown linting with the command `markdownlint ${GITHUB_WORKSPACE} -c ${GITHUB_WORKSPACE}/.github/markdownlint.yml` under `jobs`, `Markdown`, `steps`. +* `.github/workflows/linting.yml` must perform nf-core linting with the command `nf-core lint ${GITHUB_WORKSPACE}` under `jobs`, `nf-core`, `steps`. -This test fails if the following happens: +For Travis CI: -* `.travis.yml` does not contain the string `nf-core lint ${TRAVIS_BUILD_DIR}` under `script` -* `.travis.yml` does not contain the string `docker pull :dev` under `before_install` +* `.travis.yml` must contain the string `nf-core lint ${TRAVIS_BUILD_DIR}` under `script` +* `.travis.yml` must contain the string `docker pull :dev` under `before_install` * Where `` is fetched from `process.container` in the `nextflow.config` file, without the docker tag _(if we have the tag the tests fail when making a release)_ -* `.travis.yml` does not test the Nextflow version specified in the pipeline as `manifest.nextflowVersion` +* `.travis.yml` must test the Nextflow version specified in the pipeline as `manifest.nextflowVersion` * This is expected in the `env` section of the config, eg: ```yaml @@ -128,7 +182,7 @@ This test fails if the following happens: * At least one of these `NXF_VER` variables must match the `manifest.nextflowVersion` version specified in the pipeline config * Other variables can be specified on these lines as long as they are space separated. -* `.travis.yml` checks that pull requests are not opened directly to the `master` branch +* `.travis.yml` must check that pull requests are not opened directly to the `master` branch * The following is expected in the `before_install` section: ```yaml diff --git a/nf_core/lint.py b/nf_core/lint.py index fc14957d8..e22dbe13c 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -234,27 +234,28 @@ def check_files_exist(self): ['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling 'README.md', 'CHANGELOG.md', - 'docs/README.md', - 'docs/output.md', - 'docs/usage.md', - ['.travis.yml', '.github/workflows/branch.yml'], - ['.travis.yml', '.github/workflows/ci.yml'], - ['.travis.yml', '.github/workflows/linting.yml'] + os.path.join('docs','README.md'), + os.path.join('docs','output.md'), + os.path.join('docs','usage.md'), + ['.travis.yml', os.path.join('.github', 'workflows', 'branch.yml'), os.path.join('.circleci','config.yml')], + ['.travis.yml', os.path.join('.github', 'workflows','ci.yml'), os.path.join('.circleci','config.yml')], + ['.travis.yml', os.path.join('.github', 'workflows', 'linting.yml'), os.path.join('.circleci','config.yml')] ] files_warn = [ 'main.nf', 'environment.yml', - 'conf/base.config', - '.github/workflows/branch.yml', - '.github/workflows/ci.yml', - '.github/workflows/linting.yml' + os.path.join('conf','base.config'), + os.path.join('.github', 'workflows', 'branch.yml'), + os.path.join('.github', 'workflows','ci.yml'), + os.path.join('.github', 'workflows', 'linting.yml') ] files_fail_ifexists = [ 'Singularity', ] files_warn_ifexists = [ - '.travis.yml' + '.travis.yml', + os.path.join('.circleci','config.yml') ] def pf(file_path): diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml index 2d75670cf..1018f6d25 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml @@ -13,4 +13,4 @@ jobs: # PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch - name: Check PRs run: | - { [[ $(git remote get-url origin) == *{{cookiecutter.short_name}} ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]] + { [[ $(git remote get-url origin) == *{{cookiecutter.name}} ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]] From 151144bbd0235441375593392932b14d2e880c84 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Mon, 16 Dec 2019 22:57:32 +0100 Subject: [PATCH 197/224] updated number passing tests --- tests/test_lint.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_lint.py b/tests/test_lint.py index c9b361c3e..4fd8decae 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -38,7 +38,7 @@ def pf(wd, path): pf(WD, 'lint_examples/license_incomplete_example')] # The maximum sum of passed tests currently possible -MAX_PASS_CHECKS = 76 +MAX_PASS_CHECKS = 77 # The additional tests passed for releases ADD_PASS_RELEASE = 1 @@ -91,7 +91,7 @@ def test_failing_missingfiles_example(self): """Tests for missing files like Dockerfile or LICENSE""" lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) lint_obj.check_files_exist() - expectations = {"failed": 5, "warned": 2, "passed": 12} + expectations = {"failed": 5, "warned": 2, "passed": 13} self.assess_lint_status(lint_obj, **expectations) def test_mit_licence_example_pass(self): From 70fb94674257a744bd9ef9677a264571214a0e17 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Tue, 17 Dec 2019 09:46:24 +0100 Subject: [PATCH 198/224] update changelog --- CHANGELOG.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 686f99f69..42efadcbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,6 @@ * `nf-core bump-version` now also bumps the version string of the exported conda environment in the Dockerfile * Updated Blacklist of synced pipelines * Ignore pre-releases in `nf-core list` -* Lint for `Singularity` file [and remove it](https://github.com/nf-core/tools/issues/458) * Updated documentation for `nf-core download` * Fixed typo in `nf-core launch` final command * Handle missing pipeline descriptions in `nf-core list` @@ -16,8 +15,8 @@ ### Linting * Adjusted linting to enable `patch` branches from being tested -* `.travis.yml` and circleCI now optional, `Singularity` must not be present -* Warn if GitHub actions workflows do not exist +* Warn if GitHub actions workflows do not exist, warn if `.travis.yml` and circleCI are there +* Lint for `Singularity` file [and remove it](https://github.com/nf-core/tools/issues/458) * Added linting of GitHub actions workflows `linting.yml`, `ci.yml` and `branch.yml` * Warn if pipeline name contains upper case letters or non alphabetical characters [#85](https://github.com/nf-core/tools/issues/85) @@ -34,9 +33,9 @@ * Having the old camelCase versions of these will now throw an error * Add `autoMounts=true` to default singularity profile * Add in `markdownlint` checks that were being ignored by default -* Disable ansi logging in the travis CI tests. +* Disable ansi logging in the travis CI tests * Move `params`section from `base.config` to `nextflow.config` -* Use `env` scope to export `PYTHONNOUSERSITE` in `nextflow.config` to prevent conflicts with host Python environment. +* Use `env` scope to export `PYTHONNOUSERSITE` in `nextflow.config` to prevent conflicts with host Python environment * Bump minimum Nextflow version to `19.10.0` - required to properly use `env` scope in `nextflow.config` * Added support for nf-tower in the travis tests, using public mailbox nf-core@mailinator.com * Add link to [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and [Semantic Versioning](http://semver.org/spec/v2.0.0.html) to CHANGELOG @@ -44,7 +43,7 @@ * Add Python 3.7 dependency to the `environment.yml` file * Remove awsbatch profile cf [nf-core/configs#71](https://github.com/nf-core/configs/pull/71) * Make `scrape_software_versions.py` compatible with Python3 to enable miniconda3 in [base image PR](https://github.com/nf-core/tools/pull/462) -* Add `docker pull` command to GitHub actions CI workflow +* Added GitHub actions workflows and respective linting * Add `--awscli` parameter ### Base Docker image From 0f41f5d4f04b61d806aa6e5704ee64db2d3201d8 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Tue, 17 Dec 2019 10:19:04 +0100 Subject: [PATCH 199/224] added NXF_ANSI_LOG variable export to template CI --- CHANGELOG.md | 3 ++- .../{{cookiecutter.name_noslash}}/.github/workflows/ci.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42efadcbd..b63b855ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,8 @@ * Add Python 3.7 dependency to the `environment.yml` file * Remove awsbatch profile cf [nf-core/configs#71](https://github.com/nf-core/configs/pull/71) * Make `scrape_software_versions.py` compatible with Python3 to enable miniconda3 in [base image PR](https://github.com/nf-core/tools/pull/462) -* Added GitHub actions workflows and respective linting +* Add GitHub actions workflows and respective linting +* Add NXF_ANSI_LOG as global environment variable to template GitHub actions CI workflow * Add `--awscli` parameter ### Base Docker image diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml index 1fead64e7..326e5c20f 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml @@ -15,6 +15,7 @@ jobs: - name: Install Nextflow run: | {% raw %}export NXF_VER=${{ matrix.nxf_ver }}{% endraw %} + export NXF_ANSI_LOG=false wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - name: Pull image @@ -22,4 +23,4 @@ jobs: docker pull {{ cookiecutter.name_docker }}:dev && docker tag {{ cookiecutter.name_docker }}:dev {{ cookiecutter.name_docker }}:dev - name: Run test run: | - nextflow run ${GITHUB_WORKSPACE} -ansi-log false -profile test,docker + nextflow run ${GITHUB_WORKSPACE} -profile test,docker From 70e628441cc210baa8a0a06fed0305b48d2cd656 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Fri, 17 Jan 2020 13:33:13 +0100 Subject: [PATCH 200/224] ci workflow env variables --- .../{{cookiecutter.name_noslash}}/.github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml index 326e5c20f..6c7b25814 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml @@ -5,6 +5,9 @@ on: [push, pull_request] jobs: test: + env: + NXF_VER: {% raw %}${{ matrix.nxf_ver }}{% endraw %} + NXF_ANSI_LOG: false runs-on: ubuntu-18.04 strategy: matrix: @@ -14,8 +17,6 @@ jobs: - uses: actions/checkout@v1 - name: Install Nextflow run: | - {% raw %}export NXF_VER=${{ matrix.nxf_ver }}{% endraw %} - export NXF_ANSI_LOG=false wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - name: Pull image From 34ca4eefe6a8f7bd74e81f0d6f07b5fe66797d37 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Fri, 17 Jan 2020 13:56:31 +0100 Subject: [PATCH 201/224] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b63b855ad..881646b5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ * Make `scrape_software_versions.py` compatible with Python3 to enable miniconda3 in [base image PR](https://github.com/nf-core/tools/pull/462) * Add GitHub actions workflows and respective linting * Add NXF_ANSI_LOG as global environment variable to template GitHub actions CI workflow +* Fixed global environment variable in GitHub actions CI workflow * Add `--awscli` parameter ### Base Docker image From 0c474642baff95c785a488ed534a26b7ce74f374 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Sun, 19 Jan 2020 12:33:05 +0100 Subject: [PATCH 202/224] revision comments CI workflow GitHub actions --- .../{{cookiecutter.name_noslash}}/.github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml index 6c7b25814..9a4d4c943 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml @@ -6,7 +6,7 @@ on: [push, pull_request] jobs: test: env: - NXF_VER: {% raw %}${{ matrix.nxf_ver }}{% endraw %} + NXF_VER: {{ matrix.nxf_ver }} NXF_ANSI_LOG: false runs-on: ubuntu-18.04 strategy: From babe1b248f14d0171cc8deb9d8756286ea9e73b8 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Sun, 19 Jan 2020 20:32:56 +0100 Subject: [PATCH 203/224] fix ci workflow env variable --- .../{{cookiecutter.name_noslash}}/.github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml index 9a4d4c943..e91e5bf83 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml @@ -6,7 +6,7 @@ on: [push, pull_request] jobs: test: env: - NXF_VER: {{ matrix.nxf_ver }} + NXF_VER: ${{ matrix.nxf_ver }} NXF_ANSI_LOG: false runs-on: ubuntu-18.04 strategy: From f704dca22262c327a9274ab5513202e111e65509 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Sun, 19 Jan 2020 21:16:48 +0100 Subject: [PATCH 204/224] fix env var ci workflow --- .../{{cookiecutter.name_noslash}}/.github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml index e91e5bf83..6c7b25814 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml @@ -6,7 +6,7 @@ on: [push, pull_request] jobs: test: env: - NXF_VER: ${{ matrix.nxf_ver }} + NXF_VER: {% raw %}${{ matrix.nxf_ver }}{% endraw %} NXF_ANSI_LOG: false runs-on: ubuntu-18.04 strategy: From 05aedd58f5f748bb989e0c0248ac3b205f1c74d9 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 24 Jan 2020 10:44:43 +0000 Subject: [PATCH 205/224] Update CHANGELOG --- CHANGELOG.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 881646b5a..8152636e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## v1.8dev +## v1.8 ### Tools helper code @@ -16,18 +16,18 @@ * Adjusted linting to enable `patch` branches from being tested * Warn if GitHub actions workflows do not exist, warn if `.travis.yml` and circleCI are there -* Lint for `Singularity` file [and remove it](https://github.com/nf-core/tools/issues/458) +* Lint for `Singularity` file and raise error if found [#458](https://github.com/nf-core/tools/issues/458) * Added linting of GitHub actions workflows `linting.yml`, `ci.yml` and `branch.yml` * Warn if pipeline name contains upper case letters or non alphabetical characters [#85](https://github.com/nf-core/tools/issues/85) ### Template pipeline * Fixed incorrect paths in iGenomes config as described in issue [#418](https://github.com/nf-core/tools/issues/418) -* Fixed [incorrect usage of non-existant parameter](https://github.com/nf-core/tools/issues/446) in the template +* Fixed incorrect usage of non-existent parameter in the template [#446](https://github.com/nf-core/tools/issues/446) * Add UCSC genomes to `igenomes.config` and add paths to all genome indices * Change `maxMultiqcEmailFileSize` parameter to `max_multiqc_email_size` * Export conda environment in Docker file [#349](https://github.com/nf-core/tools/issues/349) -* Change remaining parameters from `camelCase` to `snakeCase` [#39](https://github.com/nf-core/hic/issues/39) +* Change remaining parameters from `camelCase` to `snake_case` [#39](https://github.com/nf-core/hic/issues/39) * `--singleEnd` to `--single_end` * `--igenomesIgnore` to `--igenomes_ignore` * Having the old camelCase versions of these will now throw an error @@ -41,10 +41,10 @@ * Add link to [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and [Semantic Versioning](http://semver.org/spec/v2.0.0.html) to CHANGELOG * Adjusted `.travis.yml` checks to allow for `patch` branches to be tested * Add Python 3.7 dependency to the `environment.yml` file -* Remove awsbatch profile cf [nf-core/configs#71](https://github.com/nf-core/configs/pull/71) +* Remove `awsbatch` profile cf [nf-core/configs#71](https://github.com/nf-core/configs/pull/71) * Make `scrape_software_versions.py` compatible with Python3 to enable miniconda3 in [base image PR](https://github.com/nf-core/tools/pull/462) * Add GitHub actions workflows and respective linting -* Add NXF_ANSI_LOG as global environment variable to template GitHub actions CI workflow +* Add `NXF_ANSI_LOG` as global environment variable to template GitHub actions CI workflow * Fixed global environment variable in GitHub actions CI workflow * Add `--awscli` parameter From 615c6a57d0cddefb7f38bac1c4ea83704bfd9214 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 24 Jan 2020 12:00:06 +0100 Subject: [PATCH 206/224] Update CHANGELOG.md Co-Authored-By: Maxime Garcia --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 881646b5a..41da8d2b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## v1.8dev +## v1.8 ### Tools helper code From 509812a77e29d5f699700b16136282802d65d7dc Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 24 Jan 2020 16:58:34 +0000 Subject: [PATCH 207/224] Rewording --- docs/lint_errors.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/lint_errors.md b/docs/lint_errors.md index 464ff334c..cc8b0099c 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -101,8 +101,8 @@ The following variables throw warnings if missing: * `params.reads` * Input parameter to specify input data (typically FastQ files / pairs) * `params.single_end` - * Specify to work with single-end sequence data instead of default paired-end - * Used with Nextflow: `.fromFilePairs( params.reads, size: params.single_end ? 1 : 2 )` + * Specify to work with single-end sequence data instead of paired-end by default + * Nextflow implementation: `.fromFilePairs( params.reads, size: params.single_end ? 1 : 2 )` The following variables are depreciated and fail the test if they are still present: From 8c6d27fdb9979b5f68036f4bf1617c3f4398d3b6 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 24 Jan 2020 17:06:00 +0000 Subject: [PATCH 208/224] Update wording --- docs/lint_errors.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/lint_errors.md b/docs/lint_errors.md index cc8b0099c..80b34463a 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -113,7 +113,8 @@ The following variables are depreciated and fail the test if they are still pres * `params.container` * The old method for specifying the dockerhub container address. Replaced by `process.container` * `singleEnd` and `igenomesIgnore` - * Now using `snake_case` for all command line options + * Changed to `single_end` and `igenomes_ignore` + * The `snake_case` convention should now be used when defining pipeline parameters ## Error #5 - Continuous Integration configuration ## {#5} From 052887609f657eafe7f21e293d04981374f5d931 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 24 Jan 2020 17:09:49 +0000 Subject: [PATCH 209/224] Fix tag --- docs/lint_errors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/lint_errors.md b/docs/lint_errors.md index 80b34463a..97fa8d9ab 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -137,7 +137,7 @@ For GitHub actions CI workflow: nxf_ver: ['19.10.0', ''] ``` -* `.github/workflows/ci.yml` must pull the container with the command `docker pull :dev && docker tag :dev :tag` under `jobs`,`test`,`steps`. E.g. for nfcore/tools container: +* `.github/workflows/ci.yml` must pull the container with the command `docker pull :dev && docker tag :dev :` under `jobs`,`test`,`steps`. E.g. for nfcore/tools container: ```yaml jobs: From c16e2833257ee6f4a1968f8623f231a907825d7b Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 24 Jan 2020 17:16:27 +0000 Subject: [PATCH 210/224] Update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 585c40ffa..0d6d0847e 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages import sys -version = '1.8dev' +version = '1.8' with open('README.md') as f: readme = f.read() From ad243f3f44369e0546962fdf15172b6bb317e672 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 24 Jan 2020 17:21:35 +0000 Subject: [PATCH 211/224] Update wording --- docs/lint_errors.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/lint_errors.md b/docs/lint_errors.md index 97fa8d9ab..69d4ca402 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -12,9 +12,9 @@ The lint test looks for the following required files: * The main nextflow config file * `Dockerfile` * A docker build script to generate a docker image with the required software -* Continuous integration tests with either [GitHub actions](https://github.com/features/actions), [Travis CI](https://travis-ci.com/) or [Circle CI](https://circleci.com/) +* Continuous integration tests with either [GitHub actions](https://github.com/features/actions) or [Travis CI](https://travis-ci.com/) * GitHub actions workflows for CI (`.github/workflows/ci.yml`), branch protection (`.github/workflows/branch.yml`) and linting (`.github/workflows/linting.yml`) - * Alternatively, `.travis.yml` or `.circleci/config.yml` continuous testing are still allowed but might be deprecated + * Alternatively, `.travis.yml` continuous testing is still allowed but may be deprecated in the near future * `LICENSE`, `LICENSE.md`, `LICENCE.md` or `LICENCE.md` * The MIT licence. Copy from [here](https://raw.githubusercontent.com/nf-core/tools/master/LICENSE). * `README.md` From d7456c5af854c2c7c3b620c97699d02e8041bba8 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 24 Jan 2020 17:24:06 +0000 Subject: [PATCH 212/224] Remove CircleCI everywhere --- docs/lint_errors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/lint_errors.md b/docs/lint_errors.md index 69d4ca402..cd4b075aa 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -118,7 +118,7 @@ The following variables are depreciated and fail the test if they are still pres ## Error #5 - Continuous Integration configuration ## {#5} -nf-core pipelines must have CI testing with GitHub actions, Travis or Circle CI. +nf-core pipelines must have CI testing with GitHub actions or Travis. This test fails if the following requirements are not met: From d1051be673ff7b0e4a451914983fa3c3d6218598 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Sun, 26 Jan 2020 20:10:24 +0000 Subject: [PATCH 213/224] Add README path to igenomes.config --- .../conf/igenomes.config | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/igenomes.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/igenomes.config index 37217cf4e..2de924228 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/igenomes.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/igenomes.config @@ -18,6 +18,7 @@ params { bismark = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Annotation/README.txt" mito_name = "MT" macs_gsize = "2.7e9" blacklist = "${baseDir}/assets/blacklists/GRCh37-blacklist.bed" @@ -42,6 +43,7 @@ params { bismark = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Annotation/README.txt" mito_name = "MT" macs_gsize = "1.87e9" blacklist = "${baseDir}/assets/blacklists/GRCm38-blacklist.bed" @@ -54,6 +56,7 @@ params { bismark = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Annotation/README.txt" mito_name = "Mt" } 'EB2' { @@ -64,6 +67,7 @@ params { bismark = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Annotation/README.txt" } 'UMD3.1' { fasta = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/WholeGenomeFasta/genome.fa" @@ -73,6 +77,7 @@ params { bismark = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Annotation/README.txt" mito_name = "MT" } 'WBcel235' { @@ -94,6 +99,7 @@ params { bismark = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Annotation/README.txt" mito_name = "MT" } 'GRCz10' { @@ -125,6 +131,7 @@ params { bismark = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Annotation/README.txt" mito_name = "MT" } 'EB1' { @@ -135,6 +142,7 @@ params { bismark = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Annotation/README.txt" } 'Galgal4' { fasta = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/WholeGenomeFasta/genome.fa" @@ -154,6 +162,7 @@ params { bismark = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Annotation/README.txt" } 'Mmul_1' { fasta = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/WholeGenomeFasta/genome.fa" @@ -163,6 +172,7 @@ params { bismark = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Annotation/README.txt" mito_name = "MT" } 'IRGSP-1.0' { @@ -183,6 +193,7 @@ params { bismark = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Annotation/README.txt" mito_name = "MT" } 'Rnor_6.0' { @@ -214,6 +225,7 @@ params { bismark = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Annotation/README.txt" mito_name = "MT" macs_gsize = "1.21e7" } @@ -225,6 +237,7 @@ params { bismark = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Annotation/README.txt" } 'Sscrofa10.2' { fasta = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/WholeGenomeFasta/genome.fa" @@ -234,6 +247,7 @@ params { bismark = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Annotation/README.txt" mito_name = "MT" } 'AGPv3' { @@ -266,6 +280,7 @@ params { bismark = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Annotation/README.txt" mito_name = "chrM" macs_gsize = "2.7e9" blacklist = "${baseDir}/assets/blacklists/hg19-blacklist.bed" @@ -278,6 +293,7 @@ params { bismark = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Annotation/README.txt" mito_name = "chrM" macs_gsize = "1.87e9" blacklist = "${baseDir}/assets/blacklists/mm10-blacklist.bed" @@ -300,6 +316,7 @@ params { bismark = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Annotation/README.txt" mito_name = "chrM" macs_gsize = "9e7" } @@ -311,6 +328,7 @@ params { bismark = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Annotation/README.txt" mito_name = "chrM" } 'danRer10' { @@ -342,6 +360,7 @@ params { bismark = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Annotation/README.txt" mito_name = "chrM" } 'galGal4' { @@ -352,6 +371,7 @@ params { bismark = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Annotation/README.txt" mito_name = "chrM" } 'panTro4' { @@ -362,6 +382,7 @@ params { bismark = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Annotation/README.txt" mito_name = "chrM" } 'rn6' { @@ -380,6 +401,7 @@ params { bowtie2 = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/BismarkIndex/" + readme = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Annotation/README.txt" mito_name = "chrM" macs_gsize = "1.2e7" } @@ -391,6 +413,7 @@ params { bismark = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/BismarkIndex/" gtf = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Annotation/Genes/genes.gtf" bed12 = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Annotation/Genes/genes.bed" + readme = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Annotation/README.txt" mito_name = "chrM" } } From 952f8fd81f66e952cfa2c4b873944a547ce354ab Mon Sep 17 00:00:00 2001 From: drpatelh Date: Sun, 26 Jan 2020 20:20:43 +0000 Subject: [PATCH 214/224] Update CHANGELOG --- CHANGELOG.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8152636e4..0c67e23d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## v1.8 +### Continuous integration + +* GitHub Actions CI workflows are now included in the template pipeline +* Travis CI tests will most likely be deprecated from the next `tools` release +* Linting will generate a warning if GitHub Actions workflows do not exist and if applicable to remove Travis CI workflow file i.e. `.travis.yml`. + ### Tools helper code * Refactored the template synchronisation code to be part of the main nf-core tool @@ -15,9 +21,9 @@ ### Linting * Adjusted linting to enable `patch` branches from being tested -* Warn if GitHub actions workflows do not exist, warn if `.travis.yml` and circleCI are there +* Warn if GitHub Actions workflows do not exist, warn if `.travis.yml` and circleCI are there * Lint for `Singularity` file and raise error if found [#458](https://github.com/nf-core/tools/issues/458) -* Added linting of GitHub actions workflows `linting.yml`, `ci.yml` and `branch.yml` +* Added linting of GitHub Actions workflows `linting.yml`, `ci.yml` and `branch.yml` * Warn if pipeline name contains upper case letters or non alphabetical characters [#85](https://github.com/nf-core/tools/issues/85) ### Template pipeline @@ -43,10 +49,11 @@ * Add Python 3.7 dependency to the `environment.yml` file * Remove `awsbatch` profile cf [nf-core/configs#71](https://github.com/nf-core/configs/pull/71) * Make `scrape_software_versions.py` compatible with Python3 to enable miniconda3 in [base image PR](https://github.com/nf-core/tools/pull/462) -* Add GitHub actions workflows and respective linting -* Add `NXF_ANSI_LOG` as global environment variable to template GitHub actions CI workflow -* Fixed global environment variable in GitHub actions CI workflow +* Add GitHub Actions workflows and respective linting +* Add `NXF_ANSI_LOG` as global environment variable to template GitHub Actions CI workflow +* Fixed global environment variable in GitHub Actions CI workflow * Add `--awscli` parameter +* Add `README.txt` path for genomes in `igenomes.config` [#461](https://github.com/nf-core/tools/issues/461) ### Base Docker image From 1efe992636b5c5e70b1417efad9e4462b38f3462 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 27 Jan 2020 10:44:04 +0100 Subject: [PATCH 215/224] Template: Fix ANSI codes in summary log messages --- CHANGELOG.md | 1 + .../{{cookiecutter.name_noslash}}/main.nf | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41da8d2b7..e18af84a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ * Add NXF_ANSI_LOG as global environment variable to template GitHub actions CI workflow * Fixed global environment variable in GitHub actions CI workflow * Add `--awscli` parameter +* Fix buggy ANSI codes in pipeline summary log messages ### Base Docker image diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index ab07fa604..26eb14f23 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -364,16 +364,16 @@ workflow.onComplete { c_reset = params.monochrome_logs ? '' : "\033[0m"; if (workflow.stats.ignoredCount > 0 && workflow.success) { - log.info "${c_purple}Warning, pipeline completed, but with errored process(es) ${c_reset}" - log.info "${c_red}Number of ignored errored process(es) : ${workflow.stats.ignoredCount} ${c_reset}" - log.info "${c_green}Number of successfully ran process(es) : ${workflow.stats.succeedCount} ${c_reset}" + log.info "-${c_purple}Warning, pipeline completed, but with errored process(es) ${c_reset}-" + log.info "-${c_red}Number of ignored errored process(es) : ${workflow.stats.ignoredCount} ${c_reset}-" + log.info "-${c_green}Number of successfully ran process(es) : ${workflow.stats.succeedCount} ${c_reset}-" } if (workflow.success) { - log.info "${c_purple}[{{ cookiecutter.name }}]${c_green} Pipeline completed successfully${c_reset}" + log.info "-${c_purple}[{{ cookiecutter.name }}]${c_green} Pipeline completed successfully${c_reset}-" } else { checkHostname() - log.info "${c_purple}[{{ cookiecutter.name }}]${c_red} Pipeline completed with errors${c_reset}" + log.info "-${c_purple}[{{ cookiecutter.name }}]${c_red} Pipeline completed with errors${c_reset}-" } } From 890d94089e06639dd7fa8cc7e1a132e6853bf296 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 27 Jan 2020 11:16:56 +0100 Subject: [PATCH 216/224] CI Tests: Force package version to be dev --- tests/test_lint.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_lint.py b/tests/test_lint.py index 4fd8decae..85d0e627d 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -42,6 +42,10 @@ def pf(wd, path): # The additional tests passed for releases ADD_PASS_RELEASE = 1 +# The minimal working example expects a development release version +if 'dev' not in nf_core.__version__: + nf_core.__version__ = '{}dev'.format(nf_core.__version__) + class TestLint(unittest.TestCase): """Class for lint tests""" @@ -127,7 +131,7 @@ def test_config_variable_error(self): """Tests that config variable existence test falls over nicely with nextflow can't run""" bad_lint_obj = nf_core.lint.PipelineLint('/non/existant/path') bad_lint_obj.check_nextflow_config() - + def test_actions_wf_branch_pass(self): """Tests that linting for GitHub actions workflow for branch protection works for a good example""" lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) @@ -135,7 +139,7 @@ def test_actions_wf_branch_pass(self): lint_obj.check_actions_branch_protection() expectations = {"failed": 0, "warned": 0, "passed": 2} self.assess_lint_status(lint_obj, **expectations) - + def test_actions_wf_branch_fail(self): """Tests that linting for Github actions workflow for branch protection fails for a bad example""" lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) @@ -492,4 +496,4 @@ def test_pipeline_name_critical(self): critical_lint_obj.pipeline_name = 'Tools123' critical_lint_obj.check_pipeline_name() expectations = {"failed": 0, "warned": 2, "passed": 0} - self.assess_lint_status(critical_lint_obj, **expectations) \ No newline at end of file + self.assess_lint_status(critical_lint_obj, **expectations) From 103ca143be893c1ea213f4160dd71e528aedc5fc Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 27 Jan 2020 11:17:53 +0100 Subject: [PATCH 217/224] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41da8d2b7..1d36ce90d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ * Lint for `Singularity` file [and remove it](https://github.com/nf-core/tools/issues/458) * Added linting of GitHub actions workflows `linting.yml`, `ci.yml` and `branch.yml` * Warn if pipeline name contains upper case letters or non alphabetical characters [#85](https://github.com/nf-core/tools/issues/85) +* Make CI tests of lint code pass for releases ### Template pipeline From 115e680c6c501e6b8896486b7b2ddba43ff4d7e4 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 27 Jan 2020 11:27:12 +0100 Subject: [PATCH 218/224] Drop Travis CI tests for Py2k --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 26b28abab..5508abc91 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ sudo: required language: python jdk: openjdk8 python: - - '2.7' - '3.5' - '3.6' - '3.7' From 0ac2d257fedc28a7fa1e1a234bef7e648d70ba5c Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 27 Jan 2020 12:12:53 +0100 Subject: [PATCH 219/224] GitHub Actions: add TODO statement to encourage customisation --- CHANGELOG.md | 1 + .../.github/workflows/ci.yml | 17 +++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8becca343..1454d8a0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ * Fixed global environment variable in GitHub actions CI workflow * Add `--awscli` parameter * Fix buggy ANSI codes in pipeline summary log messages +* Add a `TODO` line in the new GitHub Actions CI test files ### Base Docker image diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml index 6c7b25814..3e5cda940 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml @@ -1,27 +1,28 @@ name: nf-core CI # This workflow is triggered on pushes and PRs to the repository. # It runs the pipeline with the minimal test dataset to check that it completes without any syntax errors -on: [push, pull_request] +on: [push, pull_request] jobs: test: - env: + env: NXF_VER: {% raw %}${{ matrix.nxf_ver }}{% endraw %} NXF_ANSI_LOG: false - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: matrix: # Nextflow versions: check pipeline minimum and current latest nxf_ver: ['19.10.0', ''] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Install Nextflow run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - - name: Pull image - run: | - docker pull {{ cookiecutter.name_docker }}:dev && docker tag {{ cookiecutter.name_docker }}:dev {{ cookiecutter.name_docker }}:dev - - name: Run test + - name: Pull docker image + run: docker pull {{ cookiecutter.name_docker }}:dev && docker tag {{ cookiecutter.name_docker }}:dev {{ cookiecutter.name_docker }}:dev + - name: Run pipeline with test data run: | + # TODO nf-core: You can customise CI pipeline run tests as required + # (eg. adding multiple test runs with different parameters) nextflow run ${GITHUB_WORKSPACE} -profile test,docker From 95f032133fceeb88af2ed73c94dac45bc36a63d7 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 27 Jan 2020 11:14:12 +0000 Subject: [PATCH 220/224] Update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20c4698ad..26e5264d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Continuous integration * GitHub Actions CI workflows are now included in the template pipeline -* Travis CI tests will most likely be deprecated from the next `tools` release +* Travis CI tests will be deprecated from the next `tools` release * Linting will generate a warning if GitHub Actions workflows do not exist and if applicable to remove Travis CI workflow file i.e. `.travis.yml`. ### Tools helper code From d4995b819c69f93b067f08e874b94342ce264c3d Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 27 Jan 2020 11:16:51 +0000 Subject: [PATCH 221/224] Travis CI definitely going out the door --- docs/lint_errors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/lint_errors.md b/docs/lint_errors.md index cd4b075aa..43ff65af0 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -14,7 +14,7 @@ The lint test looks for the following required files: * A docker build script to generate a docker image with the required software * Continuous integration tests with either [GitHub actions](https://github.com/features/actions) or [Travis CI](https://travis-ci.com/) * GitHub actions workflows for CI (`.github/workflows/ci.yml`), branch protection (`.github/workflows/branch.yml`) and linting (`.github/workflows/linting.yml`) - * Alternatively, `.travis.yml` continuous testing is still allowed but may be deprecated in the near future + * Alternatively, `.travis.yml` continuous integration testing is still allowed but will be deprecated in the near future * `LICENSE`, `LICENSE.md`, `LICENCE.md` or `LICENCE.md` * The MIT licence. Copy from [here](https://raw.githubusercontent.com/nf-core/tools/master/LICENSE). * `README.md` From 0815ee05db7d305aeafff6b678a269c5f3258497 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 27 Jan 2020 11:18:59 +0000 Subject: [PATCH 222/224] Update path to transferred issue --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26e5264d2..5a20ac133 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,7 +54,7 @@ * Add `NXF_ANSI_LOG` as global environment variable to template GitHub Actions CI workflow * Fixed global environment variable in GitHub Actions CI workflow * Add `--awscli` parameter -* Add `README.txt` path for genomes in `igenomes.config` [#461](https://github.com/nf-core/tools/issues/461) +* Add `README.txt` path for genomes in `igenomes.config` [nf-core/atacseq#75](https://github.com/nf-core/atacseq/issues/75) * Fix buggy ANSI codes in pipeline summary log messages ### Base Docker image From 986879e61a196bec8f1f9dba24d2d30a6ed63c80 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Mon, 27 Jan 2020 11:28:01 +0000 Subject: [PATCH 223/224] Update ci.yml --- .../{{cookiecutter.name_noslash}}/.github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml index 3e5cda940..ebf5cb043 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/ci.yml @@ -20,7 +20,8 @@ jobs: wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - name: Pull docker image - run: docker pull {{ cookiecutter.name_docker }}:dev && docker tag {{ cookiecutter.name_docker }}:dev {{ cookiecutter.name_docker }}:dev + run: | + docker pull {{ cookiecutter.name_docker }}:dev && docker tag {{ cookiecutter.name_docker }}:dev {{ cookiecutter.name_docker }}:dev - name: Run pipeline with test data run: | # TODO nf-core: You can customise CI pipeline run tests as required From 493ca1e83a6eab82b7512c370d82c415aaf9de8e Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 27 Jan 2020 11:38:33 +0000 Subject: [PATCH 224/224] Update CHANGELOG agaaaaaain --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f81bff8b..e2e334c0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Continuous integration * GitHub Actions CI workflows are now included in the template pipeline + * Please update these files to match the existing tests that you have in `.travis.yml` * Travis CI tests will be deprecated from the next `tools` release * Linting will generate a warning if GitHub Actions workflows do not exist and if applicable to remove Travis CI workflow file i.e. `.travis.yml`.