diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4000ee67b..9703584f2c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,28 +26,63 @@ jobs: - "latest-everything" test: - "default" - profile: ["docker"] + profile: ["docker", "singularity", "conda"] + exclude: + - NXF_VER: "latest-everything" + profile: "conda" steps: - name: Check out pipeline code uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install Python dependencies + run: python -m pip install --upgrade pip pytest-workflow + + - uses: actions/cache@v2 + with: + path: /usr/local/bin/nextflow + key: ${{ runner.os }} + restore-keys: | + ${{ runner.os }}-nextflow- + - name: Install Nextflow uses: nf-core/setup-nextflow@v1 with: version: "${{ matrix.NXF_VER }}" - - name: Set up Python - uses: actions/setup-python@v2 + - name: Set up Singularity + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-singularity@v5 with: - python-version: "3.x" + singularity-version: 3.7.1 - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow + - name: Set up miniconda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + channels: conda-forge,bioconda,defaults + python-version: ${{ matrix.python-version }} + + - name: Conda clean + if: matrix.profile == 'conda' + run: conda clean -a - name: Run pipeline with tests settings uses: Wandalen/wretry.action@v1.0.11 with: - command: PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.test }} --symlink --kwdof --git-aware --color=yes + command: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.test }} --symlink --kwdof --git-aware --color=yes attempt_limit: 3 - name: Output log on failure @@ -91,6 +126,7 @@ jobs: - "freebayes" - "gatk4_spark" - "haplotypecaller" + - "intervals" - "manta" - "markduplicates" - "mpileup" diff --git a/CHANGELOG.md b/CHANGELOG.md index 073331b0d9..ff8dbf746c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#731](https://github.com/nf-core/sarek/pull/731) - Run the somatic test as default on `-profile test_full`, the germline can be tested with `-profile test_full_somatic` - [#733](https://github.com/nf-core/sarek/pull/733) - Add description for params.cf_chrom_len - [#734](https://github.com/nf-core/sarek/pull/734) - nf-core modules update -a +- [#736](https://github.com/nf-core/sarek/pull/736) - More extensive CI for default test ### Fixed diff --git a/conf/test.config b/conf/test.config index 0089be490b..3ec92dfe1e 100644 --- a/conf/test.config +++ b/conf/test.config @@ -253,29 +253,32 @@ process { } } -// ENABLE CI containers for testing +// Enable container engines/virtualisation envs for CI testing +// only works when specified with the profile ENV +// otherwise tests can be done with the regular provided profiles if (System.getenv('PROFILE')) { if ("$PROFILE" == "conda") { - params.enable_conda = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false + conda.createTimeout = "120 min" + params.enable_conda = true + charliecloud.enabled = false + docker.enabled = false + podman.enabled = false + shifter.enabled = false + singularity.enabled = false } else if ("$PROFILE" == "docker") { - docker.enabled = true - docker.userEmulation = { params.use_gatk_spark ? false : true }.call() - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - fixOwnership = true + docker.enabled = true + docker.userEmulation = { params.use_gatk_spark ? false : true }.call() + charliecloud.enabled = false + podman.enabled = false + shifter.enabled = false + singularity.enabled = false } else if ("$PROFILE" == "singularity") { - singularity.enabled = true - singularity.autoMounts = true - docker.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false + params.singularity_pull_docker_container = false + singularity.autoMounts = true + singularity.enabled = true + charliecloud.enabled = false + docker.enabled = false + podman.enabled = false + shifter.enabled = false } } diff --git a/nextflow.config b/nextflow.config index 5ec00cd60e..a3fe349c1b 100644 --- a/nextflow.config +++ b/nextflow.config @@ -143,63 +143,62 @@ profiles { debug { process.beforeScript = 'echo $HOSTNAME' } conda { params.enable_conda = true + charliecloud.enabled = false docker.enabled = false - singularity.enabled = false podman.enabled = false shifter.enabled = false - charliecloud.enabled = false + singularity.enabled = false } mamba { - params.enable_conda = true conda.useMamba = true + params.enable_conda = true + charliecloud.enabled = false docker.enabled = false - singularity.enabled = false podman.enabled = false shifter.enabled = false - charliecloud.enabled = false + singularity.enabled = false } docker { docker.enabled = true docker.userEmulation = { params.use_gatk_spark ? false : true }.call() - singularity.enabled = false + charliecloud.enabled = false podman.enabled = false shifter.enabled = false - charliecloud.enabled = false - fixOwnership = true + singularity.enabled = false } singularity { - singularity.enabled = true singularity.autoMounts = true + singularity.enabled = true + charliecloud.enabled = false docker.enabled = false podman.enabled = false shifter.enabled = false - charliecloud.enabled = false } podman { podman.enabled = true + charliecloud.enabled = false docker.enabled = false - singularity.enabled = false shifter.enabled = false - charliecloud.enabled = false + singularity.enabled = false } shifter { shifter.enabled = true + charliecloud.enabled = false docker.enabled = false - singularity.enabled = false podman.enabled = false - charliecloud.enabled = false + singularity.enabled = false } charliecloud { charliecloud.enabled = true docker.enabled = false - singularity.enabled = false podman.enabled = false shifter.enabled = false + singularity.enabled = false } gitpod { - executor.name = 'local' executor.cpus = 16 executor.memory = 60.GB + executor.name = 'local' } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } diff --git a/tests/test_default.yml b/tests/test_default.yml index 80794a291b..6d1a6e73d6 100644 --- a/tests/test_default.yml +++ b/tests/test_default.yml @@ -24,7 +24,7 @@ - path: results/preprocessing/recalibrated/test/test.recal.cram.crai # binary changes md5sums on reruns. - path: results/reports/bcftools/strelka/test/test.strelka.variants.bcftools_stats.txt - md5sum: 28d99fd4fa63e24e6805f1d12c3b71d3 + # conda changes md5sums for test. - path: results/reports/fastqc/test-test_L1 - path: results/reports/markduplicates/test/test.md.metrics contains: ["test 8547 767 84 523391 3882 0 0 0.385081", "1.0 767 767"] @@ -49,9 +49,9 @@ - path: results/reports/mosdepth/test/test.recal.regions.bed.gz.csi md5sum: b1c2a861f64e20a94108a6de3b76c582 - path: results/reports/samtools/test/test.md.cram.stats - md5sum: 52411f93d9b5382342b40a351c2e313c + # conda changes md5sums for test. - path: results/reports/samtools/test/test.recal.cram.stats - md5sum: 811d834ff8849e69e7c5925dc2a34257 + # conda changes md5sums for test. - path: results/reports/vcftools/strelka/test/test.strelka.variants.FILTER.summary md5sum: dd87f507da7de20d5318841af312493b - path: results/reports/vcftools/strelka/test/test.strelka.variants.TsTv.count @@ -67,60 +67,3 @@ - path: results/variant_calling/strelka/test/test.strelka.variants.vcf.gz.tbi - path: results/strelka should_exist: false -- name: Run default pipeline without intervals - command: nextflow run main.nf -profile test,no_intervals - tags: - - default - - no_intervals - - preprocessing - files: - - path: results/csv/markduplicates.csv - md5sum: 0d6120bb99e92f6810343270711ca53e - - path: results/csv/markduplicates_no_table.csv - md5sum: 2a2d3d4842befd4def39156463859ee3 - - path: results/csv/recalibrated.csv - md5sum: 42628ec994c16f565e5407b40a9c1ac3 - - path: results/multiqc - - path: results/no_intervals.bed - md5sum: f3dac01ea66b95fe477446fde2d31489 - - path: results/no_intervals.bed.gz - md5sum: f3dac01ea66b95fe477446fde2d31489 - - path: results/no_intervals.bed.gz.tbi - md5sum: f3dac01ea66b95fe477446fde2d31489 - - path: results/preprocessing/markduplicates/test/test.md.cram - # binary changes md5sums on reruns. - - path: results/preprocessing/markduplicates/test/test.md.cram.crai - # binary changes md5sums on reruns. - - path: results/preprocessing/recal_table/test/test.recal.table - md5sum: 4ac774bf5f1157e77426fd82f5ac0fbe - - path: results/preprocessing/recalibrated/test/test.recal.cram - # binary changes md5sums on reruns. - - path: results/preprocessing/recalibrated/test/test.recal.cram.crai - # binary changes md5sums on reruns. - - path: results/reports/fastqc/test-test_L1 - - path: results/reports/markduplicates/test/test.md.metrics - contains: ["test 8547 767 84 523391 3882 0 0 0.385081", "1.0 767 767"] - - path: results/reports/mosdepth/test/test.md.mosdepth.global.dist.txt - md5sum: 76fa71922a3f748e507c2364c531dfcb - - path: results/reports/mosdepth/test/test.md.mosdepth.region.dist.txt - md5sum: abc5df85e302b79985627888870882da - - path: results/reports/mosdepth/test/test.md.mosdepth.summary.txt - md5sum: d536456436eb275159b8c6af83213d80 - - path: results/reports/mosdepth/test/test.md.regions.bed.gz - md5sum: 38fe39894abe62e38f8ac214cba64f2b - - path: results/reports/mosdepth/test/test.md.regions.bed.gz.csi - md5sum: b1c2a861f64e20a94108a6de3b76c582 - - path: results/reports/mosdepth/test/test.recal.mosdepth.global.dist.txt - md5sum: 76fa71922a3f748e507c2364c531dfcb - - path: results/reports/mosdepth/test/test.recal.mosdepth.region.dist.txt - md5sum: abc5df85e302b79985627888870882da - - path: results/reports/mosdepth/test/test.recal.mosdepth.summary.txt - md5sum: d536456436eb275159b8c6af83213d80 - - path: results/reports/mosdepth/test/test.recal.regions.bed.gz - md5sum: 38fe39894abe62e38f8ac214cba64f2b - - path: results/reports/mosdepth/test/test.recal.regions.bed.gz.csi - md5sum: b1c2a861f64e20a94108a6de3b76c582 - - path: results/reports/samtools/test/test.md.cram.stats - md5sum: 52411f93d9b5382342b40a351c2e313c - - path: results/reports/samtools/test/test.recal.cram.stats - md5sum: d80ad11b8e308a3d5b1fb6f8e13cef38 diff --git a/tests/test_intervals.yml b/tests/test_intervals.yml new file mode 100644 index 0000000000..8f24169243 --- /dev/null +++ b/tests/test_intervals.yml @@ -0,0 +1,105 @@ +- name: Run intervals false pipeline + command: nextflow run main.nf -profile test --intervals false + tags: + - intervals + - intervals_false + - preprocessing + files: + - path: results/csv/markduplicates.csv + md5sum: 0d6120bb99e92f6810343270711ca53e + - path: results/csv/markduplicates_no_table.csv + md5sum: 2a2d3d4842befd4def39156463859ee3 + - path: results/csv/recalibrated.csv + md5sum: 42628ec994c16f565e5407b40a9c1ac3 + - path: results/multiqc + - path: results/preprocessing/markduplicates/test/test.md.cram + # binary changes md5sums on reruns. + - path: results/preprocessing/markduplicates/test/test.md.cram.crai + # binary changes md5sums on reruns. + - path: results/preprocessing/recal_table/test/test.recal.table + md5sum: 4ac774bf5f1157e77426fd82f5ac0fbe + - path: results/preprocessing/recalibrated/test/test.recal.cram + # binary changes md5sums on reruns. + - path: results/preprocessing/recalibrated/test/test.recal.cram.crai + # binary changes md5sums on reruns. + - path: results/reports/fastqc/test-test_L1 + - path: results/reports/markduplicates/test/test.md.metrics + contains: ["test 8547 767 84 523391 3882 0 0 0.385081", "1.0 767 767"] + - path: results/reports/mosdepth/test/test.md.mosdepth.global.dist.txt + md5sum: 76fa71922a3f748e507c2364c531dfcb + - path: results/reports/mosdepth/test/test.md.mosdepth.region.dist.txt + md5sum: abc5df85e302b79985627888870882da + - path: results/reports/mosdepth/test/test.md.mosdepth.summary.txt + md5sum: d536456436eb275159b8c6af83213d80 + - path: results/reports/mosdepth/test/test.md.regions.bed.gz + md5sum: 38fe39894abe62e38f8ac214cba64f2b + - path: results/reports/mosdepth/test/test.recal.mosdepth.global.dist.txt + md5sum: 76fa71922a3f748e507c2364c531dfcb + - path: results/reports/mosdepth/test/test.recal.mosdepth.region.dist.txt + md5sum: abc5df85e302b79985627888870882da + - path: results/reports/mosdepth/test/test.recal.mosdepth.summary.txt + md5sum: d536456436eb275159b8c6af83213d80 + - path: results/reports/mosdepth/test/test.recal.regions.bed.gz + md5sum: 38fe39894abe62e38f8ac214cba64f2b + - path: results/reports/samtools/test/test.md.cram.stats + md5sum: 52411f93d9b5382342b40a351c2e313c + - path: results/reports/samtools/test/test.recal.cram.stats + md5sum: 811d834ff8849e69e7c5925dc2a34257 + +- name: Run default pipeline without intervals + command: nextflow run main.nf -profile test,no_intervals + tags: + - intervals + - no_intervals + - preprocessing + files: + - path: results/csv/markduplicates.csv + md5sum: 0d6120bb99e92f6810343270711ca53e + - path: results/csv/markduplicates_no_table.csv + md5sum: 2a2d3d4842befd4def39156463859ee3 + - path: results/csv/recalibrated.csv + md5sum: 42628ec994c16f565e5407b40a9c1ac3 + - path: results/multiqc + - path: results/no_intervals.bed + md5sum: f3dac01ea66b95fe477446fde2d31489 + - path: results/no_intervals.bed.gz + md5sum: f3dac01ea66b95fe477446fde2d31489 + - path: results/no_intervals.bed.gz.tbi + md5sum: f3dac01ea66b95fe477446fde2d31489 + - path: results/preprocessing/markduplicates/test/test.md.cram + # binary changes md5sums on reruns. + - path: results/preprocessing/markduplicates/test/test.md.cram.crai + # binary changes md5sums on reruns. + - path: results/preprocessing/recal_table/test/test.recal.table + md5sum: 4ac774bf5f1157e77426fd82f5ac0fbe + - path: results/preprocessing/recalibrated/test/test.recal.cram + # binary changes md5sums on reruns. + - path: results/preprocessing/recalibrated/test/test.recal.cram.crai + # binary changes md5sums on reruns. + - path: results/reports/fastqc/test-test_L1 + - path: results/reports/markduplicates/test/test.md.metrics + contains: ["test 8547 767 84 523391 3882 0 0 0.385081", "1.0 767 767"] + - path: results/reports/mosdepth/test/test.md.mosdepth.global.dist.txt + md5sum: 76fa71922a3f748e507c2364c531dfcb + - path: results/reports/mosdepth/test/test.md.mosdepth.region.dist.txt + md5sum: abc5df85e302b79985627888870882da + - path: results/reports/mosdepth/test/test.md.mosdepth.summary.txt + md5sum: d536456436eb275159b8c6af83213d80 + - path: results/reports/mosdepth/test/test.md.regions.bed.gz + md5sum: 38fe39894abe62e38f8ac214cba64f2b + - path: results/reports/mosdepth/test/test.md.regions.bed.gz.csi + md5sum: b1c2a861f64e20a94108a6de3b76c582 + - path: results/reports/mosdepth/test/test.recal.mosdepth.global.dist.txt + md5sum: 76fa71922a3f748e507c2364c531dfcb + - path: results/reports/mosdepth/test/test.recal.mosdepth.region.dist.txt + md5sum: abc5df85e302b79985627888870882da + - path: results/reports/mosdepth/test/test.recal.mosdepth.summary.txt + md5sum: d536456436eb275159b8c6af83213d80 + - path: results/reports/mosdepth/test/test.recal.regions.bed.gz + md5sum: 38fe39894abe62e38f8ac214cba64f2b + - path: results/reports/mosdepth/test/test.recal.regions.bed.gz.csi + md5sum: b1c2a861f64e20a94108a6de3b76c582 + - path: results/reports/samtools/test/test.md.cram.stats + md5sum: 52411f93d9b5382342b40a351c2e313c + - path: results/reports/samtools/test/test.recal.cram.stats + md5sum: d80ad11b8e308a3d5b1fb6f8e13cef38 diff --git a/tests/test_targeted.yml b/tests/test_targeted.yml index ce1e3b36eb..a1638dec7b 100644 --- a/tests/test_targeted.yml +++ b/tests/test_targeted.yml @@ -56,50 +56,3 @@ md5sum: 52411f93d9b5382342b40a351c2e313c - path: results/reports/samtools/test/test.recal.cram.stats md5sum: 5ed26e702d5c2238a1f3cdc5e5b44f98 - -- name: Run intervals false pipeline - command: nextflow run main.nf -profile test --intervals false - tags: - - default - - preprocessing - files: - - path: results/csv/markduplicates.csv - md5sum: 0d6120bb99e92f6810343270711ca53e - - path: results/csv/markduplicates_no_table.csv - md5sum: 2a2d3d4842befd4def39156463859ee3 - - path: results/csv/recalibrated.csv - md5sum: 42628ec994c16f565e5407b40a9c1ac3 - - path: results/multiqc - - path: results/preprocessing/markduplicates/test/test.md.cram - # binary changes md5sums on reruns. - - path: results/preprocessing/markduplicates/test/test.md.cram.crai - # binary changes md5sums on reruns. - - path: results/preprocessing/recal_table/test/test.recal.table - md5sum: 4ac774bf5f1157e77426fd82f5ac0fbe - - path: results/preprocessing/recalibrated/test/test.recal.cram - # binary changes md5sums on reruns. - - path: results/preprocessing/recalibrated/test/test.recal.cram.crai - # binary changes md5sums on reruns. - - path: results/reports/fastqc/test-test_L1 - - path: results/reports/markduplicates/test/test.md.metrics - contains: ["test 8547 767 84 523391 3882 0 0 0.385081", "1.0 767 767"] - - path: results/reports/mosdepth/test/test.md.mosdepth.global.dist.txt - md5sum: 76fa71922a3f748e507c2364c531dfcb - - path: results/reports/mosdepth/test/test.md.mosdepth.region.dist.txt - md5sum: abc5df85e302b79985627888870882da - - path: results/reports/mosdepth/test/test.md.mosdepth.summary.txt - md5sum: d536456436eb275159b8c6af83213d80 - - path: results/reports/mosdepth/test/test.md.regions.bed.gz - md5sum: 38fe39894abe62e38f8ac214cba64f2b - - path: results/reports/mosdepth/test/test.recal.mosdepth.global.dist.txt - md5sum: 76fa71922a3f748e507c2364c531dfcb - - path: results/reports/mosdepth/test/test.recal.mosdepth.region.dist.txt - md5sum: abc5df85e302b79985627888870882da - - path: results/reports/mosdepth/test/test.recal.mosdepth.summary.txt - md5sum: d536456436eb275159b8c6af83213d80 - - path: results/reports/mosdepth/test/test.recal.regions.bed.gz - md5sum: 38fe39894abe62e38f8ac214cba64f2b - - path: results/reports/samtools/test/test.md.cram.stats - md5sum: 52411f93d9b5382342b40a351c2e313c - - path: results/reports/samtools/test/test.recal.cram.stats - md5sum: 811d834ff8849e69e7c5925dc2a34257 diff --git a/tests/test_tools.yml b/tests/test_tools.yml index 9d3c289be1..7d8d4619f8 100644 --- a/tests/test_tools.yml +++ b/tests/test_tools.yml @@ -298,7 +298,7 @@ - path: results/variant_calling/deepvariant/sample1/sample1.deepvariant.g.vcf.gz # binary changes md5sums on reruns. - path: results/variant_calling/deepvariant/sample1/sample1.deepvariant.g.vcf.gz.tbi - md5sum: 9870fcf1afbf7cbccc3b63d75acf1f4f + # binary changes md5sums on reruns. - path: results/variant_calling/deepvariant/sample1/sample1.deepvariant.vcf.gz # binary changes md5sums on reruns. - path: results/variant_calling/deepvariant/sample1/sample1.deepvariant.vcf.gz.tbi