Migrate pipeline pytest alignment and annotation tests to nf-test (#1708

)

<!--
# nf-core/sarek pull request

Many thanks for contributing to nf-core/sarek!

Please fill in the appropriate checklist below (delete whatever is not
relevant).
These are the most common things requested on pull requests (PRs).

Remember that PRs should be made against the dev branch, unless you're
preparing a pipeline release.

Learn more about contributing:
[CONTRIBUTING.md](https://github.com/nf-core/sarek/tree/master/.github/CONTRIBUTING.md)
-->

## PR checklist

- [ ] This comment contains a description of changes (with reason).
- [ ] If you've fixed a bug or added code that should be tested, add
tests!
- [ ] If you've added a new tool - have you followed the pipeline
conventions in the [contribution
docs](https://github.com/nf-core/sarek/tree/master/.github/CONTRIBUTING.md)
- [ ] If necessary, also make a PR on the nf-core/sarek _branch_ on the
[nf-core/test-datasets](https://github.com/nf-core/test-datasets)
repository.
- [ ] Make sure your code lints (`nf-core lint`).
- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker
--outdir <OUTDIR>`).
- [ ] Check for unexpected warnings in debug mode (`nextflow run .
-profile debug,test,docker --outdir <OUTDIR>`).
- [ ] Usage Documentation in `docs/usage.md` is updated.
- [ ] Output Documentation in `docs/output.md` is updated.
- [ ] `CHANGELOG.md` is updated.
- [ ] `README.md` is updated (including new tool citations and
authors/contributors).

CHANGELOG.md

@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [1692](https://github.com/nf-core/sarek/pull/1692) - Update ensemblvep
 - [1695](https://github.com/nf-core/sarek/pull/1695) - Update all modules
 - [1707](https://github.com/nf-core/sarek/pull/1707) - Un-hide parameters and clean up Json schema
+- [1708](https://github.com/nf-core/sarek/pull/1708) - Migrate pipeline pytest alignment and annotation tests to nf-test
 
 ### Fixed
 

conf/test.config

@@ -22,13 +22,13 @@ params {
     config_profile_description = 'Minimal test dataset to check pipeline function'
 
     // Base directory for nf-core/modules test data
-    modules_testdata_base_path = 's3://ngi-igenomes/testdata/nf-core/modules/'
+    modules_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/'
 
     // Input data
     input = "${projectDir}/tests/csv/3.0/fastq_single.csv"
 
     // small genome on igenomes
-    igenomes_base   = 's3://ngi-igenomes/testdata/nf-core/modules'
+    igenomes_base   = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/'
     genome          = 'testdata.nf-core.sarek'
 
     // Small reference genome

subworkflows/local/samplesheet_to_channel/tests/main.nf.test.snap

@@ -16,8 +16,8 @@
                             "size": 1
                         },
                         [
-                            "/ngi-igenomes/testdata/nf-core/modules/genomics/homo_sapiens/illumina/fastq/test_1.fastq.gz",
-                            "/ngi-igenomes/testdata/nf-core/modules/genomics/homo_sapiens/illumina/fastq/test_2.fastq.gz"
+                            "/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/fastq/test_1.fastq.gz",
+                            "/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/fastq/test_2.fastq.gz"
                         ]
                     ]
                 ],
@@ -35,8 +35,8 @@
                             "size": 1
                         },
                         [
-                            "/ngi-igenomes/testdata/nf-core/modules/genomics/homo_sapiens/illumina/fastq/test_1.fastq.gz",
-                            "/ngi-igenomes/testdata/nf-core/modules/genomics/homo_sapiens/illumina/fastq/test_2.fastq.gz"
+                            "/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/fastq/test_1.fastq.gz",
+                            "/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/fastq/test_2.fastq.gz"
                         ]
                     ]
                 ]
@@ -48,4 +48,4 @@
         },
         "timestamp": "2024-10-04T10:25:14.620549"
     }
-}
+}

tests/.nftignore

@@ -18,4 +18,6 @@ reference/dragmap/hash_table_stats.txt
 reports/EnsemblVEP/*/*.ann.summary.html
 reports/fastqc/**/*_fastqc.{html,zip}
 reports/markduplicates/**/*.md.cram.metrics
+reports/snpeff/*/*_snpEff.csv
+reports/snpeff/*/snpEff_summary.html
 variant_calling/**/*.vcf.{gz,gz.tbi}

tests/alignment_from_everything.nf.test

@@ -0,0 +1,46 @@
+nextflow_pipeline {
+
+    name "Test pipeline"
+    script "../main.nf"
+    tag "pipeline"
+    tag "pipeline_sarek"
+    profile "alignment_from_everything"
+
+    test("Run with profile test,alignment_from_everything | --save_mapped --save_output_as_bam") {
+
+        when {
+            params {
+                modules_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/'
+                igenomes_base   = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data'
+                outdir = "$outputDir"
+                save_mapped = true
+                save_output_as_bam = true
+            }
+        }
+
+        then {
+            // stable_name: All files + folders in ${params.outdir}/ with a stable name
+            def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}'])
+            // stable_path: All files in ${params.outdir}/ with stable content
+            def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
+            // cram_files: All cram files
+            def cram_files  = getAllFilesFromDir(params.outdir, include: ['**/*.cram'])
+            def fasta       = params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta'
+            assertAll(
+                { assert workflow.success},
+                { assert snapshot(
+                    // Number of successful tasks
+                    workflow.trace.succeeded().size(),
+                    // pipeline versions.yml file for multiqc from which Nextflow version is removed because we tests pipelines on multiple Nextflow versions
+                    removeNextflowVersion("$outputDir/pipeline_info/nf_core_sarek_software_mqc_versions.yml"),
+                    // All stable path name, with a relative path
+                    stable_name,
+                    // All files with stable contents
+                    stable_path,
+                    // All cram files
+                    cram_files.collect{ file -> [ file.getName(), cram(file.toString(), fasta).getReadsMD5() ] }
+                ).match() }
+            )
+        }
+    }
+}