Add step markduplicates & allow BAM input for all steps

.github/workflows/ci.yml

@@ -40,9 +40,12 @@ jobs:
           - "gatk4_spark"
           - "haplotypecaller"
           - "manta"
+          - "markduplicates"
           - "mutect2"
           - "msisensorpro"
           # - 'save_bam_mapped'
+          - "prepare_recalibration"
+          - "recalibrate"
           - "variantcalling_channel"
           - "skip_markduplicates"
           - "strelka"

CHANGELOG.md

@@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#512](https://github.com/nf-core/sarek/pull/512), [#531](https://github.com/nf-core/sarek/pull/531), [#537](https://github.com/nf-core/sarek/pull/537) - Subway map for pipeline
 - [#522](https://github.com/nf-core/sarek/pull/522) - Add QC for vcf files & MultiQC
 - [#533](https://github.com/nf-core/sarek/pull/533) - Add param `--only_paired_variant_calling` to allow skipping of germline variantcalling for paired samples
+- [#536](https://github.com/nf-core/sarek/pull/536) - Add `--step markduplicates` to start from duplicate marking, `--step prepare_recalibration` now ONLY starts at process `BaseRecalibrator` & adding `bam` and `cram` input support for `--step` `markduplicates`, `prepare_recalibration` and `recalibrate`
 - [#538](https://github.com/nf-core/sarek/pull/538) - Add param `--seq_platform`, default: `ILLUMINA`
 
 ### Changed

conf/modules.config

@@ -91,7 +91,7 @@ process {
     }
 
     withName: 'TABIX_DBSNP' {
-        ext.when         = { !params.dbsnp_tbi && params.dbsnp && (params.step == "mapping" || params.step == "prepare_recalibration") || params.tools && (params.tools.contains('controlfreec') || params.tools.contains('haplotypecaller') || params.tools.contains('mutect2')) }
+        ext.when         = { !params.dbsnp_tbi && params.dbsnp && (params.step == "mapping" || params.step == "markduplicates" || params.step == "prepare_recalibration") || params.tools && (params.tools.contains('controlfreec') || params.tools.contains('haplotypecaller') || params.tools.contains('mutect2')) }
         publishDir       = [
             enabled: params.save_reference,
             mode: params.publish_dir_mode,
@@ -111,7 +111,7 @@ process {
     }
 
     withName: 'TABIX_KNOWN_INDELS' {
-        ext.when         = { !params.known_indels_tbi && params.known_indels && (params.step == 'mapping' || params.step == 'prepare_recalibration') }
+        ext.when         = { !params.known_indels_tbi && params.known_indels && (params.step == 'mapping' || params.step == "markduplicates" || params.step == 'prepare_recalibration') }
         publishDir       = [
             enabled: params.save_reference,
             mode: params.publish_dir_mode,

conf/test.config

@@ -55,10 +55,26 @@ profiles {
     pair {
         params.input               = "${baseDir}/tests/csv/3.0/fastq_pair.csv"
     }
-    prepare_recalibration {
+    markduplicates_bam {
+        params.input               = "${baseDir}/tests/csv/3.0/mapped_single.csv"
+        params.step                = 'markduplicates'
+    }
+    markduplicates_cram {
+        params.input               = "${baseDir}/tests/csv/3.0/mapped_single_cram.csv"
+        params.step                = 'markduplicates'
+    }
+    prepare_recalibration_cram {
+        params.input               = "${baseDir}/tests/csv/3.0/mapped_single_cram.csv"
+        params.step                = 'prepare_recalibration'
+    }
+    prepare_recalibration_bam {
         params.input               = "${baseDir}/tests/csv/3.0/mapped_single.csv"
         params.step                = 'prepare_recalibration'
     }
+    recalibrate {
+        params.input               = "${baseDir}/tests/csv/3.0/prepare_recalibration_single_bam.csv"
+        params.step                = 'recalibrate'
+    }
     save_bam_mapped {
         params.save_bam_mapped     = true
     }

nextflow_schema.json

@@ -20,6 +20,7 @@
                     "help_text": "Only one step",
                     "enum": [
                         "mapping",
+                        "markduplicates",
                         "prepare_recalibration",
                         "recalibrate",
                         "variant_calling",

tests/csv/3.0/mapped_single_cram.csv

@@ -0,0 +1,2 @@
+patient,status,sample,cram,crai
+test,0,test,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/cram/test.paired_end.sorted.cram,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/cram/test.paired_end.sorted.cram.crai

tests/csv/3.0/prepare_recalibration_single_bam.csv

@@ -0,0 +1,2 @@
+patient,status,sample,bam,bai,table
+test1,0,test1,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/bam/test.paired_end.sorted.bam,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/bam/test.paired_end.sorted.bam.bai,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/gatk/test.baserecalibrator.table

tests/csv/3.0/recalibrated.csv

@@ -1,6 +1,5 @@
 patient,gender,status,sample,cram,crai
 test,XX,0,sample1,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/cram/test.paired_end.recalibrated.sorted.cram,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/cram/test.paired_end.recalibrated.sorted.cram.crai
-test2,XX,0,sample2,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/cram/test.paired_end.recalibrated.sorted.cram,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/cram/test.paired_end.recalibrated.sorted.cram.crai
-test2,XX,1,sample3,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.recalibrated.sorted.cram,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.recalibrated.sorted.cram.crai
+test1,XX,1,sample2,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.recalibrated.sorted.cram,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.recalibrated.sorted.cram.crai
+test3,XX,0,sample3,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/cram/test.paired_end.recalibrated.sorted.cram,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/cram/test.paired_end.recalibrated.sorted.cram.crai
 test3,XX,1,sample4,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.recalibrated.sorted.cram,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.recalibrated.sorted.cram.crai
-test3,XX,1,sample5,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.recalibrated.sorted.cram,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.recalibrated.sorted.cram.crai

tests/test_markduplicates.yml

@@ -0,0 +1,23 @@
+- name: Run Mark Duplicates
+  command: nextflow run main.nf -profile test,markduplicates,docker
+  tags:
+    - bam
+    - markduplicates
+    - preprocessing
+  files:
+    - path: results/multiqc
+    - path: results/preprocessing/test/markduplicates/test.md.cram
+    - path: results/preprocessing/test/markduplicates/test.md.cram.crai
+    - path: results/preprocessing/test/recal_table/test.recal.table
+    - path: results/preprocessing/test/recalibrated/test.recal.cram
+    - path: results/preprocessing/test/recalibrated/test.recal.cram.crai
+    - path: results/preprocessing/csv/markduplicates.csv
+    - path: results/preprocessing/csv/markduplicates_test.csv
+    - path: results/preprocessing/csv/markduplicates_no_table.csv
+    - path: results/preprocessing/csv/markduplicates_no_table_test.csv
+    - path: results/preprocessing/csv/recalibrated.csv
+    - path: results/preprocessing/csv/recalibrated_test.csv
+    - path: results/reports/qualimap/test/test.mapped
+    - path: results/reports/qualimap/test/test.recal
+    - path: results/reports/samtools_stats/test/test.md.cram.stats
+    - path: results/reports/samtools_stats/test/test.recal.cram.stats

tests/test_prepare_recalibration.yml

@@ -1,22 +1,31 @@
-- name: Run Prepare_recal
-  command: nextflow run main.nf -profile test,prepare_recalibration,docker
+- name: Run Prepare_recal starting from bam
+  command: nextflow run main.nf -profile test,prepare_recalibration_bam,docker
   tags:
+    - bam
+    - prepare_recalibration
+    - preprocessing
+  files:
+    - path: results/multiqc
+    - path: results/preprocessing/test/recal_table/test.recal.table
+    - path: results/preprocessing/test/recalibrated/test.recal.cram
+    - path: results/preprocessing/test/recalibrated/test.recal.cram.crai
+    - path: results/preprocessing/csv/recalibrated.csv
+    - path: results/preprocessing/csv/recalibrated_test.csv
+    - path: results/reports/qualimap/test/test.recal
+    - path: results/reports/samtools_stats/test/test.recal.cram.stats
+
+- name: Run Prepare_recal starting from cram
+  command: nextflow run main.nf -profile test,prepare_recalibration_cram,docker
+  tags:
+    - cram
     - prepare_recalibration
     - preprocessing
   files:
     - path: results/multiqc
-    - path: results/preprocessing/test/markduplicates/test.md.cram
-    - path: results/preprocessing/test/markduplicates/test.md.cram.crai
     - path: results/preprocessing/test/recal_table/test.recal.table
     - path: results/preprocessing/test/recalibrated/test.recal.cram
     - path: results/preprocessing/test/recalibrated/test.recal.cram.crai
-    - path: results/preprocessing/csv/markduplicates.csv
-    - path: results/preprocessing/csv/markduplicates_test.csv
-    - path: results/preprocessing/csv/markduplicates_no_table.csv
-    - path: results/preprocessing/csv/markduplicates_no_table_test.csv
     - path: results/preprocessing/csv/recalibrated.csv
     - path: results/preprocessing/csv/recalibrated_test.csv
-    - path: results/reports/qualimap/test/test.mapped
     - path: results/reports/qualimap/test/test.recal
-    - path: results/reports/samtools_stats/test/test.md.cram.stats
     - path: results/reports/samtools_stats/test/test.recal.cram.stats

tests/test_recalibrate.yml

@@ -0,0 +1,12 @@
+- name: Run Recalibration starting from bam
+  command: nextflow run main.nf -profile test,recalibrate,docker
+  tags:
+    - bam
+    - recalibrate
+    - preprocessing
+  files:
+    - path: results/multiqc
+    - path: results/preprocessing/test/recalibrated/test.recal.cram
+    - path: results/preprocessing/test/recalibrated/test.recal.cram.crai
+    - path: results/reports/qualimap/test/test.recal
+    - path: results/reports/samtools_stats/test/test.recal.cram.stats