Merge pull request #17 from CCBR/filter-blacklist-subwf

Create subworkflow to filter reads from blacklisted regions

CHANGELOG.md

@@ -1,6 +1,6 @@
-## development version
+## Development version
 
-### new modules
+### New modules
 
 - bwa/index
 - bwa/mem
@@ -10,3 +10,7 @@
 - khmer/uniquekmers (#7)
 - picard/samtofastq (#21)
 - samtools/filteraligned (#13,#20)
+
+### New subworkflows
+
+- custom/filter_blacklist (#17)

README.md

@@ -24,6 +24,14 @@ nf-core modules \
   update [module]
 ```
 
+Use the `subworkflows` command in place of the `modules` command to install or update subworkflows.
+
+```sh
+nf-core subworkflows \
+  --git-remote https://github.com/CCBR/nf-modules \
+  update [subworkflow]
+```
+
 ## Help & Contributing
 
 Come across a **bug**? Open an [issue](https://github.com/CCBR/nf-modules/issues) and include a minimal reproducible example.

data/genomics/sarscov2/illumina/fastq/README.md

@@ -0,0 +1,19 @@
+
+
+Original files downloaded from https://github.com/nf-core/test-datasets/tree/modules/data/genomics/sarscov2/illumina/fastq
+
+How subsets were created:
+
+```sh
+grep "^@" test_1.fastq | wc -l
+grep -n "^@" test_1.fastq
+head -n 40 test_1.fastq > test_1.subset.fastq
+grep -n "^@" test_2.fastq
+head -n 40 test_2.fastq > test_2.subset.fastq
+```
+
+Check tails of subset files to make sure they don't end with fastq headers:
+
+```sh
+tail -n 1 *subset.fastq
+```

modules/CCBR/samtools/filteraligned/main.nf

@@ -1,5 +1,5 @@
 
-process SAMTOOLS_FILTER_ALIGNED {
+process SAMTOOLS_FILTERALIGNED {
     '''
     Given a bam file, filter out reads that aligned.
     '''

modules/CCBR/samtools/filteraligned/meta.yml

@@ -1,4 +1,4 @@
-name: samtools_filter_aligned
+name: samtools_filteraligned
 description: Filter out aligned reads from a BAM file.
 keywords:
   - bam

subworkflows/CCBR/filter_blacklist/main.nf

@@ -0,0 +1,28 @@
+
+
+include { BWA_MEM                 } from '../../../modules/CCBR/bwa/mem'
+include { SAMTOOLS_FILTERALIGNED } from '../../../modules/CCBR/samtools/filteraligned'
+include { PICARD_SAMTOFASTQ       } from '../../../modules/CCBR/picard/samtofastq'
+
+workflow FILTER_BLACKLIST {
+    take:
+        ch_fastq_input      // channel: [ val(meta), path(fastq) ]
+        ch_blacklist_index  // channel: [ val(meta), path(bwa/*) ]
+
+    main:
+        ch_versions = Channel.empty()
+
+        BWA_MEM ( ch_fastq_input, ch_blacklist_index )
+        SAMTOOLS_FILTERALIGNED( BWA_MEM.out.bam )
+        PICARD_SAMTOFASTQ( BWA_MEM.out.bam )
+
+        ch_versions = ch_versions.mix(
+            BWA_MEM.out.versions,
+            SAMTOOLS_FILTERALIGNED.out.versions,
+            PICARD_SAMTOFASTQ.out.versions
+        )
+
+    emit:
+        reads =  PICARD_SAMTOFASTQ.out.reads  // channel: [ val(meta), path(fastq) ]
+        versions = ch_versions           // channel: [ path(versions.yml) ]
+}

subworkflows/CCBR/filter_blacklist/meta.yml

@@ -0,0 +1,32 @@
+name: filter_blacklist
+description: Filter out reads that align to an index
+keywords:
+  - bwa
+  - samtools
+  - fastq
+  - bam
+  - filter
+  - blacklist
+components:
+  - bwa/mem
+  - samtools/filteraligned
+  - picard/samtofastq
+input:
+  - ch_fastq_input:
+      description: |
+        A channel containing fastq files
+  - ch_blacklist_index:
+      description: |
+        A BWA index created by running BWA/INDEX on a fasta file of blacklisted regions/
+output:
+  - reads:
+      description: |
+        Reads from the fastq files that do not align to the blacklist
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: "versions.yml"
+authors:
+  - "@kelly-sovacool"
+maintainers:
+  - "@kelly-sovacool"

tests/config/pytest_modules.yml

@@ -25,3 +25,7 @@ picard/samtofastq:
 samtools/filteraligned:
   - modules/CCBR/samtools/filteraligned/**
   - tests/modules/CCBR/samtools/filteraligned/**
+
+subworkflows/filter_blacklist:
+  - subworkflows/CCBR/filter_blacklist/**
+  - tests/subworkflows/CCBR/filter_blacklist/**

tests/config/test_data_CCBR.config

@@ -0,0 +1,10 @@
+params {
+    test_data_base = 'https://raw.githubusercontent.com/CCBR/nf-modules/filter-blacklist-subwf/'
+
+    test_data {
+        test_1_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test_1.fastq.gz"
+        test_2_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test_2.fastq.gz"
+        test_1_subset_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test_1.subset.fastq.gz"
+        test_2_subset_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test_2.subset.fastq.gz"
+    }
+}

tests/modules/CCBR/samtools/filteraligned/main.nf

@@ -4,7 +4,7 @@ nextflow.enable.dsl = 2
 
 include { BWA_INDEX               } from '../../../../../modules/CCBR/bwa/index/main.nf'
 include { BWA_MEM                 } from '../../../../../modules/CCBR/bwa/mem/main.nf'
-include { SAMTOOLS_FILTER_ALIGNED } from '../../../../../modules/CCBR/samtools/filteraligned/main.nf'
+include { SAMTOOLS_FILTERALIGNED } from '../../../../../modules/CCBR/samtools/filteraligned/main.nf'
 
 //
 // Test with single-end data
@@ -23,7 +23,7 @@ workflow test_filter_aligned_single_end {
 
     BWA_INDEX ( fasta )
     BWA_MEM ( input, BWA_INDEX.out.index )
-    SAMTOOLS_FILTER_ALIGNED( BWA_MEM.out.bam )
+    SAMTOOLS_FILTERALIGNED( BWA_MEM.out.bam )
 }
 
 //
@@ -44,5 +44,5 @@ workflow test_filter_aligned_paired_end {
 
     BWA_INDEX ( fasta )
     BWA_MEM ( input, BWA_INDEX.out.index )
-    SAMTOOLS_FILTER_ALIGNED( BWA_MEM.out.bam )
+    SAMTOOLS_FILTERALIGNED( BWA_MEM.out.bam )
 }

tests/subworkflows/CCBR/filter_blacklist/main.nf

@@ -0,0 +1,32 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+include { BWA_INDEX        } from "../../../../modules/CCBR/bwa/index/main"
+include { FILTER_BLACKLIST } from "../../../../subworkflows/CCBR/filter_blacklist/main"
+
+
+workflow test_filter_blacklist_single {
+    input = [ [ id:'test', single_end:true ], // meta map
+               file(params.test_data['test_1_fastq_gz'], checkIfExists: true)
+    ]
+    blacklist_reads = [
+            [ id:'test', single_end:true ], // meta map
+            file(params.test_data['test_1_subset_fastq_gz'], checkIfExists: true)
+    ]
+    BWA_INDEX(blacklist_reads)
+    FILTER_BLACKLIST(input, BWA_INDEX.out.index)
+}
+
+workflow test_filter_blacklist_paired {
+    input = [ [ id:'test', single_end:false ], // meta map
+              [ file(params.test_data['test_1_fastq_gz'], checkIfExists: true),
+                file(params.test_data['test_2_fastq_gz'], checkIfExists: true) ]
+    ]
+    blacklist_reads = [
+              [ id:'test', single_end:false ], // meta map
+              file(params.test_data['test_1_subset_fastq_gz'], checkIfExists: true)
+    ]
+    BWA_INDEX(blacklist_reads)
+    FILTER_BLACKLIST(input, BWA_INDEX.out.index)
+}

tests/subworkflows/CCBR/filter_blacklist/nextflow.config

@@ -0,0 +1,6 @@
+process {
+
+    publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
+
+
+}

tests/subworkflows/CCBR/filter_blacklist/test.yml

@@ -0,0 +1,43 @@
+- name: filter_blacklist test_filter_blacklist_single
+  command: nextflow run ./tests/subworkflows/CCBR/filter_blacklist/main.nf -entry test_filter_blacklist_single -c tests/config/nextflow.config -c tests/config/test_data_CCBR.config
+  tags:
+    - subworkflows
+    - subworkflows/filter_blacklist
+    - picard
+    - picard/samtofastq
+    - samtools
+    - samtools/filteraligned
+    - bwa
+    - bwa/mem
+  files:
+    - path: output/picard/test.fastq.gz
+
+- name: filter_blacklist test_filter_blacklist_paired
+  command: nextflow run ./tests/subworkflows/CCBR/filter_blacklist/main.nf -entry test_filter_blacklist_paired -c tests/config/nextflow.config -c tests/config/test_data_CCBR.config
+  tags:
+    - subworkflows
+    - subworkflows/filter_blacklist
+    - picard
+    - picard/samtofastq
+    - samtools
+    - samtools/filteraligned
+    - bwa
+    - bwa/mem
+  files:
+    - path: output/picard/test_1.fastq.gz
+    - path: output/picard/test_2.fastq.gz
+    - path: output/picard/test.unpaired.fastq.gz
+
+- name: filter_blacklist test_filter_blacklist_single stub
+  command: nextflow run ./tests/subworkflows/CCBR/filter_blacklist/main.nf -entry test_filter_blacklist_single -c tests/config/nextflow.config -c tests/config/test_data_CCBR.config -stub
+  tags:
+    - subworkflows
+    - subworkflows/filter_blacklist
+    - picard
+    - picard/samtofastq
+    - samtools
+    - samtools/filteraligned
+    - bwa
+    - bwa/mem
+  files:
+    - path: output/picard/test.fastq.gz

tests/subworkflows/CCBR/filter_blacklist/test_filter_blacklist.py

@@ -0,0 +1,13 @@
+import gzip
+import pathlib
+import pytest
+
+
+@pytest.mark.workflow("filter_blacklist test_filter_blacklist_paired")
+def test_unpaired_is_empty(workflow_dir):
+    unpaired_fastq = pathlib.Path(
+        workflow_dir, "output", "picard", "test.unpaired.fastq.gz"
+    )
+    with gzip.open(unpaired_fastq, "rt") as infile:
+        lines = infile.readlines()
+    assert len(lines) == 0