Add DAS_Tool binning refinement

.github/workflows/ci.yml

@@ -55,7 +55,15 @@ jobs:
       matrix:
         # Run remaining test profiles with minimum nextflow version
         profile:
-          [test_host_rm, test_hybrid, test_hybrid_host_rm, test_busco_auto, test_ancient_dna, test_adapterremoval]
+          [
+            test_host_rm,
+            test_hybrid,
+            test_hybrid_host_rm,
+            test_busco_auto,
+            test_ancient_dna,
+            test_adapterremoval,
+            test_binrefinement,
+          ]
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v2

CHANGELOG.md

@@ -10,7 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#263](https://github.com/nf-core/mag/pull/263) - Restructure binning subworkflow in preparation for aDNA workflow and extended binning
 - [#247](https://github.com/nf-core/mag/pull/247) - Add ancient DNA subworkflow
 - [#263](https://github.com/nf-core/mag/pull/263) - Add MaxBin2 as second contig binning tool
-- [#284](https://github.com/nf-core/mag/pull/285) - Add AdapterRemoval2 as an alternative read trimmer
+- [#285](https://github.com/nf-core/mag/pull/285) - Add AdapterRemoval2 as an alternative read trimmer
+- [#291](https://github.com/nf-core/mag/pull/291) - Add DAS Tool for bin refinement
 
 ### `Changed`
 

CITATIONS.md

@@ -10,6 +10,10 @@
 
 ## Pipeline tools
 
+- [AdapterRemoval2](https://doi.org/10.1186/)
+
+  > Schubert, M., Lindgreen, S., and Orlando, L. 2016. "AdapterRemoval v2: Rapid Adapter Trimming, Identification, and Read Merging." BMC Research Notes 9 (February): 88. doi: 10.1186/s13104-016-1900-2
+
 - [BCFtools](https://doi.org/10.1093/gigascience/giab008)
 
   > Danecek, Petr, et al. "Twelve years of SAMtools and BCFtools." Gigascience 10.2 (2021): giab008. doi: 10.1093/gigascience/giab008
@@ -30,6 +34,10 @@
 
   > Kim, D., Song, L., Breitwieser, F. P., & Salzberg, S. L. (2016). Centrifuge: rapid and sensitive classification of metagenomic sequences. Genome research, 26(12), 1721-1729. doi: 10.1101/gr.210641.116.
 
+- [DAS Tool](https://doi.org/10.1038/s41564-018-0171-1)
+
+  > Sieber, C. M. K., et al. 2018. "Recovery of Genomes from Metagenomes via a Dereplication, Aggregation and Scoring Strategy." Nature Microbiology 3 (7): 836-43. doi: 10.1038/s41564-018-0171-1
+
 - [FastP](https://doi.org/10.1093/bioinformatics/bty560)
 
   > Chen, S., Zhou, Y., Chen, Y., & Gu, J. (2018). fastp: an ultra-fast all-in-one FASTQ preprocessor. Bioinformatics , 34(17), i884–i890. doi: 10.1093/bioinformatics/bty560.

README.md

@@ -38,6 +38,7 @@ The pipeline then:
 - (optionally) performs ancient DNA assembly validation using [PyDamage](https://github.com/maxibor/pydamage) and contig consensus sequence recalling with [Freebayes](https://github.com/freebayes/freebayes) and [BCFtools](http://samtools.github.io/bcftools/bcftools.html)
 - predicts protein-coding genes for the assemblies using [Prodigal](https://github.com/hyattpd/Prodigal)
 - performs metagenome binning using [MetaBAT2](https://bitbucket.org/berkeleylab/metabat/src/master/) and/or with [MaxBin2](https://sourceforge.net/projects/maxbin2/), and checks the quality of the genome bins using [Busco](https://busco.ezlab.org/)
+- optionally refines bins with [DAS Tool](https://github.com/cmks/DAS_Tool)
 - assigns taxonomy to bins using [GTDB-Tk](https://github.com/Ecogenomics/GTDBTk) and/or [CAT](https://github.com/dutilh/CAT)
 
 Furthermore, the pipeline creates various reports in the results directory specified, including a [MultiQC](https://multiqc.info/) report summarizing some of the findings and software versions.

bin/get_mag_depths.py

@@ -17,7 +17,9 @@ def parse_args(args=None):
     parser.add_argument('-d', '--depths'       , required=True           , metavar='FILE'                             , help="(Compressed) TSV file containing contig depths for each sample: contigName, contigLen, totalAvgDepth, sample1_avgDepth, sample1_var [, sample2_avgDepth, sample2_var, ...].")
     parser.add_argument('-a', '--assembler'    , required=True                           , type=str                   , help="Assembler name.")
     parser.add_argument('-i', '--id'           , required=True                           , type=str                   , help="Sample or group id.")
+    parser.add_argument('-m', '--binner'       , required=True                           , type=str                   , help="Binning method.")
     return parser.parse_args(args)
+# Processing contig depths for each binner again, i.e. not the most efficient way, but ok
 
 def main(args=None):
     args = parse_args(args)
@@ -43,10 +45,8 @@ def main(args=None):
 
     # Initialize output files
     n_samples = len(sample_names)
-    binners = set([ os.path.basename(file).split("-")[1] for file in args.bins ])
-    for binner in binners:
-        with open(args.assembler + "-" + binner + "-" + args.id + "-binDepths.tsv", 'w') as outfile:
-            print("bin", '\t'.join(sample_names), sep='\t', file=outfile)
+    with open(args.assembler + "-" + args.binner + "-" + args.id + "-binDepths.tsv", 'w') as outfile:
+        print("bin", '\t'.join(sample_names), sep='\t', file=outfile)
 
     # for each bin, access contig depths and compute mean bin depth (for all samples)
     for file in args.bins:
@@ -66,7 +66,7 @@ def main(args=None):
                         all_depths[sample].append(contig_depths[sample])
 
         binname = os.path.basename(file)
-        with open(args.assembler + "-" + binname.split("-")[1] + "-" + args.id + "-binDepths.tsv", 'a') as outfile:
+        with open(args.assembler + "-" + args.binner + "-" + args.id + "-binDepths.tsv", 'a') as outfile:
             print(binname, '\t'.join(str(statistics.median(sample_depths)) for sample_depths in all_depths), sep='\t', file=outfile)
 
 

conf/base.config

@@ -159,4 +159,14 @@ process {
         cpus   = { check_max (8     * task.attempt, 'cpus'   ) }
         memory = { check_max (20.GB * task.attempt, 'memory' ) }
     }
+
+    withName: MAXBIN2 {
+        // often fails when insufficient information, so we allow it to gracefully fail without failing the pipeline
+        errorStrategy = { task.exitStatus in [ 1, 255 ] ? 'ignore' : 'retry' }
+    }
+
+    withName: DASTOOL_DASTOOL {
+        // if SCGs not found, bins cannot be assigned and DAS_tool will die with exit status 1
+        errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : task.exitStatus == 1 ? 'ignore' : 'finish' }
+    }
 }

conf/modules.config

@@ -201,14 +201,23 @@ process {
         ]
     }
 
-    withName: 'MAG_DEPTHS_PLOT|MAG_DEPTHS_SUMMARY' {
+    withName: 'MAG_DEPTHS_PLOT|MAG_DEPTHS_SUMMARY|MAG_DEPTHS_PLOT_REFINED' {
         publishDir = [
-            path: { "${params.outdir}/GenomeBinning/depths" },
+            path: { "${params.outdir}/GenomeBinning/depths/bins" },
             mode: params.publish_dir_mode,
             pattern: "*.{png,tsv}"
         ]
     }
 
+    withName: 'MAG_DEPTHS_SUMMARY_REFINED' {
+        ext.prefix = "bin_refined_depths_summary"
+        publishDir = [
+            path: { "${params.outdir}/GenomeBinning/depths/bins" },
+            mode: params.publish_dir_mode,
+            pattern: "*.{tsv}"
+        ]
+    }
+
     withName: 'BIN_SUMMARY' {
         publishDir = [
             path: { "${params.outdir}/GenomeBinning" },
@@ -379,7 +388,7 @@ process {
     }
     withName: METABAT2_JGISUMMARIZEBAMCONTIGDEPTHS {
         publishDir = [
-            path: { "${params.outdir}/GenomeBinning/depths" },
+            path: { "${params.outdir}/GenomeBinning/depths/contigs" },
             mode: params.publish_dir_mode,
             pattern: '*-depth.txt.gz'
         ]
@@ -423,7 +432,6 @@ process {
         ext.prefix = { "${meta.assembler}-MaxBin2-${meta.id}" }
         // if no gene found, will crash so allow ignore so rest of pipeline
         // completes but without MaxBin2 results
-        errorStrategy = { task.exitStatus in [ 1, 255 ] ? 'ignore' : 'retry' }
     }
 
     withName: ADJUST_MAXBIN2_EXT {
@@ -456,6 +464,42 @@ process {
         ]
     }
 
+    withName: DASTOOL_FASTATOCONTIG2BIN_METABAT2 {
+        ext.prefix = { "${meta.assembler}-MetaBAT2-${meta.id}" }
+    }
+
+    withName: DASTOOL_FASTATOCONTIG2BIN_MAXBIN2 {
+        ext.prefix = { "${meta.assembler}-MaxBin2-${meta.id}" }
+    }
+
+    withName: DASTOOL_DASTOOL {
+        publishDir = [
+            [
+                path: { "${params.outdir}/GenomeBinning/DASTool" },
+                mode: params.publish_dir_mode,
+                pattern: '*.{tsv,log,eval,seqlength}'
+            ],
+        ]
+        ext.prefix = { "${meta.assembler}-DASTool-${meta.id}" }
+        ext.args = "--write_bins --write_unbinned --write_bin_evals --score_threshold ${params.refine_bins_dastool_threshold}"
+    }
+
+    withName: RENAME_POSTDASTOOL {
+        publishDir = [
+            [
+                path: { "${params.outdir}/GenomeBinning/DASTool/unbinned" },
+                mode: params.publish_dir_mode,
+                pattern: '*-DASToolUnbinned-*.fa'
+            ],
+            [
+                path: { "${params.outdir}/GenomeBinning/DASTool/bins" },
+                mode: params.publish_dir_mode,
+                // pattern needs to be updated in case of new binning methods
+                pattern: '*-{MetaBAT2,MaxBin2}Refined-*.fa'
+            ]
+        ]
+    }
+
     withName: CUSTOM_DUMPSOFTWAREVERSIONS {
         publishDir = [
             path: { "${params.outdir}/pipeline_info" },

conf/test.config

@@ -20,12 +20,12 @@ params {
     max_time   = '6.h'
 
     // Input data
-    input                       = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv'
-    centrifuge_db               = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz"
-    kraken2_db                  = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz"
-    skip_krona                  = true
-    min_length_unbinned_contigs = 1
-    max_unbinned_contigs        = 2
-    busco_reference             = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz"
-    gtdb                        = false
+    input                         = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv'
+    centrifuge_db                 = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz"
+    kraken2_db                    = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz"
+    skip_krona                    = true
+    min_length_unbinned_contigs   = 1
+    max_unbinned_contigs          = 2
+    busco_reference               = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz"
+    gtdb                          = false
 }

conf/test_ancient_dna.config

@@ -33,4 +33,6 @@ params {
     skip_spades                   = false
     skip_spadeshybrid             = true
     bcftools_view_variant_quality = 0
+    refine_bins_dastool           = true
+    refine_bins_dastool_threshold = 0
 }

conf/test_binrefinement.config

@@ -0,0 +1,34 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Nextflow config file for running minimal tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Defines input files and everything required to run a fast and simple pipeline test.
+
+    Use as follows:
+        nextflow run nf-core/mag -profile test_binrefinement,<docker/singularity> --outdir <OUTDIR>
+
+----------------------------------------------------------------------------------------
+*/
+
+params {
+    config_profile_name        = 'Test profile'
+    config_profile_description = 'Minimal test dataset to check pipeline function'
+
+    // Limit resources so that this can run on GitHub Actions
+    max_cpus   = 2
+    max_memory = '6.GB'
+    max_time   = '6.h'
+
+    // Input data
+    input                         = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv'
+    centrifuge_db                 = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz"
+    kraken2_db                    = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz"
+    skip_krona                    = true
+    min_length_unbinned_contigs   = 1
+    max_unbinned_contigs          = 2
+    busco_reference               = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz"
+    gtdb                          = false
+    refine_bins_dastool           = true
+    refine_bins_dastool_threshold = 0
+    postbinning_input             = 'both'
+}