vib-singlecell-nf · dweemx · Jul 9, 2021 · Feb 12, 2021 · Feb 12, 2021 · Feb 12, 2021
diff --git a/conf/generic.config b/conf/generic.config
@@ -1,4 +1,9 @@
+import static groovy.json.JsonOutput.*
+
 params {
+   breakPrettyPrintMap = { p ->
+      throw new Exception(prettyPrint(toJson(p)))
+   }
    // This closure facilitates the usage of sample specific parameters
    parseConfig = { sample, paramsGlobal, paramsLocal ->
          def lv = { a,b -> return org.codehaus.groovy.runtime.MethodRankHelper.delDistance(a, b) }

diff --git a/conf/nemesh.config b/conf/nemesh.config
@@ -7,7 +7,7 @@ params {
         qsubaccount = ''
     }
 
-    sc {
+    tools {
         nemesh {
             // User can extract custom cell barcodes by providing it with a TSV containing all the barcodes
             // custom_selected_barcodes = ''

diff --git a/conf/test__bbknn.config b/conf/test__bbknn.config
@@ -8,10 +8,7 @@ params {
             cellranger_mex = "testdata/*/outs/"
         }
     }
-    sc {
-        file_annotator {
-            metadataFilePath = ''
-        }
+    tools {
         scanpy {
             filter {
                 cellFilterMinNGenes = 1

diff --git a/conf/test__bbknn_scenic.config b/conf/test__bbknn_scenic.config
@@ -8,10 +8,7 @@ params {
             cellranger_mex = "testdata/*/outs/"
         }
     }
-    sc {
-        file_annotator {
-            metadataFilePath = ''
-        }
+    tools {
         scanpy {
             filter {
                 cellFilterMinNGenes = 1

diff --git a/conf/test__cell_annotate_filter.config b/conf/test__cell_annotate_filter.config
@@ -8,16 +8,18 @@ params {
             cellranger_mex = 'sample_data/outs'
         }
     }
-    sc {
+    tools {
+        scanpy {
+            container = 'vibsinglecellnf/scanpy:1.8.1'
+        }
+    }
+    utils {
         file_converter {
             off = 'h5ad'
             tagCellWithSampleId = false
             useFilteredMatrix = true
             makeVarIndexUnique = false
         }
-        scanpy {
-            container = 'vibsinglecellnf/scanpy:1.8.1'
-        }
         cell_annotate {
             off = 'h5ad'
             method = 'aio'

diff --git a/conf/test__decontx.config b/conf/test__decontx.config
@@ -8,10 +8,7 @@ params {
             cellranger_mex = 'sample_data/outs'
         }
     }
-    sc {
-        file_annotator {
-            metadataFilePath = ''
-        }
+    tools {
         celda {
             container = 'vibsinglecellnf/celda:1.4.5'
             decontx {

diff --git a/conf/test__harmony.config b/conf/test__harmony.config
@@ -8,10 +8,7 @@ params {
             cellranger_mex = "testdata/*/outs/"
         }
     }
-    sc {
-        file_annotator {
-            metadataFilePath = ''
-        }
+    tools {
         scanpy {
             filter {
                 cellFilterMinNGenes = 1

diff --git a/conf/test__harmony_scenic.config b/conf/test__harmony_scenic.config
@@ -8,7 +8,12 @@ params {
             cellranger_mex = "testdata/*/outs/"
         }
     }
-    sc {
+    utils {
+        file_annotator {
+            metadataFilePath = ''
+        }
+    }
+    tools {
         file_annotator {
             metadataFilePath = ''
         }

diff --git a/conf/test__mnncorrect.config b/conf/test__mnncorrect.config
@@ -8,10 +8,7 @@ params {
             cellranger_mex = "testdata/*/outs/"
         }
     }
-    sc {
-        file_annotator {
-            metadataFilePath = ''
-        }
+    tools {
         scanpy {
             filter {
                 cellFilterMinNGenes = 1

diff --git a/conf/test__scenic.config b/conf/test__scenic.config
@@ -3,7 +3,7 @@ params {
     global {
         project_name = 'scenic_CI'
     }
-    sc {
+    tools {
         file_annotator {
             metadataFilePath = ''
         }

diff --git a/conf/test__scenic_multiruns.config b/conf/test__scenic_multiruns.config
@@ -3,10 +3,7 @@ params {
     global {
         project_name = 'scenic_multiruns_CI'
     }
-    sc {
-        file_annotator {
-            metadataFilePath = ''
-        }
+    tools {
         scenic {
             filteredLoom = 'https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/expr_mat_small.loom'
             grn {

diff --git a/conf/test__single_sample.config b/conf/test__single_sample.config
@@ -8,10 +8,7 @@ params {
             cellranger_mex = 'sample_data/outs'
         }
     }
-    sc {
-        file_annotator {
-            metadataFilePath = ''
-        }
+    tools {
         scanpy {
             filter {
                 cellFilterMinNGenes = 1

diff --git a/conf/test__single_sample_decontx_correct.config b/conf/test__single_sample_decontx_correct.config
@@ -8,10 +8,7 @@ params {
             cellranger_mex = 'sample_data/outs'
         }
     }
-    sc {
-        file_annotator {
-            metadataFilePath = ''
-        }
+    tools {
         scanpy {
             filter {
                 cellFilterMinNGenes = 1

diff --git a/conf/test__single_sample_decontx_correct_scrublet.config b/conf/test__single_sample_decontx_correct_scrublet.config
@@ -8,10 +8,7 @@ params {
             cellranger_mex = 'sample_data/outs'
         }
     }
-    sc {
-        file_annotator {
-            metadataFilePath = ''
-        }
+    tools {
         scanpy {
             filter {
                 cellFilterMinNGenes = 1

diff --git a/conf/test__single_sample_decontx_filter.config b/conf/test__single_sample_decontx_filter.config
@@ -8,10 +8,7 @@ params {
             cellranger_mex = 'sample_data/outs'
         }
     }
-    sc {
-        file_annotator {
-            metadataFilePath = ''
-        }
+    tools {
         scanpy {
             filter {
                 cellFilterMinNGenes = 1

diff --git a/conf/test__single_sample_param_exploration.config b/conf/test__single_sample_param_exploration.config
@@ -8,10 +8,7 @@ params {
             cellranger_mex = 'sample_data/outs'
         }
     }
-    sc {
-        file_annotator {
-            metadataFilePath = ''
-        }
+    tools {
         scanpy {
             filter {
                 cellFilterMinNGenes = 1

diff --git a/conf/test__single_sample_scenic.config b/conf/test__single_sample_scenic.config
@@ -8,10 +8,7 @@ params {
             cellranger_mex = 'sample_data/outs'
         }
     }
-    sc {
-        file_annotator {
-            metadataFilePath = ''
-        }
+    tools {
         scanpy {
             filter {
                 cellFilterMinNGenes = 1

diff --git a/conf/test__single_sample_scenic_multiruns.config b/conf/test__single_sample_scenic_multiruns.config
@@ -8,10 +8,7 @@ params {
             cellranger_mex = 'sample_data/outs'
         }
     }
-    sc {
-        file_annotator {
-            metadataFilePath = ''
-        }
+    tools {
         scanpy {
             filter {
                 cellFilterMinNGenes = 1

diff --git a/conf/test__single_sample_scrublet.config b/conf/test__single_sample_scrublet.config
@@ -8,10 +8,7 @@ params {
             cellranger_mex = 'sample_data/outs'
         }
     }
-    sc {
-        file_annotator {
-            metadataFilePath = ''
-        }
+    tools {
         scanpy {
             filter {
                 cellFilterMinNGenes = 1

diff --git a/docs/development.rst b/docs/development.rst
@@ -92,7 +92,7 @@ Steps:
     .. code:: groovy
 
         params {
-            sc {
+            tools {
                 harmony {
                     container = 'vibsinglecellnf/harmony:1.0'
                     report_ipynb = "${params.misc.test.enabled ? '../../..' : ''}/src/harmony/bin/reports/sc_harmony_report.ipynb"
@@ -239,7 +239,7 @@ Steps:
 
         process SC__HARMONY__HARMONY_MATRIX {
 
-            container params.sc.harmony.container
+            container params.tools.harmony.container
             publishDir "${params.global.outdir}/data/intermediate", mode: 'symlink'
             clusterOptions "-l nodes=1:ppn=${params.global.threads} -l walltime=1:00:00 -A ${params.global.qsubaccount}"
 
@@ -250,7 +250,7 @@ Steps:
                 tuple val(sampleId), path("${sampleId}.SC__HARMONY__HARMONY_MATRIX.tsv")
 
             script:
-                def sampleParams = params.parseConfig(sampleId, params.global, params.sc.harmony)
+                def sampleParams = params.parseConfig(sampleId, params.global, params.tools.harmony)
                 processParams = sampleParams.local
                 varsUseAsArguments = processParams.varsUse.collect({ '--vars-use' + ' ' + it }).join(' ')
                 """
@@ -364,7 +364,7 @@ Steps:
 
                 // Run clustering
                 // Define the parameters for clustering
-                def clusteringParams = SC__SCANPY__CLUSTERING_PARAMS( clean(params.sc.scanpy.clustering) )
+                def clusteringParams = SC__SCANPY__CLUSTERING_PARAMS( clean(params.tools.scanpy.clustering) )
                 CLUSTER_IDENTIFICATION(
                     normalizedTransformedData,
                     DIM_REDUCTION_TSNE_UMAP.out.dimred_tsne_umap,
@@ -400,7 +400,7 @@ Steps:
                 )
                 harmony_report = GENERATE_DUAL_INPUT_REPORT(
                     becDualDataPrePost,
-                    file(workflow.projectDir + params.sc.harmony.report_ipynb),
+                    file(workflow.projectDir + params.tools.harmony.report_ipynb),
                     "SC_BEC_HARMONY_report",
                     clusteringParams.isParameterExplorationModeOn()
                 )
@@ -490,10 +490,10 @@ Steps:
                     SC__FILE_CONVERTER | \
                     FILTER_AND_ANNOTATE_AND_CLEAN
 
-                if(params.sc.scanpy.containsKey("filter")) {
+                if(params.tools.scanpy.containsKey("filter")) {
                     out = QC_FILTER( out ).filtered // Remove concat
                 }
-                if(params.sc.containsKey("file_concatenator")) {
+                if(params.utils.file_concatenator) {
                     out = SC__FILE_CONCATENATOR( 
                         out.map {
                             it -> it[1]
@@ -502,7 +502,7 @@ Steps:
                         ) 
                     )
                 }
-                if(params.sc.scanpy.containsKey("data_transformation") && params.sc.scanpy.containsKey("normalization")) {
+                if(params.tools.scanpy.containsKey("data_transformation") && params.tools.scanpy.containsKey("normalization")) {
                     out = NORMALIZE_TRANSFORM( out )
                 }
                 out = HVG_SELECTION( out )
@@ -527,7 +527,7 @@ Steps:
 
                 // Conversion
                 // Convert h5ad to X (here we choose: loom format)
-                if(params.sc.containsKey("file_concatenator")) {
+                if(params.utils?.file_concatenator) {
                     filteredloom = SC__H5AD_TO_FILTERED_LOOM( SC__FILE_CONCATENATOR.out )
                     scopeloom = FILE_CONVERTER(
                         BEC_HARMONY.out.data.groupTuple(),
@@ -552,7 +552,7 @@ Steps:
 
                 // Collect the reports:
                 // Define the parameters for clustering
-                def clusteringParams = SC__SCANPY__CLUSTERING_PARAMS( clean(params.sc.scanpy.clustering) )
+                def clusteringParams = SC__SCANPY__CLUSTERING_PARAMS( clean(params.tools.scanpy.clustering) )
                 // Pairing clustering reports with bec reports
                 if(!clusteringParams.isParameterExplorationModeOn()) {
                     clusteringBECReports = BEC_HARMONY.out.cluster_report.map {
@@ -722,8 +722,8 @@ Workflows import multiple processes and define the workflow by name:
     workflow CELLRANGER {
 
         main:
-            SC__CELLRANGER__MKFASTQ(file(params.sc.cellranger.mkfastq.csv), path(params.sc.cellranger.mkfastq.runFolder))
-            SC__CELLRANGER__COUNT(file(params.sc.cellranger.count.transcriptome), SC__CELLRANGER__MKFASTQ.out.flatten())
+            SC__CELLRANGER__MKFASTQ(file(params.tools.cellranger.mkfastq.csv), path(params.tools.cellranger.mkfastq.runFolder))
+            SC__CELLRANGER__COUNT(file(params.tools.cellranger.count.transcriptome), SC__CELLRANGER__MKFASTQ.out.flatten())
         emit:
             SC__CELLRANGER__COUNT.out
 
@@ -775,7 +775,7 @@ The parameter structure internally (post-merge) is:
             project_name = "MCF7"
             ...
         }
-        sc {
+        tools {
             utils {
                 file_converter {
                     ...