Updated Nextflow for the tests. Found a way to combine process option…

…s from different profiles.

Signed-off-by: Philip R. Kensche <p.kensche@dkfz-heidelberg.de>

nextflow.config

@@ -19,29 +19,23 @@ manifest {
 
 // The workflow may refer to an older container version, e.g. if the container was not updated.
 ext.containerVersion = '1.0.0'
-ext.containerUrl = "docker://ghcr.io/dkfz-odcf/nf-bam2fastq:${ext.containerVersion}"
+ext.containerUrl = "ghcr.io/dkfz-odcf/nf-bam2fastq:${ext.containerVersion}"
 ext.taskEnvYaml = "${projectDir}/task-environment.yml"
 
 profiles {
 
-    // It seems the `container` field is not used in the "lsf" profile, unless it is explicitly
-    // added as to the "lsf" block. I added it to all relevant "process" blocks. Whether it is
-    // going to be used, will depend on the `enabled` toggles in the `docker` and `singularity`
-    // profiles.
+    // Note, that here we don't use `process { ... }` but `executor { ... }` blocks, as these
+    // would override the settings from other profiles. The current settings allow to combine
+    // profiles, e.g. test,lsf,singularity, to run the workflow with test resources on LSF using
+    // Singularity containers.
 
     test {
-        process {
-            cpus = 1
-            memory = 1.GB
-            container = ext.containerUrl
-        }
+        process.cpus = 1
+        process.memory = 1.GB
     }
 
     local {
-        process {
-            executor = 'local'
-            container = ext.containerUrl
-        }
+        process.executor = 'local'
     }
 
     conda {
@@ -51,13 +45,14 @@ profiles {
 
     mamba {
         conda.enabled = true
-        useMamba = true
+        conda.useMamba = true
         conda.cacheDir = "${projectDir}/cache/conda"
     }
 
     docker {
         docker.enabled = true
         docker.runOptions='-u $(id -u):$(id -g)'
+        process.container = ext.containerUrl
     }
 
     singularity {
@@ -69,19 +64,16 @@ profiles {
         // Don't mount the home directory by default, because Bash may setup the environment from
         // there thus breaking the environment encapsulation and reproducibility of the workflow.
         singularity.runOptions = "--no-home"
+        process.container = "docker://${ext.containerUrl}"
     }
 
     lsf {
-        process {
-            executor = 'lsf'
-            clusterOptions = '-env none'
-            container = ext.containerUrl
-        }
-        executor {
-            // scratch = '$SCRATCHDIR/$LSB_JOBID'
-            perTaskReserve = false
-            perJobMemLimit = true
-        }
+        process.executor = 'lsf'
+        process.clusterOptions = '-env none'
+
+        // executor.scratch = '$SCRATCHDIR/$LSB_JOBID'
+        executor.perTaskReserve = false
+        executor.perJobMemLimit = true
     }
 
 }

test-environment.yml

@@ -7,6 +7,6 @@ dependencies:
   - mamba
   - bash=5.0.018
   - samtools=1.11
-  - nextflow=22.10.1
+  - nextflow=23.10.1
   - gradle=7.4.2
   - openjdk=11.0.15