Adds lint config to ignore some tests (linting)

.github/workflows/ci.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - dev
+      - master
   pull_request:
   release:
     types: [published]
@@ -18,9 +19,10 @@ jobs:
       NXF_VER: ${{ matrix.nxf_ver }}
       NXF_ANSI_LOG: false
     strategy:
+      fail-fast: false
       matrix:
         # Nextflow versions: check pipeline minimum and current latest
-        nxf_ver: ['20.04.0', '']
+        nxf_ver: ['20.01.0', '']
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v2
@@ -68,7 +70,7 @@ jobs:
           sudo mv nextflow /usr/local/bin/
         env:
           # Only check Nextflow pipeline minimum version
-          NXF_VER: '20.04.0'
+          NXF_VER: '20.01.0'
       - name: Pull docker image
         run: |
           docker pull nfcore/sarek:2.7
@@ -91,7 +93,7 @@ jobs:
           sudo mv nextflow /usr/local/bin/
         env:
           # Only check Nextflow pipeline minimum version
-          NXF_VER: '20.04.0'
+          NXF_VER: '20.01.0'
       - name: Pull docker image
         run: docker pull nfcore/sarek:2.7
       - name: Get test data
@@ -119,7 +121,7 @@ jobs:
           sudo mv nextflow /usr/local/bin/
         env:
           # Only check Nextflow pipeline minimum version
-          NXF_VER: '20.04.0'
+          NXF_VER: '20.01.0'
       - name: Pull docker image
         run: docker pull nfcore/sarek:2.7
       - name: Run test for minimal genomes
@@ -140,7 +142,7 @@ jobs:
           sudo mv nextflow /usr/local/bin/
         env:
           # Only check Nextflow pipeline minimum version
-          NXF_VER: '20.04.0'
+          NXF_VER: '20.01.0'
       - name: Pull docker image
         run: docker pull nfcore/sarek:2.7
       - name: Run ${{ matrix.profile }} test
@@ -161,7 +163,7 @@ jobs:
           sudo mv nextflow /usr/local/bin/
         env:
           # Only check Nextflow pipeline minimum version
-          NXF_VER: '20.04.0'
+          NXF_VER: '20.01.0'
       - name: Pull docker image
         run: docker pull nfcore/sarek:2.7
       - name: Run ${{ matrix.profile }} test
@@ -192,7 +194,7 @@ jobs:
           sudo mv nextflow /usr/local/bin/
         env:
           # Only check Nextflow pipeline minimum version
-          NXF_VER: '20.04.0'
+          NXF_VER: '20.01.0'
       - name: Pull docker image
         run: docker pull nfcore/sarek:2.7
       - name: Run ${{ matrix.tool }} test

.nf-core-lint.yml

@@ -0,0 +1,17 @@
+actions_awsfulltest : False
+actions_awstest : False
+actions_ci : False
+actions_schema_validation : False
+conda_dockerfile : False
+conda_env_yaml : False
+files_exist : False
+files_unchanged : False
+merge_markers : False
+nextflow_config : False
+pipeline_name_conventions : False
+pipeline_todos : False
+readme : False
+schema_lint : False
+schema_params : False
+template_strings : False
+version_consistency : False

conf/base.config

@@ -15,9 +15,9 @@ process {
   time = {check_resource(24.h * task.attempt)}
   shell = ['/bin/bash', '-euo', 'pipefail']
 
-  errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' }
+errorStrategy = { task.attempt == process.maxRetries ? params.errorStrategy : task.exitStatus in [3,9,10,14,143,137,104,134,139] ? 'retry' : params.errorStrategy }
   maxErrors = '-1'
-  maxRetries = 3
+  maxRetries = 10
 
   // Process-specific resource requirements
   // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors
@@ -43,30 +43,39 @@ process {
 
   withLabel:cpus_1 {
     cpus = {check_resource(1)}
+    memory = {check_resource(2.GB)}
   }
   withLabel:cpus_2 {
     cpus = {check_resource(2)}
+    memory = {check_resource(4.GB)}
   }
   withLabel:cpus_4 {
     cpus = {check_resource(4)}
+    memory = {check_resource(8.GB)}
   }
   withLabel:cpus_8 {
     cpus = {check_resource(8)}
+    memory = {check_resource(16.GB)}
   }
   withLabel:cpus_16 {
     cpus = {check_resource(16)}
+    memory = {check_resource(32.GB)}
   }
   withLabel:cpus_max {
     cpus = {params.max_cpus}
+    memory = {params.max_memory}
   }
 
   withLabel:memory_singleCPU_2_task {
+    cpus = {check_resource(1 * task.attempt)}
     memory = {check_resource((params.single_cpu_mem as nextflow.util.MemoryUnit) * 2 * task.attempt)}
   }
   withLabel:memory_singleCPU_task_sq {
+    cpus = {check_resource(1 * task.attempt)}
     memory = {check_resource((params.single_cpu_mem as nextflow.util.MemoryUnit) * task.attempt * task.attempt)}
   }
   withLabel:memory_max {
+    cpus = 1
     memory = {params.max_memory}
   }
 

conf/google.config

@@ -0,0 +1,96 @@
+// This config is specific to google-life-science
+
+includeConfig 'base.config'
+
+google {
+    lifeSciences.bootDiskSize = 50.GB
+    lifeSciences.preemptible = true
+    zone = params.gls_zone
+    network = params.gls_network
+    subnetwork = params.gls_subnetwork
+}
+
+docker.enabled = true
+
+executor {
+    name = 'google-lifesciences'
+}
+
+params {
+  single_cpu_mem = '2.GB'
+}
+
+process {
+	disk = 100.GB
+
+	withName: 'sra' {
+        	disk = "100.GB"
+		cpus = {check_max(16, 'cpus')}
+        	memory = {check_max(60.GB, 'memory')}
+		errorStrategy = {task.attempt == process.maxRetries ? 'ignore' : task.exitStatus in [1,3,9,10,14,143,137,104,134,139] ? 'retry' : 'ignore' }
+	}
+
+	withName: 'BaseRecalibrator' {
+		executor = 'local'
+		disk = "30.GB"
+		cpus = {check_max(1, 'cpus')}
+        	memory = {check_max(4.GB, 'memory')}
+	}
+
+    	withName: 'MapReads' {disk = "100.GB"}
+        withName: 'MergeMpileup' {disk = "80.GB"}
+        withName: 'ControlFREEC' {disk = "250.GB"}
+        withName: 'ControlFREECSingle' {disk = "200.GB"}
+        withName: 'MarkDuplicates' {disk = "50.GB"}
+        withName: 'MergeBamRecal' {disk = "40.GB"}
+        withName: 'ConcatVCF' {disk = "30.GB"}
+        withName: 'Mpileup' {disk = "20.GB"}
+        withName: 'ApplyBQSR' {disk = "30.GB"}
+        withName: 'Vcftools' {disk = "30.GB"}
+        withName: 'FreeBayes' {disk = "30.GB"}
+        withName: 'FreebayesSingle' {disk = "30.GB"}
+        withName: 'BamQC' {disk = "50.GB"}
+        withName: 'FastQCFQ' {disk = "50.GB"}
+        withName: 'GatherBQSRReports' {disk = "30.GB"}
+        withName: 'ControlFreecVizSingle' {disk = "30.GB"}
+        withName: 'ControlFreecViz' {disk = "30.GB"}
+        withName: 'get_software_versions' {disk = "30.GB"}
+        withName: 'SamtoolsStats' {disk = "30.GB"}
+        withName: 'Output_documentation' {disk = "30.GB"}
+        withName: 'BcftoolsStats' {disk = "30.GB"}
+        withName: 'CreateIntervalBeds' {disk = "30.GB"}
+        withName: 'MultiQC' {disk = "50.GB"}
+}
+
+// Function to ensure that resource requirements don't go beyond
+// a maximum limit
+def check_max(obj, type) {
+  if (type == 'memory') {
+    try {
+      if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1)
+        return params.max_memory as nextflow.util.MemoryUnit
+      else
+        return obj
+    } catch (all) {
+      println "   ### ERROR ###   Max memory '${params.max_memory}' is not valid! Using default value: $obj"
+      return obj
+    }
+  } else if (type == 'time') {
+    try {
+      if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1)
+        return params.max_time as nextflow.util.Duration
+      else
+        return obj
+    } catch (all) {
+      println "   ### ERROR ###   Max time '${params.max_time}' is not valid! Using default value: $obj"
+      return obj
+    }
+  } else if (type == 'cpus') {
+    try {
+      return Math.min( obj, params.max_cpus as int )
+    } catch (all) {
+      println "   ### ERROR ###   Max cpus '${params.max_cpus}' is not valid! Using default value: $obj"
+      return obj
+    }
+  }
+}

conf/local.config

@@ -0,0 +1,11 @@
+/*
+ * -------------------------------------------------
+ *  Nextflow config file for running on a local machine eg. laptop
+ * -------------------------------------------------
+ */
+
+docker.enabled = true
+
+params {
+    executor = 'local'
+}

conf/slurm.config

@@ -0,0 +1,14 @@
+/*
+ * -------------------------------------------------
+ *  Nextflow config file for running on a SLURM managed cluster
+ * -------------------------------------------------
+ */
+
+process {
+  executor = 'slurm'
+  beforeScript = 'module load singularity'
+}
+singularity {
+  enabled = true
+  autoMounts = true
+}

main.nf

@@ -239,10 +239,13 @@ if (workflow.profile.contains('awsbatch')) {
 
 // MultiQC
 // Stage config files
-ch_multiqc_config = file("$projectDir/assets/multiqc_config.yaml", checkIfExists: true)
+ch_multiqc_config = file("${params.projectDir}/assets/multiqc_config.yaml", checkIfExists: true)
 ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config, checkIfExists: true) : Channel.empty()
-ch_output_docs = file("$projectDir/docs/output.md", checkIfExists: true)
-ch_output_docs_images = file("$projectDir/docs/images/", checkIfExists: true)
+
+if (!skipQC.contains('documentation')) {
+ch_output_docs = file("${params.projectDir}/docs/output.md", checkIfExists: true)
+ch_output_docs_images = file("${params.projectDir}/docs/images/", checkIfExists: true)
+}
 
 // Handle input
 tsvPath = null
@@ -1996,7 +1999,7 @@ process HaplotypeCaller {
     intervalsOptions = params.no_intervals ? "" : "-L ${intervalBed}"
     dbsnpOptions = params.dbsnp ? "--D ${dbsnp}" : ""
     """
-    gatk --java-options "-Xmx${task.memory.toGiga()}g -Xms6000m -XX:GCTimeLimit=50 -XX:GCHeapFreeLimit=10" \
+    gatk --java-options "-Xmx${task.memory.toGiga()}g -Xms4000m -XX:GCTimeLimit=50 -XX:GCHeapFreeLimit=10" \
         HaplotypeCaller \
         -R ${fasta} \
         -I ${bam} \
@@ -3891,7 +3894,9 @@ process MultiQC {
 
 ch_multiqc_report.dump(tag:'MultiQC')
 
+
 // Output Description HTML
+if (!skipQC.contains('documentation')) {
 process Output_documentation {
     publishDir "${params.outdir}/pipeline_info", mode: params.publish_dir_mode
 
@@ -3909,6 +3914,8 @@ process Output_documentation {
     markdown_to_html.py $output_docs -o results_description.html
     """
 }
+}
+if (params.on_complete) {
 
 // Completion e-mail notification
 workflow.onComplete {
@@ -3962,18 +3969,18 @@ workflow.onComplete {
 
     // Render the TXT template
     def engine = new groovy.text.GStringTemplateEngine()
-    def tf = new File("$projectDir/assets/email_template.txt")
+    def tf = new File("${params.projectDir}/assets/email_template.txt")
     def txt_template = engine.createTemplate(tf).make(email_fields)
     def email_txt = txt_template.toString()
 
     // Render the HTML template
-    def hf = new File("$projectDir/assets/email_template.html")
+    def hf = new File("${params.projectDir}/assets/email_template.html")
     def html_template = engine.createTemplate(hf).make(email_fields)
     def email_html = html_template.toString()
 
     // Render the sendmail template
-    def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile: mqc_report, mqcMaxSize: params.max_multiqc_email_size.toBytes() ]
-    def sf = new File("$projectDir/assets/sendmail_template.txt")
+    def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "${params.projectDir}", mqcFile: mqc_report, mqcMaxSize: params.max_multiqc_email_size.toBytes() ]
+    def sf = new File("${params.projectDir}/assets/sendmail_template.txt")
     def sendmail_template = engine.createTemplate(sf).make(smail_fields)
     def sendmail_html = sendmail_template.toString()
 
@@ -4023,6 +4030,7 @@ workflow.onComplete {
         log.info "-${c_purple}[nf-core/sarek]${c_red} Pipeline completed with errors${c_reset}-"
     }
 }
+}
 
 /*
 ================================================================================

main.pbs

@@ -0,0 +1,19 @@
+#!/bin/bash
+#SBATCH -o logs.%j.out
+#SBATCH -e logs.%j.err
+#SBATCH --mail-type=END,FAIL
+#SBATCH --mail-user=$USER@my-org.org
+#SBATCH --mem=20000
+#SBATCH --cpus-per-task=4
+#SBATCH -p compute
+#SBATCH -q batch
+#SBATCH -t 3-00:00:00 
+
+cd $SLURM_SUBMIT_DIR
+date;hostname;pwd
+
+module load singularity
+
+curl -fsSL https://get.nextflow.io | bash
+
+./nextflow run /absolut/path/to/pipeline/repo/main.nf --outdir ${SLURM_SUBMIT_DIR} --param1 param1_value