Include preprocessing

.gitignore

@@ -1,6 +1,7 @@
 .nextflow*
 work/
 data/
+references/
 results/
 .DS_Store
 tests/test_data

.travis.yml

@@ -2,41 +2,37 @@ sudo: required
 language: python
 jdk: openjdk8
 services: docker
+
+addons:
+  apt:
+    update: true
+
 python: '3.6'
 cache: pip
+
 matrix:
   fast_finish: true
 
+env:
+  - NXF_VER=19.04.0
+
 before_install:
   # PRs to master are only ok if coming from dev branch
   - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ])'
   # Pull the docker image first so the test doesn't wait for this
   - docker pull nfcore/sarek:dev
   # Fake the tag locally so that the pipeline runs properly
-  # Looks weird when this is :dev to :dev, but makes sense when testing code for a release (:dev to :1.0.1)
   - docker tag nfcore/sarek:dev nfcore/sarek:dev
 
 install:
   # Install Nextflow
   - mkdir /tmp/nextflow && cd /tmp/nextflow
   - wget -qO- get.nextflow.io | bash
   - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow
-  # Install nf-core/tools
-  - pip install --upgrade pip
-  - pip install nf-core
   # Reset
   - mkdir ${TRAVIS_BUILD_DIR}/tests && cd ${TRAVIS_BUILD_DIR}/tests
 
-env:
-  - NXF_VER=19.04.0
-
-jobs:
-  include:
-    - stage: lint
-      script: nf-core lint ${TRAVIS_BUILD_DIR}
-    - stage: built
-      script: skip
-      script: git clone --single-branch --branch sarek https://github.com/nf-core/test-datasets.git test-data
-      script: nextflow run ${TRAVIS_BUILD_DIR}/build.nf -profile docker --genome smallGRCh37 --refdir test-data/reference --outdir References
-    - stage: test
-      script: nextflow run ${TRAVIS_BUILD_DIR}/main.nf -profile docker --help
+script:
+  - git clone --single-branch --branch sarek https://github.com/nf-core/test-datasets.git data
+  - nextflow run ${TRAVIS_BUILD_DIR}/build.nf -profile docker --genome smallGRCh37 --refdir data/reference --outdir references --publishDirMode link --max_memory 7.GB --max_cpus 2
+  - nextflow run ${TRAVIS_BUILD_DIR}/main.nf -profile docker --genome smallGRCh37 --igenomes_base references --sampleDir data/testdata/tiny/normal --publishDirMode link --max_memory 7.GB --max_cpus 2

Jenkinsfile

@@ -13,13 +13,18 @@ pipeline {
         }
         stage('Build') {
             steps {
-              sh "git clone --single-branch --branch sarek https://github.com/nf-core/test-datasets.git test-data"
-              sh "nextflow run build.nf -profile docker --genome smallGRCh37 --refdir test-data/reference --outdir References"
+              sh "git clone --single-branch --branch sarek https://github.com/nf-core/test-datasets.git data"
+              sh "nextflow run build.nf -profile docker --genome smallGRCh37 --refdir data/reference --outdir references --publishDirMode link"
             }
         }
-        stage('Test') {
+        stage('SampleDir') {
             steps {
-                sh "nextflow run main.nf -profile docker --help"
+                sh "nextflow run main.nf -profile docker --genome smallGRCh37 --igenomes_base references --sampleDir data/testdata/tiny/normal --publishDirMode link"
+            }
+        }
+        stage('Multiple') {
+            steps {
+                sh "nextflow run main.nf -profile docker --genome smallGRCh37 --igenomes_base references --sample data/testdata/tsv/tiny-multiple.tsv --publishDirMode link"
             }
         }
     }

bin/scrape_software_versions.py

@@ -5,16 +5,46 @@
 
 # TODO nf-core: Add additional regexes for new tools in process get_software_versions
 regexes = {
-    'nf-core/sarek': ['v_pipeline.txt', r"(\S+)"],
-    'Nextflow': ['v_nextflow.txt', r"(\S+)"],
+    'AlleleCount': ['v_allelecount.txt', r"(\S+)"],
+    'ASCAT': ['v_ascat.txt', r"(\d\.\d+)"],
+    'bcftools': ['v_bcftools.txt', r"bcftools (\S+)"],
+    'BWA': ['v_bwa.txt', r"Version: (\S+)"],
     'FastQC': ['v_fastqc.txt', r"FastQC v(\S+)"],
+    'FreeBayes': ['v_freebayes.txt', r"version:  v(\d\.\d\.\d+)"],
+    'GATK': ['v_gatk.txt', r"Version:(\S+)"],
+    'htslib': ['v_samtools.txt', r"htslib (\S+)"],
+    'Manta': ['v_manta.txt', r"([0-9.]+)"],
     'MultiQC': ['v_multiqc.txt', r"multiqc, version (\S+)"],
+    'Nextflow': ['v_nextflow.txt', r"(\S+)"],
+    'nf-core/sarek': ['v_pipeline.txt', r"(\S+)"],
+    'Qualimap': ['v_qualimap.txt', r"QualiMap v.(\S+)"],
+    'R': ['v_r.txt', r"R version (\S+)"],
+    'samtools': ['v_samtools.txt', r"samtools (\S+)"],
+    'SnpEff': ['v_snpeff.txt', r"version SnpEff (\S+)"],
+    'Strelka': ['v_strelka.txt', r"([0-9.]+)"],
+    'vcftools': ['v_vcftools.txt', r"([0-9.]+)"],
+    'VEP': ['v_vep.txt', r"ensembl-vep          : (\S+)"],
 }
 results = OrderedDict()
 results['nf-core/sarek'] = '<span style="color:#999999;\">N/A</span>'
 results['Nextflow'] = '<span style="color:#999999;\">N/A</span>'
+results['AlleleCount'] = '<span style="color:#999999;\">N/A</span>'
+results['ASCAT'] = '<span style="color:#999999;\">N/A</span>'
+results['bcftools'] = '<span style="color:#999999;\">N/A</span>'
+results['BWA'] = '<span style="color:#999999;\">N/A</span>'
 results['FastQC'] = '<span style="color:#999999;\">N/A</span>'
+results['FreeBayes'] = '<span style="color:#999999;\">N/A</span>'
+results['GATK'] = '<span style="color:#999999;\">N/A</span>'
+results['htslib'] = '<span style="color:#999999;\">N/A</span>'
+results['Manta'] = '<span style="color:#999999;\">N/A</span>'
 results['MultiQC'] = '<span style="color:#999999;\">N/A</span>'
+results['Qualimap'] = '<span style="color:#999999;\">N/A</span>'
+results['R'] = '<span style="color:#999999;\">N/A</span>'
+results['samtools'] = '<span style="color:#999999;\">N/A</span>'
+results['SnpEff'] = '<span style="color:#999999;\">N/A</span>'
+results['Strelka'] = '<span style="color:#999999;\">N/A</span>'
+results['vcftools'] = '<span style="color:#999999;\">N/A</span>'
+results['VEP'] = '<span style="color:#999999;\">N/A</span>'
 
 # Search each file using its regex
 for k, v in regexes.items():

conf/base.config

@@ -17,8 +17,8 @@ process {
   time = { check_max( 2.h * task.attempt, 'time' ) }
 
   errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' }
-  maxRetries = 1
   maxErrors = '-1'
+  maxRetries = 1
 
   // Process-specific resource requirements
   // TODO nf-core: Customise requirements for specific processes.
@@ -27,8 +27,9 @@ process {
 
 params {
   // Defaults only, expecting to be overwritten
-  max_memory = 128.GB
+  igenomes_base = 's3://ngi-igenomes/igenomes/'
+  markdup_java_options = '"-Xms4000m -Xmx7g"' //Established values for markDuplicate memory consumption, see issue PR #689 for details
   max_cpus = 16
+  max_memory = 128.GB
   max_time = 240.h
-  igenomes_base = 's3://ngi-igenomes/igenomes/'
 }