SciLifeLab · szilvajuhos · Apr 30, 2018 · Apr 26, 2018 · Apr 26, 2018 · Apr 26, 2018
diff --git a/.gitignore b/.gitignore
@@ -2,13 +2,11 @@ Annotation/
 Preprocessing/
 References/
 Reports/
+Sarek-data/
 VariantCalling/
 work/
 .*swp
+.DS_Store
 .nextflow*
 *.img
 *.tar.gz
-report.html*
-timeline.html*
-trace.txt*
-.DS_Store
diff --git a/buildReferences.nf b/buildReferences.nf
@@ -56,14 +56,7 @@ if (params.help) exit 0, helpMessage()
 if (!SarekUtils.isAllowedParams(params)) exit 1, "params unknown, see --help for more information"
 if (!checkUppmaxProject()) exit 1, "No UPPMAX project ID found! Use --project <UPPMAX Project ID>"
 
-if (!params.download && params.refDir == "" ) exit 1, "No --refDir specified"
-if (params.download && params.refDir != "" ) exit 1, "No need to specify --refDir"
-
-ch_referencesFiles = defReferencesFiles(params.genome)
-
-if (params.download && params.genome != "smallGRCh37") exit 1, "Not possible to download ${params.genome} references files"
-
-if (!params.download) ch_referencesFiles.each{checkFile(params.refDir + "/" + it)}
+ch_referencesFiles = Channel.fromPath("${params.refDir}/*")
 
 /*
 ================================================================================
@@ -73,37 +66,10 @@ if (!params.download) ch_referencesFiles.each{checkFile(params.refDir + "/" + it
 
 startMessage()
 
-process ProcessReference {
-  tag params.download ? {"Download: " + f_reference} : {"Link: " + f_reference}
-
-  input:
-    val(f_reference) from ch_referencesFiles
-
-  output:
-    file(f_reference) into ch_processedFiles
-
-  script:
-
-  if (params.download)
-  """
-  wget https://github.com/szilvajuhos/smallRef/raw/master/${f_reference}
-  """
-
-  else
-  """
-  ln -s ${params.refDir}/${f_reference} .
-  """
-}
-
-
-if (params.verbose) ch_processedFiles = ch_processedFiles.view {
-  "Files preprocessed  : ${it.fileName}"
-}
-
 ch_compressedfiles = Channel.create()
 ch_notCompressedfiles = Channel.create()
 
-ch_processedFiles
+ch_referencesFiles
   .choice(ch_compressedfiles, ch_notCompressedfiles) {it =~ ".(gz|tar.bz2)" ? 0 : 1}
 
 process DecompressFile {
@@ -294,9 +260,6 @@ def helpMessage() {
   this.sarekMessage()
   log.info "    Usage:"
   log.info "       nextflow run buildReferences.nf --refDir <pathToRefDir> --genome <genome>"
-  log.info "       nextflow run buildReferences.nf --download --genome smallGRCh37"
-  log.info "    --download"
-  log.info "       Download reference files. (only with --genome smallGRCh37)"
   log.info "    --refDir <Directoy>"
   log.info "       Specify a directory containing reference files."
   log.info "    --outDir <Directoy>"

diff --git a/data/tiny/dummy/normal/dummy_n_R1_xxx.fastq.gz b/data/tiny/dummy/normal/dummy_n_R1_xxx.fastq.gz
diff --git a/data/tiny/dummy/normal/dummy_n_R2_xxx.fastq.gz b/data/tiny/dummy/normal/dummy_n_R2_xxx.fastq.gz
diff --git a/data/tiny/dummy/tumor/dummy_t_R1_xxx.fastq.gz b/data/tiny/dummy/tumor/dummy_t_R1_xxx.fastq.gz
diff --git a/data/tiny/dummy/tumor/dummy_t_R2_xxx.fastq.gz b/data/tiny/dummy/tumor/dummy_t_R2_xxx.fastq.gz
diff --git a/data/tiny/manta/normal/C097F_N_111207.1.AGTTGCTT_R1_xxx.fastq.gz b/data/tiny/manta/normal/C097F_N_111207.1.AGTTGCTT_R1_xxx.fastq.gz
diff --git a/data/tiny/manta/normal/C097F_N_111207.1.AGTTGCTT_R2_xxx.fastq.gz b/data/tiny/manta/normal/C097F_N_111207.1.AGTTGCTT_R2_xxx.fastq.gz
diff --git a/data/tiny/manta/normal/C097F_N_111207.2.AGTTGCTT_R1_xxx.fastq.gz b/data/tiny/manta/normal/C097F_N_111207.2.AGTTGCTT_R1_xxx.fastq.gz
diff --git a/data/tiny/manta/normal/C097F_N_111207.2.AGTTGCTT_R2_xxx.fastq.gz b/data/tiny/manta/normal/C097F_N_111207.2.AGTTGCTT_R2_xxx.fastq.gz
diff --git a/data/tiny/manta/normal/C09DF_N_111207.4.AGTTGCTT_R1_xxx.fastq.gz b/data/tiny/manta/normal/C09DF_N_111207.4.AGTTGCTT_R1_xxx.fastq.gz
diff --git a/data/tiny/manta/normal/C09DF_N_111207.4.AGTTGCTT_R2_xxx.fastq.gz b/data/tiny/manta/normal/C09DF_N_111207.4.AGTTGCTT_R2_xxx.fastq.gz
diff --git a/data/tiny/manta/normal/D0F23_N_111212.1.AGTTGCTT_R1_xxx.fastq.gz b/data/tiny/manta/normal/D0F23_N_111212.1.AGTTGCTT_R1_xxx.fastq.gz
diff --git a/data/tiny/manta/normal/D0F23_N_111212.1.AGTTGCTT_R2_xxx.fastq.gz b/data/tiny/manta/normal/D0F23_N_111212.1.AGTTGCTT_R2_xxx.fastq.gz
diff --git a/data/tiny/manta/normal/D0F23_N_111212.2.AGTTGCTT_R1_xxx.fastq.gz b/data/tiny/manta/normal/D0F23_N_111212.2.AGTTGCTT_R1_xxx.fastq.gz
diff --git a/data/tiny/manta/normal/D0F23_N_111212.2.AGTTGCTT_R2_xxx.fastq.gz b/data/tiny/manta/normal/D0F23_N_111212.2.AGTTGCTT_R2_xxx.fastq.gz
diff --git a/data/tiny/manta/normal/D0F23_N_111212.3.AGTTGCTT_R1_xxx.fastq.gz b/data/tiny/manta/normal/D0F23_N_111212.3.AGTTGCTT_R1_xxx.fastq.gz
diff --git a/data/tiny/manta/normal/D0F23_N_111212.3.AGTTGCTT_R2_xxx.fastq.gz b/data/tiny/manta/normal/D0F23_N_111212.3.AGTTGCTT_R2_xxx.fastq.gz
diff --git a/data/tiny/manta/tumor/C09D_T_111207.3.ACCAACTG_R1_xxx.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.3.ACCAACTG_R1_xxx.fastq.gz
diff --git a/data/tiny/manta/tumor/C09D_T_111207.3.ACCAACTG_R2_xxx.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.3.ACCAACTG_R2_xxx.fastq.gz
diff --git a/data/tiny/manta/tumor/C09D_T_111207.5.ACCAACTG_R1_xxx.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.5.ACCAACTG_R1_xxx.fastq.gz
diff --git a/data/tiny/manta/tumor/C09D_T_111207.5.ACCAACTG_R2_xxx.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.5.ACCAACTG_R2_xxx.fastq.gz
diff --git a/data/tiny/manta/tumor/C09D_T_111207.6.ACCAACTG_R1_xxx.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.6.ACCAACTG_R1_xxx.fastq.gz
diff --git a/data/tiny/manta/tumor/C09D_T_111207.6.ACCAACTG_R2_xxx.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.6.ACCAACTG_R2_xxx.fastq.gz
diff --git a/data/tiny/manta/tumor/C09D_T_111207.7.ACCAACTG_R1_xxx.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.7.ACCAACTG_R1_xxx.fastq.gz
diff --git a/data/tiny/manta/tumor/C09D_T_111207.7.ACCAACTG_R2_xxx.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.7.ACCAACTG_R2_xxx.fastq.gz
diff --git a/data/tiny/manta/tumor/C09D_T_111207.8.ACCAACTG_R1_xxx.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.8.ACCAACTG_R1_xxx.fastq.gz
diff --git a/data/tiny/manta/tumor/C09D_T_111207.8.ACCAACTG_R2_xxx.fastq.gz b/data/tiny/manta/tumor/C09D_T_111207.8.ACCAACTG_R2_xxx.fastq.gz
diff --git a/data/tiny/tiny/normal/tiny_n_L001_R1_xxx.fastq.gz b/data/tiny/tiny/normal/tiny_n_L001_R1_xxx.fastq.gz
diff --git a/data/tiny/tiny/normal/tiny_n_L001_R2_xxx.fastq.gz b/data/tiny/tiny/normal/tiny_n_L001_R2_xxx.fastq.gz
diff --git a/data/tiny/tiny/normal/tiny_n_L002_R1_xxx.fastq.gz b/data/tiny/tiny/normal/tiny_n_L002_R1_xxx.fastq.gz
diff --git a/data/tiny/tiny/normal/tiny_n_L002_R2_xxx.fastq.gz b/data/tiny/tiny/normal/tiny_n_L002_R2_xxx.fastq.gz
diff --git a/data/tiny/tiny/normal/tiny_n_L004_R1_xxx.fastq.gz b/data/tiny/tiny/normal/tiny_n_L004_R1_xxx.fastq.gz
diff --git a/data/tiny/tiny/normal/tiny_n_L004_R2_xxx.fastq.gz b/data/tiny/tiny/normal/tiny_n_L004_R2_xxx.fastq.gz
diff --git a/data/tiny/tiny/normal/tiny_n_L007_R1_xxx.fastq.gz b/data/tiny/tiny/normal/tiny_n_L007_R1_xxx.fastq.gz
diff --git a/data/tiny/tiny/normal/tiny_n_L007_R2_xxx.fastq.gz b/data/tiny/tiny/normal/tiny_n_L007_R2_xxx.fastq.gz
diff --git a/data/tiny/tiny/normal/tiny_n_L008_R1_xxx.fastq.gz b/data/tiny/tiny/normal/tiny_n_L008_R1_xxx.fastq.gz
diff --git a/data/tiny/tiny/normal/tiny_n_L008_R2_xxx.fastq.gz b/data/tiny/tiny/normal/tiny_n_L008_R2_xxx.fastq.gz
diff --git a/data/tiny/tiny/tumor/tiny_t_L001_R1_xxx.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L001_R1_xxx.fastq.gz
diff --git a/data/tiny/tiny/tumor/tiny_t_L001_R2_xxx.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L001_R2_xxx.fastq.gz
diff --git a/data/tiny/tiny/tumor/tiny_t_L002_R1_xxx.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L002_R1_xxx.fastq.gz
diff --git a/data/tiny/tiny/tumor/tiny_t_L002_R2_xxx.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L002_R2_xxx.fastq.gz
diff --git a/data/tiny/tiny/tumor/tiny_t_L003_R1_xxx.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L003_R1_xxx.fastq.gz
diff --git a/data/tiny/tiny/tumor/tiny_t_L003_R2_xxx.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L003_R2_xxx.fastq.gz
diff --git a/data/tiny/tiny/tumor/tiny_t_L005_R1_xxx.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L005_R1_xxx.fastq.gz
diff --git a/data/tiny/tiny/tumor/tiny_t_L005_R2_xxx.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L005_R2_xxx.fastq.gz
diff --git a/data/tiny/tiny/tumor/tiny_t_L006_R1_xxx.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L006_R1_xxx.fastq.gz
diff --git a/data/tiny/tiny/tumor/tiny_t_L006_R2_xxx.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L006_R2_xxx.fastq.gz
diff --git a/data/tiny/tiny/tumor/tiny_t_L007_R1_xxx.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L007_R1_xxx.fastq.gz
diff --git a/data/tiny/tiny/tumor/tiny_t_L007_R2_xxx.fastq.gz b/data/tiny/tiny/tumor/tiny_t_L007_R2_xxx.fastq.gz
diff --git a/data/tiny/vcf/Strelka_1234N_variants.vcf.gz b/data/tiny/vcf/Strelka_1234N_variants.vcf.gz
diff --git a/data/tiny/vcf/Strelka_1234N_variants.vcf.gz.tbi b/data/tiny/vcf/Strelka_1234N_variants.vcf.gz.tbi
diff --git a/data/tiny/vcf/Strelka_9876T_variants.vcf.gz b/data/tiny/vcf/Strelka_9876T_variants.vcf.gz
diff --git a/data/tiny/vcf/Strelka_9876T_variants.vcf.gz.tbi b/data/tiny/vcf/Strelka_9876T_variants.vcf.gz.tbi
diff --git a/data/tsv/dream-normal-s3.tsv b/data/tsv/dream-normal-s3.tsv
diff --git a/data/tsv/sample.tsv b/data/tsv/sample.tsv
diff --git a/data/tsv/tiny-manta-s3.tsv b/data/tsv/tiny-manta-s3.tsv
diff --git a/data/tsv/tiny-manta.tsv b/data/tsv/tiny-manta.tsv
diff --git a/data/tsv/tiny-s3.tsv b/data/tsv/tiny-s3.tsv
diff --git a/data/tsv/tiny.tsv b/data/tsv/tiny.tsv
diff --git a/data/tumor_genes.bed → doc/Various/tumor_genes.bed b/data/tumor_genes.bed → doc/Various/tumor_genes.bed
diff --git a/main.nf b/main.nf
@@ -83,7 +83,7 @@ if (params.sample) tsvPath = params.sample
 // No need for tsv file for step annotate
 if (!params.sample && !params.sampleDir) {
   tsvPaths = [
-      'mapping': "${workflow.projectDir}/data/tsv/tiny.tsv",
+      'mapping': "${workflow.projectDir}/Sarek-data/testdata/tsv/tiny.tsv",
       'realign': "${directoryMap.nonRealigned}/nonRealigned.tsv",
       'recalibrate': "${directoryMap.nonRecalibrated}/nonRecalibrated.tsv"
   ]
@@ -721,12 +721,8 @@ def extractFastq(tsvFile) {
       def status     = returnStatus(list[2].toInteger())
       def idSample   = list[3]
       def idRun      = list[4]
-
-      // Normally path to files starts from workflow.launchDir
-      // But when executing workflow from Github
-      // Path to hosted FASTQ files starts from workflow.projectDir
-      def fastqFile1 = workflow.commitId && params.test ? returnFile("${workflow.projectDir}/${list[5]}") : returnFile(list[5])
-      def fastqFile2 = workflow.commitId && params.test ? returnFile("${workflow.projectDir}/${list[6]}") : returnFile(list[6])
+      def fastqFile1 = returnFile(list[5])
+      def fastqFile2 = returnFile(list[6])
 
       checkFileExtension(fastqFile1,".fastq.gz")
       checkFileExtension(fastqFile2,".fastq.gz")

diff --git a/scripts/do_all.sh b/scripts/do_all.sh
@@ -48,7 +48,7 @@ done
 
 if [[ $GENOME = smallGRCh37 ]]
 then
-    $GENOME = GRCh37
+    GENOME=GRCh37
 fi
 
 function toLower() {

diff --git a/scripts/test.sh b/scripts/test.sh
@@ -5,7 +5,7 @@ BUILD=false
 KEEP=false
 GENOME=smallGRCh37
 PROFILE=singularity
-SAMPLE=data/tsv/tiny.tsv
+SAMPLE=Sarek-data/testdata/tsv/tiny.tsv
 TEST=ALL
 TRAVIS=${TRAVIS:-false}
 
@@ -47,26 +47,31 @@ do
   esac
 done
 
-function nf_test() {
-  echo "$(tput setaf 1)nextflow run $@ -profile $PROFILE --genome $GENOME -resume --verbose$(tput sgr0)"
-  nextflow run $@ -profile $PROFILE --genome $GENOME -resume --genome_base $PWD/References/$GENOME --verbose
-}
-
 function run_wrapper() {
   ./scripts/wrapper.sh $@ --profile $PROFILE --genome $GENOME --genomeBase $PWD/References/$GENOME --verbose
 }
 
 function clean_repo() {
-  if [[ $TRAVIS == false ]] && [[ $KEEP == true ]]
+  if [[ $TRAVIS == false ]] && [[ $KEEP == false ]]
   then
+    echo "$(tput setaf 1)Cleaning directory$(tput sgr0)"
     rm -rf work .nextflow* Preprocessing Reports Annotation VariantCalling Results
   fi
 }
 
 # Build references only for smallGRCh37
 if [[ $GENOME == smallGRCh37 ]] && [[ $TEST != BUILDCONTAINERS ]] && [[ BUILD ]]
 then
-  nf_test buildReferences.nf --download --outDir References/$GENOME
+  if [[ ! -d Sarek-data ]]
+  then
+    echo "$(tput setaf 1)Cloning Sarek-data repository$(tput sgr0)"
+    git clone https://github.com/SciLifeLab/Sarek-data.git
+  fi
+  if [[ ! -d References ]]
+  then
+    echo "$(tput setaf 1)Building references$(tput sgr0)"
+    nextflow run buildReferences.nf --refDir Sarek-data/reference --outDir References/$GENOME -profile $PROFILE --genome $GENOME --verbose
+  fi
   # Remove images only on TRAVIS
   if [[ $PROFILE == docker ]] && [[ $TRAVIS == true ]]
   then
@@ -79,21 +84,21 @@ fi
 
 if [[ ALL,DIR =~ $TEST ]]
 then
-  run_wrapper --germline --sampleDir data/tiny/tiny/normal
+  run_wrapper --germline --sampleDir Sarek-data/testdata/tiny/normal
   clean_repo
 fi
 
 if [[ ALL,STEP =~ $TEST ]]
 then
-  run_wrapper --germline --sampleDir data/tiny/tiny/normal
+  run_wrapper --germline --sampleDir Sarek-data/testdata/tiny/normal
   run_wrapper --germline --step realign --noReports
   run_wrapper --germline --step recalibrate --noReports
   clean_repo
 fi
 
 if [[ ALL,GERMLINE =~ $TEST ]]
 then
-  run_wrapper --germline --sampleDir data/tiny/tiny/normal --variantCalling --tools HaplotypeCaller
+  run_wrapper --germline --sampleDir Sarek-data/testdata/tiny/normal --variantCalling --tools HaplotypeCaller
   clean_repo
 fi
 
@@ -104,8 +109,8 @@ fi
 
 if [[ ALL,MANTA =~ $TEST ]]
 then
-  run_wrapper --somatic --sample data/tsv/tiny-manta.tsv --variantCalling --tools Manta --noReports
-  run_wrapper --somatic --sample data/tsv/tiny-manta.tsv --variantCalling --tools Manta,Strelka --noReports --strelkaBP
+  run_wrapper --somatic --sample Sarek-data/testdata/tsv/tiny-manta.tsv --variantCalling --tools Manta --noReports
+  run_wrapper --somatic --sample Sarek-data/testdata/tsv/tiny-manta.tsv --variantCalling --tools Manta,Strelka --noReports --strelkaBP
   clean_repo
 fi
 
@@ -131,13 +136,12 @@ then
     rm -rf work/singularity/sarek-latest.img
     rm -rf work/singularity/picard-latest.img
   fi
-  run_wrapper --annotate --tools ${ANNOTATOR} --annotateVCF data/tiny/vcf/Strelka_1234N_variants.vcf.gz --noReports
-  run_wrapper --annotate --tools ${ANNOTATOR} --annotateVCF data/tiny/vcf/Strelka_1234N_variants.vcf.gz,data/tiny/vcf/Strelka_9876T_variants.vcf.gz
+  run_wrapper --annotate --tools ${ANNOTATOR} --annotateVCF Sarek-data/testdata/vcf/Strelka_1234N_variants.vcf.gz --noReports
+  run_wrapper --annotate --tools ${ANNOTATOR} --annotateVCF Sarek-data/testdata/vcf/Strelka_1234N_variants.vcf.gz,Sarek-data/testdata/vcf/Strelka_9876T_variants.vcf.gz
   clean_repo
 fi
 
 if [[ ALL,BUILDCONTAINERS =~ $TEST ]] && [[ $PROFILE == docker ]]
 then
-  nf_test buildContainers.nf --docker --containers gatk,igvtools,mutect1,picard,qctools,runallelecount,r-base,snpeff,sarek
-  clean_repo
+  ./scripts/do_all.sh --genome $GENOME
 fi
diff --git a/scripts/wrapper.sh b/scripts/wrapper.sh
@@ -98,61 +98,61 @@ function run_sarek() {
 
 if [[ $GERMLINE == true ]] && [[ $SOMATIC == true ]]
 then
-  echo "Germline and Somatic"
+  echo "$(tput setaf 1)Germline and Somatic$(tput sgr0)"
   exit
 fi
 
 if [[ $GERMLINE == true ]] && [[ $ANNOTATE == true ]]
 then
-  echo "Germline and Annotate"
+  echo "$(tput setaf 1)Germline and Annotate$(tput sgr0)"
   exit
 fi
 
 if [[ $SOMATIC == true ]] && [[ $SAMPLEDIR != '' ]]
 then
-  echo "Directory defined for Somatic"
+  echo "$(tput setaf 1)Directory defined for Somatic$(tput sgr0)"
   exit
 fi
 
 if [[ $GERMLINE == true ]] && [[ $SAMPLEDIR != '' ]]
 then
-  echo "Germline with SampleDir"
+  echo "$(tput setaf 1)Germline with SampleDir$(tput sgr0)"
   run_sarek main.nf --step $STEP --sampleDir $SAMPLEDIR
 fi
 
 if [[ $GERMLINE == true ]] && [[ $SAMPLETSV != '' ]]
 then
-  echo "Germline with TSV"
+  echo "$(tput setaf 1)Germline with TSV$(tput sgr0)"
   run_sarek main.nf --step $STEP --sample $SAMPLETSV
 fi
 
 if [[ $GERMLINE == true ]] && [[ $VARIANTCALLING == true ]]
 then
-  echo "GermlineVC"
+  echo "$(tput setaf 1)GermlineVC$(tput sgr0)"
   run_sarek germlineVC.nf --tools $TOOLS
 fi
 
 if [[ $SOMATIC == true ]] && [[ $SAMPLETSV != '' ]]
 then
-  echo "Somatic with TSV"
+  echo "$(tput setaf 1)Somatic with TSV$(tput sgr0)"
   run_sarek main.nf --step $STEP --sample $SAMPLETSV
 fi
 
 if [[ $SOMATIC == true ]] && [[ $VARIANTCALLING == true ]]
 then
-  echo "SomaticVC"
+  echo "$(tput setaf 1)SomaticVC$(tput sgr0)"
   run_sarek germlineVC.nf --tools $TOOLS
   run_sarek somaticVC.nf --tools $TOOLS
 fi
 
 if [[ $ANNOTATE == true ]]
 then
-  echo "Annotate"
+  echo "$(tput setaf 1)Annotate$(tput sgr0)"
   run_sarek annotate.nf --tools $TOOLS --annotateVCF $ANNOTATEVCF
 fi
 
 if [[ $REPORTS == true ]]
 then
-  echo "Reports"
+  echo "$(tput setaf 1)Reports$(tput sgr0)"
   run_sarek runMultiQC.nf
 fi