Merge pull request #34 from CCBR/feat_nftemplate

Nextflow CCBR Template
CCBR · Mar 7, 2024 · 32ca752 · 32ca752
2 parents 453680a + 18a471a
commit 32ca752
Show file tree

Hide file tree

Showing 32 changed files with 1,748 additions and 1,461 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -36,4 +36,11 @@ jobs:
           mkdir tmp && cd tmp
           which logan
           logan init
-          logan run -profile ci_stub,docker -stub
+          logan run -profile ci_stub,docker \
+          --fastq_input "/opt2/.tests/*R{1,2}.fastq.gz" \
+          --vc --cnv --sv \
+          --genome hg38 \
+          --outdir /opt2/output_tn_fqs \
+          --interval /opt2/.tests/interval.bed \
+          -stub
+
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
diff --git a/.tests/interval.bed b/.tests/interval.bed
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,6 @@
 # LOGAN development version
 
-- Python-based CLI
+- Changed over to Nextflow CCBR template and pip packaging
+    - Processes moved to `modules/local` directory
+    - Workflows under the `subworkflows/local` directory
+    - Processes fall under low/med/high, but adding a somaticvariant caller process
diff --git a/Docker_hubmodules b/Docker_hubmodules
diff --git a/assets/slurm_header_biowulf.sh b/assets/slurm_header_biowulf.sh
@@ -3,7 +3,7 @@
 #SBATCH --mem=1g
 #SBATCH --time=1-00:00:00
 #SBATCH --parsable
-#SBATCH -J "tool_name"
+#SBATCH -J "LOGAN"
 #SBATCH --mail-type=BEGIN,END,FAIL
 #SBATCH --output "log/slurm_%j.log"
 #SBATCH --output "log/slurm_%j.log"

diff --git a/assets/slurm_header_frce.sh b/assets/slurm_header_frce.sh
@@ -3,7 +3,7 @@
 #SBATCH --mem=1g
 #SBATCH --time=1-00:00:00
 #SBATCH --parsable
-#SBATCH -J "tool_name"
+#SBATCH -J "LOGAN"
 #SBATCH --mail-type=BEGIN,END,FAIL
 #SBATCH --output "log/slurm_%j.log"
 #SBATCH --output "log/slurm_%j.log"

diff --git a/bin/freec_paired.pl → bin/make_freec_genome_paired.pl b/bin/freec_paired.pl → bin/make_freec_genome_paired.pl
@@ -26,18 +26,19 @@
 print C "chrFiles = $chrFiles\n";
 print C "minimalSubclonePresence = 20\nmaxThreads = 8\n";
 print C "outputDir = $ARGV[0]\n\n";
- 
+
 print C '[sample]' . "\n\n";
- 
+
 print C "mateFile = $tumormateFile\n";
 print C "inputFormat = BAM\nmateOrientation = FR\n\n";
 
-print C '[BAF]' . "\n\n";
+print C '[control]' . "\n\n";
 
 print C "mateFile = $controlmateFile\n";
 print C "inputFormat = BAM\nmateOrientation = FR\n\n";
-
+
+print C '[BAF]' . "\n\n";
 print C "makePileup = $makePileup\n";
 print C "fastaFile = $fastaFile\n";
 print C "minimalCoveragePerPosition = 20\nminimalQualityPerPosition = 20\n";
-print C "SNPfile = $SNPfile";
+print C "SNPfile = $SNPfile";
diff --git a/conf/base.config b/conf/base.config
@@ -51,11 +51,27 @@ process {
     withLabel:process_high_memory {
         memory = { check_max( 200.GB * task.attempt, 'memory' ) }
     }
+    withLabel:process_somaticcaller {
+        cpus   = { check_max( 4     * task.attempt, 'cpus'    ) }
+        memory = { check_max( 64.GB * task.attempt, 'memory'  ) }
+        time   = { check_max( 72.h   * task.attempt, 'time'    ) }
+    }
+    withLabel:process_somaticcaller_high {
+        cpus   = { check_max( 16     * task.attempt, 'cpus'    ) }
+        memory = { check_max( 96.GB * task.attempt, 'memory'  ) }
+        time   = { check_max( 72.h   * task.attempt, 'time'    ) }
+    }
+    withLabel:process_highmem {
+        cpus   = { check_max( 4     * task.attempt, 'cpus'    ) }
+        memory = { check_max( 48.GB * task.attempt, 'memory'  ) }
+        time   = { check_max( 72.h   * task.attempt, 'time'    ) }
+    }
     withLabel:error_ignore {
         errorStrategy = 'ignore'
     }
     withLabel:error_retry {
         errorStrategy = 'retry'
         maxRetries    = 2
     }
+
 }
diff --git a/conf/biowulf.config b/conf/biowulf.config
@@ -21,14 +21,17 @@ singularity {
     enabled = true
     autoMounts = true
     cacheDir = "/data/CCBR_Pipeliner/SIFS"
-    envWhitelist='https_proxy,http_proxy,ftp_proxy,DISPLAY,SLURM_JOBID,SINGULARITY_BINDPATH'
+    envWhitelist = 'https_proxy,http_proxy,ftp_proxy,DISPLAY,SLURM_JOBID,SINGULARITY_BINDPATH'
+    runOptions = '-B /gs10,/gs11,/gs12,/gs9,/spin1,/data/CCBR_Pipeliner/,/data/CCBR/projects/,/vf/users,/gpfs,/fdb'
 }
 
 env.SINGULARITY_CACHEDIR = "/data/CCBR_Pipeliner/SIFS"
 
-process.clusterOptions = ' --gres=lscratch:200 '
-process.scratch = '/lscratch/$SLURM_JOBID'
-process.stageInMode = 'symlink'
-process.stageOutMode = 'rsync'
-// for running pipeline on group sharing data directory, this can avoid inconsistent files timestamps
-process.cache = 'lenient'
+process {
+    clusterOptions = ' --gres=lscratch:200 '
+    scratch = '/lscratch/$SLURM_JOBID'
+    stageInMode = 'symlink'
+    stageOutMode = 'rsync'
+    // for running pipeline on group sharing data directory, this can avoid inconsistent files timestamps
+    cache = 'lenient'
+}
diff --git a/conf/ci_stub.config b/conf/ci_stub.config
@@ -6,7 +6,7 @@ params {
     outdir = 'results/test'
 
     max_cpus = 2        // for GitHub Actions https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
-    max_memory = '6.GB'
+    max_memory = '4.GB'
     max_time   = '6.h'
 
     publish_dir_mode = "symlink"
@@ -15,4 +15,10 @@ params {
 process {
     cpus = 1
     memory = '1.GB'
+    scratch = false
+    singularity {
+        enabled = false
+    }
 }
+
+stubRun = true
diff --git a/conf/containers.config b/conf/containers.config
@@ -2,6 +2,9 @@
 params {
     containers {
         base = 'nciccbr/ccbr_ubuntu_base_20.04:v6.1'
-        logan = 'docker://dnousome/ccbr_logan_base:v0.3.0' 
+        logan = 'docker://dnousome/ccbr_logan_base:v0.3.4'
+        vcf2maf = 'docker://dnousome/ccbr_vcf2maf:v102.0.0'
+        lofreq = 'docker://dnousome/ccbr_lofreq:v0.0.1'
+        octopus = 'docker://dancooke/octopus:latest'
     }
 }
diff --git a/conf/frce.config b/conf/frce.config
@@ -19,11 +19,13 @@ singularity {
     envWhitelist='https_proxy,http_proxy,ftp_proxy,DISPLAY,SLURM_JOBID,SINGULARITY_BINDPATH'
 }
 
-process.scratch = null // TODO
+process {
+    scratch = null // TODO
 
-process.stageInMode = 'symlink'
-process.stageOutMode = 'rsync'
+    stageInMode = 'symlink'
+    stageOutMode = 'rsync'
 
-// for running pipeline on group sharing data directory, this can avoid inconsistent files timestamps
-process.cache = 'lenient'
-}
+    // for running pipeline on group sharing data directory, this can avoid inconsistent files timestamps
+    cache = 'lenient'
+
+}
diff --git a/conf/genomes.config b/conf/genomes.config
@@ -0,0 +1,68 @@
+params {
+    genomes {
+        'hg38' {
+            genome = "/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/bwamem2/Homo_sapiens_assembly38.fasta"
+            genomefai = "/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/bwamem2/Homo_sapiens_assembly38.fasta.fai"
+            bwagenome= "/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/genome/Homo_sapiens_assembly38.fasta"
+            genomedict= "/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/genome/Homo_sapiens_assembly38.dict"
+            wgsregion = "/data/CCBR_Pipeliner/Pipelines/LOGAN/resources/hg38/resources_broad_hg38_v0_wgs_calling_regions.hg38.interval_list"
+            intervals= "${projectDir}/assets/hg38_v0_wgs_calling_regions.hg38.bed"
+            //millsindel = "/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/GATK_resource_bundle/Mills_and_1000G_gold_standard.indels.hg38.vcf.gz" //Mills_and_1000G_gold_standard.indels.hg38.vcf.gz"
+            //shapeitindel =  "/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/GATK_resource_bundle/ALL.wgs.1000G_phase3.GRCh38.ncbi_remapper.20150424.shapeit2_indels.vcf.gz" //ALL.wgs.1000G_phase3.GRCh38.ncbi_remapper.20150424.shapeit2_indels.vcf.gz" //file(params.gold_indels2) //
+            KNOWNINDELS= '/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/GATK_resource_bundle/Mills_and_1000G_gold_standard.indels.hg38.vcf.gz'
+            KNOWNRECAL = '--known-sites /data/CCBR_Pipeliner/Pipelines/XAVIER/resources/hg38/GATK_resource_bundle/dbsnp_138.hg38.vcf.gz --known-sites /data/CCBR_Pipeliner/Pipelines/XAVIER/resources/hg38/GATK_resource_bundle/Mills_and_1000G_gold_standard.indels.hg38.vcf.gz --known-sites /data/CCBR_Pipeliner/Pipelines/XAVIER/resources/hg38/GATK_resource_bundle/ALL.wgs.1000G_phase3.GRCh38.ncbi_remapper.20150424.shapeit2_indels.vcf.gz'
+            dbsnp = "/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/GATK_resource_bundle/dbsnp_138.hg38.vcf.gz"
+            dbsnp_indel = "/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/GATK_resource_bundle/Mills_and_1000G_gold_standard.indels.hg38.vcf.gz"
+            gnomad = '--germline-resource /data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/GNOMAD/somatic-hg38-af-only-gnomad.hg38.vcf.gz' // /data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/GNOMAD/somatic-hg38-af-only-gnomad.hg38.vcf.gz
+            pon = "/data/CCBR_Pipeliner/Pipelines/LOGAN/resources/hg38/PON/updatedpon.vcf.gz"    //pon="/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/PON/hg38.noCOSMIC_ClinVar.pon.vcf.gz" //file{params.pon}
+            kgp = "/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/GATK_resource_bundle/1000G_phase1.snps.high_confidence.hg38.vcf.gz"
+            KRAKENBACDB = "/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/kraken/20180907_standard_kraken2"
+            snpeff_genome = "GRCh38.86"
+            snpeff_config = "/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/snpEff/4.3t/snpEff.config"
+            snpeff_bundle = "/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/snpEff/4.3t/"
+            sites_vcf= "/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/somalier/sites.hg38.vcf.gz"
+            somalier_ancestrydb="/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/somalier/1kg-somalier"
+            vepcache = "/fdb/VEP/102/cache"
+            vepspecies = "homo_sapiens"
+            vepbuild = "GRCh38"
+            octopus_sforest= "--somatic-forest /data/CCBR_Pipeliner/Pipelines/LOGAN/resources/hg38/octopus/somatic.v0.7.4.forest"
+            octopus_gforest= "--forest /data/CCBR_Pipeliner/Pipelines/LOGAN/resources/hg38/octopus/germline.v0.7.4.forest"
+            SEQUENZAGC = "/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/hg38/SEQUENZA/hg38_gc50Base.txt.gz"
+            chromosomes = ['chr1','chr2','chr3','chr4','chr5','chr6','chr7','chr8','chr9','chr10','chr11','chr12','chr13','chr14','chr15','chr16','chr17','chr18','chr19','chr20','chr21','chr22','chrX','chrY','chrM']
+        }
+
+        'mm10' {
+            genome = "/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/genome/bwamem2index/genome.fa" // file(params.genome)
+            genomefai = "/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/genome/bwamem2index/genome.fa.fai" // file(params.genome)
+            bwagenome= "/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/genome/bwaindex/genome.fa"
+            genomedict= "/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/genome/bwamem2index/genome.dict"
+            intervals="/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/genome/bwamem2index/mm10_wgsregions.bed"
+            KNOWNINDELS = "/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/dbsnp/mm10_known_indels.vcf.gz"
+            KNOWNRECAL = "-known-sites /data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/dbsnp/mm10_known_indels.vcf.gz -known-sites /data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/dbsnp/mm10_known_snps.vcf.gz"
+            dbsnp = "/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/dbsnp/mm10_allstrains_dbSNP142.vcf.gz"
+            pon = "/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/dbsnp/mm10_dbSNP_allStrains_compSet_noIND.vcf.gz"
+            kgp = "/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/dbsnp/mm10_knownSNPs_sites.vcf.gz"
+            KRAKENBACDB = "/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/kraken/20180907_standard_kraken2"
+            gnomad= "--germline-resource  /data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/dbsnp/mm10_allstrains_dbSNP142.vcf.gz"
+            snpeff_genome = "GRCm38.86"
+            snpeff_config = "/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/snpEff/4.3t/snpEff.config"
+            snpeff_bundle = "/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/snpEff/4.3t/"
+            sites_vcf = "/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/somalier/mm10.sites.vcf.gz"
+            //EDIT SOMALIER ANCESTRY AFTER!
+            somalier_ancestrydb="/data/CCBR_Pipeliner/CCBR_Pipeliner_Legacy/Exome-seek/hg38/somalier/1kg-somalier"
+            vepcache = "/fdb/VEP/102/cache"
+            vepspecies = "mus_musculus"
+            vepbuild= "GRCm38"
+            octopus_sforest = ""
+            octopus_gforest = ""
+	 	    SEQUENZAGC =  '/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/SEQUENZA/mm10.gc50Base.wig.gz'
+            FREEC {
+                FREECLENGTHS = "/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/FREEC/mm10.fa.fai"
+                FREECCHROMS = "/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/FREEC/Chromosomes"
+                FREECPILEUP = "/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/FREEC/mm10_dbSNP137.ucsc.freec.bed"
+                FREECSNPS= "/data/CCBR_Pipeliner/Pipelines/XAVIER/resources/mm10/FREEC/mm10_dbSNP137.ucsc.freec.txt.gz"
+            }
+            chromosomes = ['chr1','chr2','chr3','chr4','chr5','chr6','chr7','chr8','chr9','chr10','chr11','chr12','chr13','chr14','chr15','chr16','chr17','chr18','chr19','chrX','chrY','chrM']
+        }
+    }
+}
diff --git a/conf/interactive.config b/conf/interactive.config
@@ -3,5 +3,17 @@ params {
     max_memory = '220 GB'
     max_cpus = 56
     max_time = '12 h'
+
+
 }
 process.scratch = false
+
+
+singularity {
+    enabled = true
+    autoMounts = true
+    cacheDir = "/data/CCBR_Pipeliner/SIFS"
+    envWhitelist='https_proxy,http_proxy,ftp_proxy,DISPLAY,SLURM_JOBID,SINGULARITY_BINDPATH'
+}
+
+env.SINGULARITY_CACHEDIR = "/data/CCBR_Pipeliner/SIFS"