Merge pull request #55 from CCBR/hotfix_somaticmerge

Hotfix- somatic merge rules and bugs in hostname

README.md

@@ -54,6 +54,16 @@ xavier run \
 --mode slurm \
 --runmode init 
 
+# Second, do a dry run to visualize outputs
+xavier run \
+--input data/*.R?.fastq.gz \
+--output /data/$USER/xavier_hg38 \
+--genome hg38 \
+--pairs pairs.txt \
+--targets Targets_hg38.bed \
+--mode slurm \
+--runmode dryrun 
+
 # Then do a complete run
 xavier run \
 --input data/*.R?.fastq.gz \

bin/redirect

@@ -17,24 +17,40 @@ TOOLDIR=$(dirname "$SCRIPTDIRNAME")
 TOOLNAME=$(basename "$SCRIPTNAME")
 echo $TOOLNAME
 
-# load conda
-if [[ "$HOSTNAME" == "biowulf.nih.gov" || "$SLURM_SUBMIT_HOST" == "biowulf.nih.gov" ]];then
+# find out if you are running on biowulf or frce and load conda 
+nbiowulf=$(scontrol show config | grep -i -c biowulf)
+if [[ "$nbiowulf" > 0 ]];then ISBIOWULF=true; else ISBIOWULF=false;fi
+nfrce=$(scontrol show config | grep -i -c fsitgl)
+if [[ "$nfrce" > 0 ]];then ISFRCE=true; else ISFRCE=false;fi
+
+
+if [[ $ISBIOWULF == true ]];then
        . "/data/CCBR_Pipeliner/db/PipeDB/Conda/etc/profile.d/conda.sh"
-elif [[ "$HOSTNAME" == "fsitgl-head01p.ncifcrf.gov" || "$SLURM_SUBMIT_HOST" == "fsitgl-head01p.ncifcrf.gov" ]];then
+	conda activate py311
+elif [[ $ISFRCE == true ]];then
        . "/mnt/projects/CCBR-Pipelines/resources/miniconda3/etc/profile.d/conda.sh"
+	conda activate py311
+else
+	echo "You are NOT running on BIOWULF or on FRCE"
+	echo "Please make sure that:"
+	echo "  - py311 conda environment is activated"
+	echo "  - singularity is in PATH"
+	echo "  - snakemake is in PATH"
 fi
-conda activate py311
 
 # load required modules
-module purge
-module load singularity 
-if [[ "$HOSTNAME" == "biowulf.nih.gov" ]];then
+if [[ $ISBIOWULF == true ]];then
+	module purge
+	module load singularity
 	module load snakemake
-elif [[ "$HOSTNAME" == "fsitgl-head01p.ncifcrf.gov" ]];then
+elif [[ $ISFRCE == true ]];then
 # snakemake module on FRCE does not work as expected
 # use the conda installed version of snakemake instead
+	module purge
+	module load singularity
 	export PATH="/mnt/projects/CCBR-Pipelines/bin:$PATH"
 fi
 
+
 ${TOOLDIR}/${TOOLNAME} "$@" || true
 

resources/runner

@@ -194,6 +194,7 @@ function submit(){
           # Create directory for logfiles
           mkdir -p "$3"/logfiles/slurmfiles/
           CLUSTER_OPTS="sbatch --cpus-per-task {cluster.threads} -p {cluster.partition} -t {cluster.time} --mem {cluster.mem} --job-name {cluster.name} --output {cluster.output} --error {cluster.error}"
+          snakemakeVer=$(snakemake --version 2>/dev/null)
           # Submit the master job to the cluster
           # sbatch --parsable -J {jobname} --time=5-00:00:00 --mail-type=BEGIN,END,FAIL 
           # --cpus-per-task=24 --mem=96g --gres=lscratch:500 
@@ -220,6 +221,12 @@ function submit(){
           #if [[ ${6#\'} != /lscratch* ]]; then
           #  CLUSTER_OPTS="sbatch --cpus-per-task {cluster.threads} -p {cluster.partition} -t {cluster.time} --mem {cluster.mem} --job-name={params.rname} -e $SLURM_DIR/slurm-%j_{params.rname}.out -o $SLURM_DIR/slurm-%j_{params.rname}.out"
           #fi
+          verlte() {
+        [  "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ]
+          }
+          verlt() {
+        [ "$1" = "$2" ] && return 1 || verlte $1 $2
+          }
           snakemakeOld=$(verlt $snakemakeVer 7.8 && echo "yes" || echo "no") # check if snakemake is older than 7.8
           if [ "$snakemakeOld" == "no" ];then
             triggeroptions="--rerun-triggers mtime"

resources/spooker

@@ -11,9 +11,11 @@ set -o pipefail
 PIPELINE_OUTDIR=$1
 
 # check if you are on biowulf or frce
-if [[ "$HOSTNAME" == "fsitgl-head01p.ncifcrf.gov" || "$HOSTNAME" == "biowulf.nih.gov" || "$SLURM_SUBMIT_HOST" == "biowulf.nih.gov" || "$SLURM_SUBMIT_HOST" == "fsitgl-head01p.ncifcrf.gov" ]]; then
+nbiowulf=$(scontrol show config | grep -i -c biowulf)
+nfrce=$(scontrol show config | grep -i -c fsitgl)
 
-    if [[ "$HOSTNAME" == "biowulf.nih.gov" || "$SLURM_SUBMIT_HOST" == "biowulf.nih.gov" ]];then
+if [[ "$nbiowulf" > 0  ||  "$nfrce" > 0 ]]; then
+    if [[ "$nbiowulf" > 0 ]]; then
         SPOOK=$(type -P spook)
         if [[ "$SPOOK" == "" ]];then
             echo "spook is not in PATH."
@@ -28,7 +30,7 @@ if [[ "$HOSTNAME" == "fsitgl-head01p.ncifcrf.gov" || "$HOSTNAME" == "biowulf.nih
         fi
         SPOOK_COPY2DIR="/scratch/ccbrpipeliner"
     fi
-    if [[  "$HOSTNAME" == "fsitgl-head01p.ncifcrf.gov" || "$SLURM_SUBMIT_HOST" == "fsitgl-head01p.ncifcrf.gov" ]];then
+    if [[ "$nfrce" > 0 ]]; then
         SPOOK_COPY2DIR="/mnt/projects/CCBR-Pipelines/pipelines/userdata/ccbrpipeliner"
     fi
 
@@ -71,12 +73,12 @@ if [[ "$HOSTNAME" == "fsitgl-head01p.ncifcrf.gov" || "$HOSTNAME" == "biowulf.nih
         # copy over the archive to secrete location and delete it
         if [ -f "${archivefile}" ];then
             gzip -f $archivefile
-            if [[ "$HOSTNAME" == "biowulf.nih.gov" ]];then
+            if [[ "$nbiowulf" > 0 ]];then
                 cmd="$SPOOK -f ${archivefile}.gz -d $SPOOK_COPY2DIR"
                 echo $cmd
                 $cmd
             fi
-            if [[ "$HOSTNAME" == "fsitgl-head01p.ncifcrf.gov" ]];then
+            if [[ "$nfrce" > 0 ]];then
                 cmd="cp -rv ${archivefile}.gz $SPOOK_COPY2DIR"
                 echo $cmd
                 $cmd

src/__init__.py

@@ -7,4 +7,4 @@
 sys.path.append(os.path.dirname(os.path.realpath(__file__)))
 
 # Ground source of truth for version information
-version ='v3.0'
+version ='v3.0.1'

src/run.py

@@ -174,17 +174,20 @@ def setup(sub_args, repo_path, output_path, create_nidap_folder_YN = 'no',links=
     ifiles = sym_safe(input_data = links, target = output_path)
     mixed_inputs(ifiles)
 
-    hostname = os.getenv("HOSTNAME")
-    slurmsubmithost = os.getenv("SLURM_SUBMIT_HOST")
-    if hostname == "fsitgl-head01p.ncifcrf.gov" or slurmsubmithost == "fsitgl-head01p.ncifcrf.gov" :
-        shorthostname = "frce"
-    elif hostname == "biowulf.nih.gov" or slurmsubmithost == "biowulf.nih.gov":
+    hpcget = subprocess.run(
+        "scontrol show config", shell=True, capture_output=True, text=True
+    )
+    hpcname = ""
+
+    if "biowulf" in hpcget.stdout:
         shorthostname = "biowulf"
-    #elif re.match("cn[0-9][0-9][0-9][0-9]",hostname):
-    #    shorthostname = "biowulf"
+        print("Thank you for running XAVIER on Biowulf")
+    elif "fsitgl" in hpcget.stdout:
+        shorthostname = "frce"
+        print("Thank you for running XAVIER on FRCE")
     else:
         shorthostname = "biowulf"
-        print("%s unknown host. Configuration files for references may not be correct. Defaulting to Biowulf config"%(hostname))
+        print("%s unknown host. Configuration files for references may not be correct. Defaulting to Biowulf config"%(hpcget))
 
     genome_config = os.path.join(repo_path,'config','genomes', sub_args.genome + '.' + shorthostname + '.json')
 

workflow/Snakefile

@@ -361,7 +361,7 @@ if (len(caller_list) >= 1):
     samples_for_caller_merge=pairs_ids
     somatic_callers_dirs=list(somatic_callers_dirs + [merge_outdir])
 
-VCF2MAF_WRAPPER=config['scripts']['vcf2maf_wrapper']
+#VCF2MAF_WRAPPER=config['scripts']['vcf2maf_wrapper']
 SOBDetector_out=os.path.join(output_somatic_base,"ffpe_filter","sobdetector")
 SOBDetector_JARFILE=os.path.join(SOBDetector_out, "jarfile","SOBDetector_v1.0.2.jar")
 
@@ -478,13 +478,6 @@ if 'FFPE_FILTER' in config['input_params']:
     if config['input_params']['FFPE_FILTER'].lower() in ['true','t','yes']:
         include: "rules/ffpe.smk"
 
-if config['project']['annotation']=='mm10':
-    include: "rules/qc_mm10.smk"
-elif config['project']['annotation']=='hg38':
-    include: "rules/qc_human.smk"
-elif config['project']['annotation']=='hg19':
-    include: 'rules/qc_human.smk'
-
 if tn_mode=="paired":
     include: "rules/somatic_snps.paired.smk"
 elif tn_mode=="tumor_only":