fix: make GNU parallel more robust with --plain and --workdir in tmpd…

…ir (#328)
bihealth · Jan 10, 2023 · c033fad · c033fad
1 parent 1e24712
commit c033fad
Show file tree

Hide file tree

Showing 9 changed files with 23 additions and 7 deletions.
diff --git a/snappy_wrappers/wrappers/bcftools/call_joint/wrapper.py b/snappy_wrappers/wrappers/bcftools/call_joint/wrapper.py
@@ -48,11 +48,17 @@
 # Hack: get back bin directory of base/root environment.
 export PATH=$PATH:$(dirname $(dirname $(which conda)))/bin
 
+export TMPDIR=$(mktemp -d)
+trap "rm -rf $TMPDIR" EXIT
+mkdir -p $TMPDIR/parallel
+
 snappy-genome_windows \
     --fai-file $REF.fai \
     --window-size {snakemake.config[step_config][somatic_variant_calling][bcftools_joint][window_length]} \
     {args_ignore_chroms} \
 | parallel \
+    --plain \
+    --workdir $TMPDIR/parallel \
     --keep-order \
     --verbose \
     --max-procs {snakemake.config[step_config][somatic_variant_calling][bcftools_joint][num_threads]} \

diff --git a/snappy_wrappers/wrappers/bcftools_call/wrapper.py b/snappy_wrappers/wrappers/bcftools_call/wrapper.py
@@ -125,9 +125,10 @@
 export -f run-shard
 
 # Perform parallel execution
+mkdir -p $TMPDIR/parallel
 num_threads={snakemake.config[step_config][variant_calling][gatk4_hc_joint][num_threads]}
 cat $TMPDIR/final_intervals.txt \
-| parallel -j $num_threads 'run-shard {{#}} {{}}'
+| parallel --plain --workdir $TMPDIR/parallel -j $num_threads 'run-shard {{#}} {{}}'
 
 # Merge the individual shards' output VCF
 bcftools concat \

diff --git a/snappy_wrappers/wrappers/gatk3_hc/wrapper.py b/snappy_wrappers/wrappers/gatk3_hc/wrapper.py
@@ -108,9 +108,10 @@
 export -f run-shard
 
 # Perform parallel execution
+mkdir -p $TMPDIR/parallel
 num_threads={snakemake.config[step_config][variant_calling][gatk3_hc][num_threads]}
 cat $TMPDIR/final_intervals.txt \
-| parallel -j $num_threads 'run-shard {{#}} {{}}'
+| parallel --plain --workdir $TMPDIR/parallel -j $num_threads 'run-shard {{#}} {{}}'
 
 # Merge the individual shards' output VCF
 bcftools concat \

diff --git a/snappy_wrappers/wrappers/gatk3_ug/wrapper.py b/snappy_wrappers/wrappers/gatk3_ug/wrapper.py
@@ -109,9 +109,10 @@
 export -f run-shard
 
 # Perform parallel execution
+mkdir -p $TMPDIR/parallel
 num_threads={snakemake.config[step_config][variant_calling][gatk3_ug][num_threads]}
 cat $TMPDIR/final_intervals.txt \
-| parallel -j $num_threads 'run-shard {{#}} {{}}'
+| parallel --plain --workdir $TMPDIR/parallel -j $num_threads 'run-shard {{#}} {{}}'
 
 # Merge the individual shards' output VCF
 bcftools concat \

diff --git a/snappy_wrappers/wrappers/gatk4_hc/combine_gvcfs/wrapper.py b/snappy_wrappers/wrappers/gatk4_hc/combine_gvcfs/wrapper.py
@@ -110,9 +110,10 @@
 export -f run-shard
 
 # Perform parallel execution
+mkdir -p $TMPDIR/parallel
 num_threads={snakemake.config[step_config][variant_calling][gatk4_hc_gvcf][num_threads]}
 cat $TMPDIR/final_intervals.txt \
-| parallel -j $num_threads 'run-shard {{#}} {{}}'
+| parallel --workdir $TMPDIR/parallel --plain -j $num_threads 'run-shard {{#}} {{}}'
 
 # Merge the individual shards' output VCF
 bcftools concat \

diff --git a/snappy_wrappers/wrappers/gatk4_hc/discover/wrapper.py b/snappy_wrappers/wrappers/gatk4_hc/discover/wrapper.py
@@ -113,9 +113,10 @@
 export -f run-shard
 
 # Perform parallel execution
+mkdir -p $TMPDIR/parallel
 num_threads={snakemake.config[step_config][variant_calling][gatk4_hc_gvcf][num_threads]}
 cat $TMPDIR/final_intervals.txt \
-| parallel -j $num_threads 'run-shard {{#}} {{}}'
+| parallel --workdir $TMPDIR/parallel --plain -j $num_threads 'run-shard {{#}} {{}}'
 
 # Merge the individual shards' output VCF
 bcftools concat \

diff --git a/snappy_wrappers/wrappers/gatk4_hc/genotype/wrapper.py b/snappy_wrappers/wrappers/gatk4_hc/genotype/wrapper.py
@@ -106,9 +106,10 @@
 export -f run-shard
 
 # Perform parallel execution
+mkdir -p $TMPDIR/parallel
 num_threads={snakemake.config[step_config][variant_calling][gatk4_hc_gvcf][num_threads]}
 cat $TMPDIR/final_intervals.txt \
-| parallel -j $num_threads 'run-shard {{#}} {{}}'
+| parallel --workdir $TMPDIR/parallel --plain -j $num_threads 'run-shard {{#}} {{}}'
 
 # Merge the individual shards' output VCF
 bcftools concat \

diff --git a/snappy_wrappers/wrappers/gatk4_hc/joint/wrapper.py b/snappy_wrappers/wrappers/gatk4_hc/joint/wrapper.py
@@ -111,9 +111,10 @@
 export -f run-shard
 
 # Perform parallel execution
+mkdir -p $TMPDIR/parallel
 num_threads={snakemake.config[step_config][variant_calling][gatk4_hc_joint][num_threads]}
 cat $TMPDIR/final_intervals.txt \
-| parallel -j $num_threads 'run-shard {{#}} {{}}'
+| parallel --plain --workdir $TMPDIR/parallel -j $num_threads 'run-shard {{#}} {{}}'
 
 # Merge the individual shards' output VCF
 bcftools concat \

diff --git a/snappy_wrappers/wrappers/varscan/call_joint/wrapper.py b/snappy_wrappers/wrappers/varscan/call_joint/wrapper.py
@@ -108,9 +108,12 @@
 }}
 
 
+mkdir -p $TMPDIR/parallel
 for vartype in snp indel; do
     generate_intervals \
     | parallel \
+        --plain \
+        --workdir $TMPDIR/parallel
         --keep-order \
         --verbose \
         --max-procs {snakemake.config[step_config][somatic_variant_calling][varscan_joint][num_cores]} \