feat: support for snakemake --batch, pipeline.sh for array jobs (#325) (

#326)

snappy_pipeline/apps/snappy_snake.py

@@ -176,6 +176,8 @@ def run(wrapper_args):  # noqa: C901
         snakemake_argv.append("--dryrun")
     if wrapper_args.reason:
         snakemake_argv.append("--reason")
+    if wrapper_args.batch:
+        snakemake_argv += ["--batch", wrapper_args.batch]
     if not wrapper_args.snappy_pipeline_use_profile:
         snakemake_argv += ["--cores", str(wrapper_args.cores or 1)]
     if wrapper_args.conda_create_envs_only:
@@ -281,7 +283,11 @@ def main(argv=None):
         default=rerun_triggers_default,
         help=f"Expose --rerun-triggers from snakemake and set to {rerun_triggers_default} by default",
     )
-
+    group.add_argument(
+        "--batch",
+        metavar="RULE=BATCH/BATCHES",
+        help="Create the given batch for the given rule.  See Snakemake documentation for more info.",
+    )
     group = parser.add_argument_group(
         "Snakemake Verbosity / Debugging",
         "Arguments from Snakemake that are useful for debugging, such as " "increasing verbosity",

snappy_pipeline/apps/tpls/pipeline_job.sh

@@ -110,8 +110,24 @@ set -x
 
 # Kick off Snakemake --------------------------------------------------------
 
+# Interpret array jobs.
+set +e
+SNAKEMAKE_BATCH_END=$SLURM_ARRAY_TASK_MAX
+SNAKEMAKE_BATCH_CUR=$SLURM_ARRAY_TASK_ID
+if [[ ! -z "$SNAKEMAKE_BATCH_CUR" ]]; then
+    if [[ $SLURM_ARRAY_TASK_ID -lt 1 ]]; then
+        >&2 echo "Snakemake batches start couting at 1"
+        exit 1
+    fi
+    SNAKEMAKE_BATCH_ARG="--batch ${SNAKEMAKE_BATCH_RULE-default}=${SNAKEMAKE_BATCH_CUR}/${SNAKEMAKE_BATCH_END}"
+else
+    SNAKEMAKE_BATCH_ARG=
+fi
+set -e
+
 # Using the medium project/queue is a sensible default.
 snappy-snake --printshellcmds \
+    ${SNAKEMAKE_BATCH_ARG} \
     --snappy-pipeline-use-profile "cubi-v1" \
     --snappy-pipeline-jobs $MAX_JOBS \
     --restart-times ${RESTART_TIMES} \