From 4d237b4d906e0dc07d36ef08ac5129ac92889124 Mon Sep 17 00:00:00 2001 From: Michal Szczerba Date: Sat, 2 Sep 2017 12:38:19 +0200 Subject: [PATCH 1/6] templates for submitting batch jobs can log job ID now --- mcpartools/scheduler/data/submit_slurm.sh | 6 +++++- mcpartools/scheduler/data/submit_torque.sh | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/mcpartools/scheduler/data/submit_slurm.sh b/mcpartools/scheduler/data/submit_slurm.sh index dafc465..a448dbe 100755 --- a/mcpartools/scheduler/data/submit_slurm.sh +++ b/mcpartools/scheduler/data/submit_slurm.sh @@ -1,2 +1,6 @@ #!/bin/bash -sbatch {options_args:s} --array=1-{jobs_no:d} --output="{log_dir:s}/output_%j_%a.log" --error="{log_dir:s}/error_%j_%a.log" {script_path:s} + +LOGFILE="$(dirname $0)/submit.log" +printf "Job ID: " > "$LOGFILE" + +sbatch {options_args:s} --array=1-{jobs_no:d} --output="{log_dir:s}/output_%j_%a.log" --error="{log_dir:s}/error_%j_%a.log" --parsable {script_path:s} | cut -d ";" -f 1 >> "$LOGFILE" diff --git a/mcpartools/scheduler/data/submit_torque.sh b/mcpartools/scheduler/data/submit_torque.sh index 5e32eeb..7aa5e7a 100644 --- a/mcpartools/scheduler/data/submit_torque.sh +++ b/mcpartools/scheduler/data/submit_torque.sh @@ -1,3 +1,6 @@ #!/usr/bin/env bash -qsub {options_args:s} -t 1-{jobs_no:d} -o {log_dir:s} -e {log_dir:s} {script_path:s} +LOGFILE="$(dirname $0)/submit.log" +printf "Job ID: " > "$LOGFILE" + +qsub {options_args:s} -t 1-{jobs_no:d} -o {log_dir:s} -e {log_dir:s} -terse {script_path:s} >> "$LOGFILE" From c3e11beaa2edb7e610634602729576adc827c700 Mon Sep 17 00:00:00 2001 From: Michal Szczerba Date: Sat, 2 Sep 2017 13:41:11 +0200 Subject: [PATCH 2/6] added comments about whereabouts of a log file and also displaying info about logging to a file --- mcpartools/scheduler/data/submit_slurm.sh | 5 +++-- mcpartools/scheduler/data/submit_torque.sh | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mcpartools/scheduler/data/submit_slurm.sh b/mcpartools/scheduler/data/submit_slurm.sh index a448dbe..b0a5851 100755 --- a/mcpartools/scheduler/data/submit_slurm.sh +++ b/mcpartools/scheduler/data/submit_slurm.sh @@ -1,6 +1,7 @@ #!/bin/bash -LOGFILE="$(dirname $0)/submit.log" +# Log file submit.log will be created in the same directory submit.sh is located +LOGFILE="$(cd $(dirname $0) && pwd)/submit.log" printf "Job ID: " > "$LOGFILE" -sbatch {options_args:s} --array=1-{jobs_no:d} --output="{log_dir:s}/output_%j_%a.log" --error="{log_dir:s}/error_%j_%a.log" --parsable {script_path:s} | cut -d ";" -f 1 >> "$LOGFILE" +sbatch {options_args:s} --array=1-{jobs_no:d} --output="{log_dir:s}/output_%j_%a.log" --error="{log_dir:s}/error_%j_%a.log" --parsable {script_path:s} | cut -d ";" -f 1 >> "$LOGFILE" && echo "Saving logs to $LOGFILE" diff --git a/mcpartools/scheduler/data/submit_torque.sh b/mcpartools/scheduler/data/submit_torque.sh index 7aa5e7a..30d97fe 100644 --- a/mcpartools/scheduler/data/submit_torque.sh +++ b/mcpartools/scheduler/data/submit_torque.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -LOGFILE="$(dirname $0)/submit.log" +# Log file submit.log will be created in the same directory submit.sh is located +LOGFILE="$(cd $(dirname $0) && pwd)/submit.log" printf "Job ID: " > "$LOGFILE" -qsub {options_args:s} -t 1-{jobs_no:d} -o {log_dir:s} -e {log_dir:s} -terse {script_path:s} >> "$LOGFILE" +qsub {options_args:s} -t 1-{jobs_no:d} -o {log_dir:s} -e {log_dir:s} -terse {script_path:s} >> "$LOGFILE" && echo "Saving logs to $LOGFILE" From ba844767913891637bb70d4857fc609e1a844ab9 Mon Sep 17 00:00:00 2001 From: Michal Szczerba Date: Sat, 2 Sep 2017 19:55:45 +0200 Subject: [PATCH 3/6] added logging submission time --- mcpartools/scheduler/data/submit_slurm.sh | 5 ++++- mcpartools/scheduler/data/submit_torque.sh | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/mcpartools/scheduler/data/submit_slurm.sh b/mcpartools/scheduler/data/submit_slurm.sh index b0a5851..069c968 100755 --- a/mcpartools/scheduler/data/submit_slurm.sh +++ b/mcpartools/scheduler/data/submit_slurm.sh @@ -2,6 +2,9 @@ # Log file submit.log will be created in the same directory submit.sh is located LOGFILE="$(cd $(dirname $0) && pwd)/submit.log" +echo "Saving logs to $LOGFILE" printf "Job ID: " > "$LOGFILE" -sbatch {options_args:s} --array=1-{jobs_no:d} --output="{log_dir:s}/output_%j_%a.log" --error="{log_dir:s}/error_%j_%a.log" --parsable {script_path:s} | cut -d ";" -f 1 >> "$LOGFILE" && echo "Saving logs to $LOGFILE" +sbatch {options_args:s} --array=1-{jobs_no:d} --output="{log_dir:s}/output_%j_%a.log" --error="{log_dir:s}/error_%j_%a.log" --parsable {script_path:s} | cut -d ";" -f 1 >> "$LOGFILE" + +printf "Submission time: `date +"%Y-%m-%d %H:%M:%S"`" >> "$LOGFILE" diff --git a/mcpartools/scheduler/data/submit_torque.sh b/mcpartools/scheduler/data/submit_torque.sh index 30d97fe..f195d6d 100644 --- a/mcpartools/scheduler/data/submit_torque.sh +++ b/mcpartools/scheduler/data/submit_torque.sh @@ -2,6 +2,9 @@ # Log file submit.log will be created in the same directory submit.sh is located LOGFILE="$(cd $(dirname $0) && pwd)/submit.log" +echo "Saving logs to $LOGFILE" printf "Job ID: " > "$LOGFILE" -qsub {options_args:s} -t 1-{jobs_no:d} -o {log_dir:s} -e {log_dir:s} -terse {script_path:s} >> "$LOGFILE" && echo "Saving logs to $LOGFILE" +qsub {options_args:s} -t 1-{jobs_no:d} -o {log_dir:s} -e {log_dir:s} -terse {script_path:s} >> "$LOGFILE" + +printf "Submission time: `date +"%Y-%m-%d %H:%M:%S"`" >> "$LOGFILE From b9b6c5b830e789e0ad76063e0c54d16c97582b81 Mon Sep 17 00:00:00 2001 From: Michal Szczerba Date: Sun, 3 Sep 2017 10:32:23 +0200 Subject: [PATCH 4/6] added logging stderr from batch system command --- mcpartools/scheduler/data/submit_slurm.sh | 25 +++++++++++++++++++--- mcpartools/scheduler/data/submit_torque.sh | 25 +++++++++++++++++++--- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/mcpartools/scheduler/data/submit_slurm.sh b/mcpartools/scheduler/data/submit_slurm.sh index 069c968..8aaf6ad 100755 --- a/mcpartools/scheduler/data/submit_slurm.sh +++ b/mcpartools/scheduler/data/submit_slurm.sh @@ -1,10 +1,29 @@ #!/bin/bash # Log file submit.log will be created in the same directory submit.sh is located +# submit.log is for storing stdout and stderr of sbatch command, for log info from individual jobs see {log_dir:s} directory LOGFILE="$(cd $(dirname $0) && pwd)/submit.log" +echo -n "" > "$LOGFILE" + +OUT=`mktemp` +ERR=`mktemp` +sbatch {options_args:s} --array=1-{jobs_no:d} --output="{log_dir:s}/output_%j_%a.log" --error="{log_dir:s}/error_%j_%a.log" --parsable {script_path:s} > $OUT 2>$ERR + echo "Saving logs to $LOGFILE" -printf "Job ID: " > "$LOGFILE" -sbatch {options_args:s} --array=1-{jobs_no:d} --output="{log_dir:s}/output_%j_%a.log" --error="{log_dir:s}/error_%j_%a.log" --parsable {script_path:s} | cut -d ";" -f 1 >> "$LOGFILE" +# if sbatch command ended with a success log following info +if [ $? -eq 0 ] ; then + echo "Job ID: `cat $OUT | cut -d ";" -f 1`" > "$LOGFILE" + echo "Submission time: `date +"%Y-%m-%d %H:%M:%S"`" >> "$LOGFILE" +fi + +# if output from stderr isn't an empty string then log it as well to submit.log +if [ "`cat $ERR`" != "" ] ; then + echo "---------------------" >> "$LOGFILE" + echo "ERROR MESSAGE" >>"$LOGFILE" + echo "---------------------" >> "$LOGFILE" + cat $ERR >> "$LOGFILE" +fi -printf "Submission time: `date +"%Y-%m-%d %H:%M:%S"`" >> "$LOGFILE" +rm $OUT +rm $ERR diff --git a/mcpartools/scheduler/data/submit_torque.sh b/mcpartools/scheduler/data/submit_torque.sh index f195d6d..c0e54e8 100644 --- a/mcpartools/scheduler/data/submit_torque.sh +++ b/mcpartools/scheduler/data/submit_torque.sh @@ -1,10 +1,29 @@ #!/usr/bin/env bash # Log file submit.log will be created in the same directory submit.sh is located +# submit.log is for storing stdout and stderr of qsub command, for log info from individual jobs see {log_dir:s} directory LOGFILE="$(cd $(dirname $0) && pwd)/submit.log" +echo -n "" > "$LOGFILE" + +OUT=`mktemp` +ERR=`mktemp` +qsub {options_args:s} -t 1-{jobs_no:d} -o {log_dir:s} -e {log_dir:s} -terse {script_path:s} > $OUT 2> $ERR + echo "Saving logs to $LOGFILE" -printf "Job ID: " > "$LOGFILE" -qsub {options_args:s} -t 1-{jobs_no:d} -o {log_dir:s} -e {log_dir:s} -terse {script_path:s} >> "$LOGFILE" +# if qsub command ended with a success log following info +if [ $? -eq 0 ] ; then + echo "Job ID: `cat $OUT | cut -d ";" -f 1`" > "$LOGFILE" + echo "Submission time: `date +"%Y-%m-%d %H:%M:%S"`" >> "$LOGFILE" +fi + +# if output from stderr isn't an empty string then log it as well to submit.log +if [ "`cat $ERR`" != "" ] ; then + echo "---------------------" >> "$LOGFILE" + echo "ERROR MESSAGE" >>"$LOGFILE" + echo "---------------------" >> "$LOGFILE" + cat $ERR >> "$LOGFILE" +fi -printf "Submission time: `date +"%Y-%m-%d %H:%M:%S"`" >> "$LOGFILE +rm $OUT +rm $ERR From 58ccaf9815089718e95d76db76a29b8a6163cdd1 Mon Sep 17 00:00:00 2001 From: Michal Szczerba Date: Thu, 7 Sep 2017 14:07:49 +0200 Subject: [PATCH 5/6] added exit traps for deleteing temporary files --- mcpartools/scheduler/data/submit_slurm.sh | 4 +--- mcpartools/scheduler/data/submit_torque.sh | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/mcpartools/scheduler/data/submit_slurm.sh b/mcpartools/scheduler/data/submit_slurm.sh index 8aaf6ad..e9b1d5d 100755 --- a/mcpartools/scheduler/data/submit_slurm.sh +++ b/mcpartools/scheduler/data/submit_slurm.sh @@ -7,6 +7,7 @@ echo -n "" > "$LOGFILE" OUT=`mktemp` ERR=`mktemp` +trap "rm -f $OUT $ERR" EXIT sbatch {options_args:s} --array=1-{jobs_no:d} --output="{log_dir:s}/output_%j_%a.log" --error="{log_dir:s}/error_%j_%a.log" --parsable {script_path:s} > $OUT 2>$ERR echo "Saving logs to $LOGFILE" @@ -24,6 +25,3 @@ if [ "`cat $ERR`" != "" ] ; then echo "---------------------" >> "$LOGFILE" cat $ERR >> "$LOGFILE" fi - -rm $OUT -rm $ERR diff --git a/mcpartools/scheduler/data/submit_torque.sh b/mcpartools/scheduler/data/submit_torque.sh index c0e54e8..3e36914 100644 --- a/mcpartools/scheduler/data/submit_torque.sh +++ b/mcpartools/scheduler/data/submit_torque.sh @@ -7,6 +7,7 @@ echo -n "" > "$LOGFILE" OUT=`mktemp` ERR=`mktemp` +trap "rm -f $OUT $ERR" EXIT qsub {options_args:s} -t 1-{jobs_no:d} -o {log_dir:s} -e {log_dir:s} -terse {script_path:s} > $OUT 2> $ERR echo "Saving logs to $LOGFILE" @@ -24,6 +25,3 @@ if [ "`cat $ERR`" != "" ] ; then echo "---------------------" >> "$LOGFILE" cat $ERR >> "$LOGFILE" fi - -rm $OUT -rm $ERR From 252526d274411d2971c2613518d5d3a8fe4a00ab Mon Sep 17 00:00:00 2001 From: Michal Szczerba Date: Fri, 8 Sep 2017 19:36:26 +0200 Subject: [PATCH 6/6] added comments for clearer code understanding --- mcpartools/scheduler/data/submit_slurm.sh | 9 ++++++--- mcpartools/scheduler/data/submit_torque.sh | 7 +++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/mcpartools/scheduler/data/submit_slurm.sh b/mcpartools/scheduler/data/submit_slurm.sh index e9b1d5d..294d3df 100755 --- a/mcpartools/scheduler/data/submit_slurm.sh +++ b/mcpartools/scheduler/data/submit_slurm.sh @@ -5,20 +5,23 @@ LOGFILE="$(cd $(dirname $0) && pwd)/submit.log" echo -n "" > "$LOGFILE" +# Create temporary files for parsing stdout and stderr output from sbatch command before storing them in submit.log OUT=`mktemp` ERR=`mktemp` +# On exit or if the script is interrupted (i.e. by receiving SIGINT signal) delete temporary files trap "rm -f $OUT $ERR" EXIT -sbatch {options_args:s} --array=1-{jobs_no:d} --output="{log_dir:s}/output_%j_%a.log" --error="{log_dir:s}/error_%j_%a.log" --parsable {script_path:s} > $OUT 2>$ERR + +sbatch {options_args:s} --array=1-{jobs_no:d} --output="{log_dir:s}/output_%j_%a.log" --error="{log_dir:s}/error_%j_%a.log" --parsable {script_path:s} > $OUT 2> $ERR echo "Saving logs to $LOGFILE" -# if sbatch command ended with a success log following info +# If sbatch command ended with a success log following info if [ $? -eq 0 ] ; then echo "Job ID: `cat $OUT | cut -d ";" -f 1`" > "$LOGFILE" echo "Submission time: `date +"%Y-%m-%d %H:%M:%S"`" >> "$LOGFILE" fi -# if output from stderr isn't an empty string then log it as well to submit.log +# If output from stderr isn't an empty string then log it as well to submit.log if [ "`cat $ERR`" != "" ] ; then echo "---------------------" >> "$LOGFILE" echo "ERROR MESSAGE" >>"$LOGFILE" diff --git a/mcpartools/scheduler/data/submit_torque.sh b/mcpartools/scheduler/data/submit_torque.sh index 3e36914..6404d64 100644 --- a/mcpartools/scheduler/data/submit_torque.sh +++ b/mcpartools/scheduler/data/submit_torque.sh @@ -5,20 +5,23 @@ LOGFILE="$(cd $(dirname $0) && pwd)/submit.log" echo -n "" > "$LOGFILE" +# Create temporary files for parsing stdout and stderr output from qsub command before storing them in submit.log OUT=`mktemp` ERR=`mktemp` +# On exit or if the script is interrupted (i.e. by receiving SIGINT signal) delete temporary files trap "rm -f $OUT $ERR" EXIT + qsub {options_args:s} -t 1-{jobs_no:d} -o {log_dir:s} -e {log_dir:s} -terse {script_path:s} > $OUT 2> $ERR echo "Saving logs to $LOGFILE" -# if qsub command ended with a success log following info +# If qsub command ended with a success log following info if [ $? -eq 0 ] ; then echo "Job ID: `cat $OUT | cut -d ";" -f 1`" > "$LOGFILE" echo "Submission time: `date +"%Y-%m-%d %H:%M:%S"`" >> "$LOGFILE" fi -# if output from stderr isn't an empty string then log it as well to submit.log +# If output from stderr isn't an empty string then log it as well to submit.log if [ "`cat $ERR`" != "" ] ; then echo "---------------------" >> "$LOGFILE" echo "ERROR MESSAGE" >>"$LOGFILE"