|
| 1 | +#!/bin/bash |
| 2 | + |
| 3 | +if [ "$#" -lt "3" ]; then |
| 4 | + echo -e "Not enough arguments!" 1>&2 |
| 5 | + exit 1 |
| 6 | +fi |
| 7 | + |
| 8 | +CONTAINING_DIRECTORY="$( dirname "${BASH_SOURCE[0]}" )" |
| 9 | +SPARK_JANELIA="${SPARK_JANELIA:-${CONTAINING_DIRECTORY}/spark-janelia/spark-janelia}" |
| 10 | +# MVN="${MVN:-/misc/local/maven-3.2.2/bin/mvn}" |
| 11 | + |
| 12 | +RUNTIME="${RUNTIME:-8:00}" |
| 13 | +SPARK_VERSION="${SPARK_VERSION:-2.3.1}" |
| 14 | +TERMINATE="${TERMINATE:-1}" |
| 15 | +MIN_WORKERS="${MIN_WORKERS:-1}" |
| 16 | + |
| 17 | +N_EXECUTORS_PER_NODE="${N_EXECUTORS_PER_NODE:-6}" |
| 18 | +N_CORES_PER_EXECUTOR="${N_CORES_PER_EXECUTOR:-5}" |
| 19 | +MEMORY_PER_NODE="${MEMORY_PER_NODE:-300}" |
| 20 | +SPARK_OPTIONS="${SPARK_OPTIONS:-}" |
| 21 | + |
| 22 | +N_DRIVER_THREADS="${N_DRIVER_THREADS:-16}" |
| 23 | + |
| 24 | +N_NODES=$1; shift |
| 25 | +COORDINATE="$1"; shift |
| 26 | +CLASS=$1; shift |
| 27 | +ARGV="$@" |
| 28 | + |
| 29 | +ARTIFACT="$(echo $COORDINATE | tr ':' '\n' | head -n2 | tail -n1)" |
| 30 | +WORKSPACE="$(jgo --repository scijava.public=https://maven.scijava.org/content/groups/public --resolve-only $COORDINATE)" |
| 31 | +MAIN_JAR="$(ls ${WORKSPACE}/*jar -1 | grep -E "${ARTIFACT}-[0-9]+")" |
| 32 | +ALL_JARS="$(ls ${WORKSPACE}/*jar -1 | tr '\n' ',' | sed 's/,$//')" |
| 33 | + |
| 34 | +# echo "$MEMORY_PER_NODE / $N_EXECUTORS_PER_NODE" |
| 35 | +# echo "$N_NODES * $N_EXECUTORS_PER_NODE" |
| 36 | +export MEMORY_PER_EXECUTOR="$(($MEMORY_PER_NODE / $N_EXECUTORS_PER_NODE))" |
| 37 | +export N_EXECUTORS="$(($N_NODES * $N_EXECUTORS_PER_NODE))" |
| 38 | +export PARALLELISM="$(($N_EXECUTORS * $N_CORES_PER_EXECUTOR * 3))" |
| 39 | + |
| 40 | +SUBMIT_ARGS="${SUBMIT_ARGS} --verbose" |
| 41 | +SUBMIT_ARGS="${SUBMIT_ARGS} --conf spark.default.parallelism=$PARALLELISM" |
| 42 | +SUBMIT_ARGS="${SUBMIT_ARGS} --conf spark.executor.instances=$N_EXECUTORS_PER_NODE" |
| 43 | +SUBMIT_ARGS="${SUBMIT_ARGS} --conf spark.executor.cores=$N_CORES_PER_EXECUTOR" |
| 44 | +SUBMIT_ARGS="${SUBMIT_ARGS} --conf spark.executor.memory=${MEMORY_PER_EXECUTOR}g" |
| 45 | +SUBMIT_ARGS="${SUBMIT_ARGS} ${SPARK_OPTIONS}" |
| 46 | +SUBMIT_ARGS="${SUBMIT_ARGS} --jars $ALL_JARS" |
| 47 | +SUBMIT_ARGS="${SUBMIT_ARGS} --class $CLASS" |
| 48 | +SUBMIT_ARGS="${SUBMIT_ARGS} ${MAIN_JAR}" |
| 49 | +SUBMIT_ARGS="${SUBMIT_ARGS} ${ARGV}" |
| 50 | + |
| 51 | +LOG_FILE="${HOME}/.sparklogs/${CLASS}.o%J" |
| 52 | + |
| 53 | +"${SPARK_JANELIA}" \ |
| 54 | + --nnodes="${N_NODES}" \ |
| 55 | + --no_check \ |
| 56 | + --driveronspark \ |
| 57 | + --silentlaunch \ |
| 58 | + --minworkers="${MIN_WORKERS}" \ |
| 59 | + --hard_runtime=${RUNTIME} \ |
| 60 | + --submitargs="${SUBMIT_ARGS}" \ |
| 61 | + --driveroutfile=${LOG_FILE} \ |
| 62 | + lsd |
0 commit comments