From cb89755fd99ca52a926b89fedbc24fbca43dc0ad Mon Sep 17 00:00:00 2001 From: Mike Ihbe Date: Mon, 3 Oct 2016 14:33:19 -0700 Subject: [PATCH 1/3] [SPARK-11653][Deploy] Add a SPARK_NO_DAEMONIZE env var for running foreground operations. --- sbin/spark-daemon.sh | 54 ++++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/sbin/spark-daemon.sh b/sbin/spark-daemon.sh index 59823571124f..28c0f181bf26 100755 --- a/sbin/spark-daemon.sh +++ b/sbin/spark-daemon.sh @@ -27,6 +27,7 @@ # SPARK_PID_DIR The pid files are stored. /tmp by default. # SPARK_IDENT_STRING A string representing this instance of spark. $USER by default # SPARK_NICENESS The scheduling priority for daemons. Defaults to 0. +# SPARK_NO_DAEMONIZE If set, will run the proposed command in the foreground. It will not output a PID file. ## usage="Usage: spark-daemon.sh [--config ] (start|stop|submit|status) " @@ -122,6 +123,35 @@ if [ "$SPARK_NICENESS" = "" ]; then export SPARK_NICENESS=0 fi +execute_command() { + command="$@" + if [ "$SPARK_NO_DAEMONIZE" != "" ]; then + eval $command + else + eval "nohup $command >> \"$log\" 2>&1 < /dev/null &" + newpid="$!" + + echo "$newpid" > "$pid" + + #Poll for up to 5 seconds for the java process to start + for i in {1..10} + do + if [[ $(ps -p "$newpid" -o comm=) =~ "java" ]]; then + break + fi + sleep 0.5 + done + + sleep 2 + # Check if the process has died; in that case we'll tail the log so the user can see + if [[ ! $(ps -p "$newpid" -o comm=) =~ "java" ]]; then + echo "failed to launch $command:" + tail -2 "$log" | sed 's/^/ /' + echo "full log in $log" + fi + fi +} + run_command() { mode="$1" shift @@ -146,13 +176,11 @@ run_command() { case "$mode" in (class) - nohup nice -n "$SPARK_NICENESS" "${SPARK_HOME}"/bin/spark-class $command "$@" >> "$log" 2>&1 < /dev/null & - newpid="$!" + execute_command "nice -n \"$SPARK_NICENESS\" \"${SPARK_HOME}/bin/spark-class\" $command $@" ;; (submit) - nohup nice -n "$SPARK_NICENESS" "${SPARK_HOME}"/bin/spark-submit --class $command "$@" >> "$log" 2>&1 < /dev/null & - newpid="$!" + execute_command "nice -n \"$SPARK_NICENESS\" \"${SPARK_HOME}/bin/spark-submit\" --class $command $@" ;; (*) @@ -161,24 +189,6 @@ run_command() { ;; esac - echo "$newpid" > "$pid" - - #Poll for up to 5 seconds for the java process to start - for i in {1..10} - do - if [[ $(ps -p "$newpid" -o comm=) =~ "java" ]]; then - break - fi - sleep 0.5 - done - - sleep 2 - # Check if the process has died; in that case we'll tail the log so the user can see - if [[ ! $(ps -p "$newpid" -o comm=) =~ "java" ]]; then - echo "failed to launch $command:" - tail -2 "$log" | sed 's/^/ /' - echo "full log in $log" - fi } case $option in From 42c9874ac35c124d6cfd93c272dda6e28b4ce9d3 Mon Sep 17 00:00:00 2001 From: Mike Ihbe Date: Tue, 11 Oct 2016 16:29:45 -0700 Subject: [PATCH 2/3] Updates based on PR feedback: https://github.com/apache/spark/pull/15338#pullrequestreview-3743395. Remove evals and make variables local. --- sbin/spark-daemon.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sbin/spark-daemon.sh b/sbin/spark-daemon.sh index 28c0f181bf26..b4fd9b14c312 100755 --- a/sbin/spark-daemon.sh +++ b/sbin/spark-daemon.sh @@ -124,11 +124,9 @@ if [ "$SPARK_NICENESS" = "" ]; then fi execute_command() { - command="$@" - if [ "$SPARK_NO_DAEMONIZE" != "" ]; then - eval $command - else - eval "nohup $command >> \"$log\" 2>&1 < /dev/null &" + local command="$@" + if [ -z ${SPARK_NO_DAEMONIZE+set} ]; then + nohup -- $command >> $log 2>&1 < /dev/null & newpid="$!" echo "$newpid" > "$pid" @@ -149,6 +147,8 @@ execute_command() { tail -2 "$log" | sed 's/^/ /' echo "full log in $log" fi + else + $command fi } @@ -176,11 +176,11 @@ run_command() { case "$mode" in (class) - execute_command "nice -n \"$SPARK_NICENESS\" \"${SPARK_HOME}/bin/spark-class\" $command $@" + execute_command nice -n $SPARK_NICENESS ${SPARK_HOME}/bin/spark-class $command $@ ;; (submit) - execute_command "nice -n \"$SPARK_NICENESS\" \"${SPARK_HOME}/bin/spark-submit\" --class $command $@" + execute_command nice -n $SPARK_NICENESS bash ${SPARK_HOME}/bin/spark-submit --class $command $@ ;; (*) From 7010b28c7e7ba73a26cdee4bb6e0698152b27c0d Mon Sep 17 00:00:00 2001 From: Mike Ihbe Date: Fri, 14 Oct 2016 10:42:45 -0700 Subject: [PATCH 3/3] Re-add quotes to env vars. Add the new SPARK_NO_DAEMONIZE flag to the spark-env.sh.template file --- conf/spark-env.sh.template | 1 + sbin/spark-daemon.sh | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/conf/spark-env.sh.template b/conf/spark-env.sh.template index c750c72d1988..5c1e876ef9af 100755 --- a/conf/spark-env.sh.template +++ b/conf/spark-env.sh.template @@ -63,3 +63,4 @@ # - SPARK_PID_DIR Where the pid file is stored. (Default: /tmp) # - SPARK_IDENT_STRING A string representing this instance of spark. (Default: $USER) # - SPARK_NICENESS The scheduling priority for daemons. (Default: 0) +# - SPARK_NO_DAEMONIZE Run the proposed command in the foreground. It will not output a PID file. diff --git a/sbin/spark-daemon.sh b/sbin/spark-daemon.sh index b4fd9b14c312..061019a55e99 100755 --- a/sbin/spark-daemon.sh +++ b/sbin/spark-daemon.sh @@ -131,7 +131,7 @@ execute_command() { echo "$newpid" > "$pid" - #Poll for up to 5 seconds for the java process to start + # Poll for up to 5 seconds for the java process to start for i in {1..10} do if [[ $(ps -p "$newpid" -o comm=) =~ "java" ]]; then @@ -176,11 +176,11 @@ run_command() { case "$mode" in (class) - execute_command nice -n $SPARK_NICENESS ${SPARK_HOME}/bin/spark-class $command $@ + execute_command nice -n "$SPARK_NICENESS" "${SPARK_HOME}"/bin/spark-class $command $@ ;; (submit) - execute_command nice -n $SPARK_NICENESS bash ${SPARK_HOME}/bin/spark-submit --class $command $@ + execute_command nice -n "$SPARK_NICENESS" bash "${SPARK_HOME}"/bin/spark-submit --class $command $@ ;; (*)