From 08579af6d2aa2239da5b6532094301a4c4afe86b Mon Sep 17 00:00:00 2001 From: Chen Chao Date: Tue, 3 Jun 2014 23:02:26 +0800 Subject: [PATCH 1/4] connect to cluster automatically --- bin/spark-shell | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/bin/spark-shell b/bin/spark-shell index 850e9507ec38f..b143efecb6c6e 100755 --- a/bin/spark-shell +++ b/bin/spark-shell @@ -49,8 +49,21 @@ function main(){ $FWDIR/bin/spark-submit spark-shell "$@" --class org.apache.spark.repl.Main stty icanon echo > /dev/null 2>&1 else + . $FWDIR/bin/load-spark-env.sh export SPARK_SUBMIT_OPTS - $FWDIR/bin/spark-submit spark-shell "$@" --class org.apache.spark.repl.Main + MASTER_URL="" + DEFAULT_SPARK_MASTER_PORT=7077 + if echo "$@" | grep -v -e "--master">/dev/null ; then + if [ "x" != "x$SPARK_MASTER_IP" ]; then + if [ "y" != "y$SPARK_MASTER_PORT" ]; then + SPARK_MASTER_PORT="${SPARK_MASTER_PORT}" + else + SPARK_MASTER_PORT=$DEFAULT_SPARK_MASTER_PORT + fi + MASTER_URL="--master spark://${SPARK_MASTER_IP}:${SPARK_MASTER_PORT}" + fi + fi + $FWDIR/bin/spark-submit spark-shell $MASTER_URL "$@" --class org.apache.spark.repl.Main fi } From 5f978a3b7ea1124f41e73f721df76a4cc7c3bd58 Mon Sep 17 00:00:00 2001 From: Chen Chao Date: Wed, 4 Jun 2014 10:21:10 +0800 Subject: [PATCH 2/4] rename MASTER_URL to MASTER_OPTION --- bin/spark-shell | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/spark-shell b/bin/spark-shell index b143efecb6c6e..5077ef5f90dcd 100755 --- a/bin/spark-shell +++ b/bin/spark-shell @@ -51,7 +51,7 @@ function main(){ else . $FWDIR/bin/load-spark-env.sh export SPARK_SUBMIT_OPTS - MASTER_URL="" + MASTER_OPTION="" DEFAULT_SPARK_MASTER_PORT=7077 if echo "$@" | grep -v -e "--master">/dev/null ; then if [ "x" != "x$SPARK_MASTER_IP" ]; then @@ -60,7 +60,7 @@ function main(){ else SPARK_MASTER_PORT=$DEFAULT_SPARK_MASTER_PORT fi - MASTER_URL="--master spark://${SPARK_MASTER_IP}:${SPARK_MASTER_PORT}" + MASTER_OPTION="--master spark://${SPARK_MASTER_IP}:${SPARK_MASTER_PORT}" fi fi $FWDIR/bin/spark-submit spark-shell $MASTER_URL "$@" --class org.apache.spark.repl.Main From 289175a5d6172bb6ceddaab6ec4d5435c66e1e31 Mon Sep 17 00:00:00 2001 From: CrazyJvm Date: Wed, 4 Jun 2014 15:35:22 +0800 Subject: [PATCH 3/4] modify in SparkSubmitArguments rather than shell. Note than we should read "SPARK_MASTER_IP". --- bin/spark-shell | 15 +-------------- .../spark/deploy/SparkSubmitArguments.scala | 11 ++++++++++- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/bin/spark-shell b/bin/spark-shell index 5077ef5f90dcd..850e9507ec38f 100755 --- a/bin/spark-shell +++ b/bin/spark-shell @@ -49,21 +49,8 @@ function main(){ $FWDIR/bin/spark-submit spark-shell "$@" --class org.apache.spark.repl.Main stty icanon echo > /dev/null 2>&1 else - . $FWDIR/bin/load-spark-env.sh export SPARK_SUBMIT_OPTS - MASTER_OPTION="" - DEFAULT_SPARK_MASTER_PORT=7077 - if echo "$@" | grep -v -e "--master">/dev/null ; then - if [ "x" != "x$SPARK_MASTER_IP" ]; then - if [ "y" != "y$SPARK_MASTER_PORT" ]; then - SPARK_MASTER_PORT="${SPARK_MASTER_PORT}" - else - SPARK_MASTER_PORT=$DEFAULT_SPARK_MASTER_PORT - fi - MASTER_OPTION="--master spark://${SPARK_MASTER_IP}:${SPARK_MASTER_PORT}" - fi - fi - $FWDIR/bin/spark-submit spark-shell $MASTER_URL "$@" --class org.apache.spark.repl.Main + $FWDIR/bin/spark-submit spark-shell "$@" --class org.apache.spark.repl.Main fi } diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index 153eee3bc5889..726361f770d3e 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -124,7 +124,16 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) { } // Global defaults. These should be keep to minimum to avoid confusing behavior. - master = Option(master).getOrElse("local[*]") + master = Option(master).getOrElse { + if (System.getenv("SPARK_MASTER_IP") != null) { + val host = System.getenv("SPARK_MASTER_IP") + val port = Option(System.getenv("SPARK_MASTER_PORT")).getOrElse("7077").toInt + s"spark://$host:$port" + } + else { + "local[*]" + } + } // Set name from main class if not given name = Option(name).orElse(Option(mainClass)).orNull From 2b828d2475d9ebeedb9eff1f42f34db645c57c39 Mon Sep 17 00:00:00 2001 From: Chen Chao Date: Wed, 30 Jul 2014 09:09:41 +0800 Subject: [PATCH 4/4] restore to original code --- .../apache/spark/deploy/SparkSubmitArguments.scala | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index 726361f770d3e..153eee3bc5889 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -124,16 +124,7 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) { } // Global defaults. These should be keep to minimum to avoid confusing behavior. - master = Option(master).getOrElse { - if (System.getenv("SPARK_MASTER_IP") != null) { - val host = System.getenv("SPARK_MASTER_IP") - val port = Option(System.getenv("SPARK_MASTER_PORT")).getOrElse("7077").toInt - s"spark://$host:$port" - } - else { - "local[*]" - } - } + master = Option(master).getOrElse("local[*]") // Set name from main class if not given name = Option(name).orElse(Option(mainClass)).orNull