From be4736bcff503380f254f1e7f8482d00182e236a Mon Sep 17 00:00:00 2001 From: Cheng Lian Date: Thu, 14 Aug 2014 21:28:44 +0800 Subject: [PATCH 1/4] Report better error message when running JDBC/CLI without hive-thriftserver profile enabled --- bin/spark-sql | 12 +++++++++++- .../org/apache/spark/deploy/SparkSubmit.scala | 14 +++++++++++++- sbin/start-thriftserver.sh | 12 +++++++++++- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/bin/spark-sql b/bin/spark-sql index 564f1f419060..ce5b6d91c710 100755 --- a/bin/spark-sql +++ b/bin/spark-sql @@ -24,6 +24,7 @@ set -o posix CLASS="org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver" +CLASS_NOT_FOUND_EXIT_STATUS=1 # Figure out where Spark is installed FWDIR="$(cd `dirname $0`/..; pwd)" @@ -91,4 +92,13 @@ while (($#)); do esac done -exec "$FWDIR"/bin/spark-submit --class $CLASS "${SUBMISSION_ARGS[@]}" spark-internal "${CLI_ARGS[@]}" +eval "$FWDIR"/bin/spark-submit --class $CLASS ${SUBMISSION_ARGS[*]} spark-internal ${CLI_ARGS[*]} +exit_status=$? + +if [[ exit_status -eq CLASS_NOT_FOUND_EXIT_STATUS ]]; then + echo + echo "Failed to load Spark SQL CLI main class $CLASS." + echo "You need to build Spark with -Phive." +fi + +exit $exit_status diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index f8cdbc3c392b..550ee7253890 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -54,6 +54,8 @@ object SparkSubmit { private val SPARK_SHELL = "spark-shell" private val PYSPARK_SHELL = "pyspark-shell" + private val CLASS_NOT_FOUND_EXIT_STATUS = 1 + // Exposed for testing private[spark] var exitFn: () => Unit = () => System.exit(-1) private[spark] var printStream: PrintStream = System.err @@ -311,8 +313,18 @@ object SparkSubmit { System.setProperty(key, value) } - val mainClass = Class.forName(childMainClass, true, loader) + var mainClass: Class[_] = null + + try { + mainClass = Class.forName(childMainClass, true, loader) + } catch { + case e: ClassNotFoundException => + e.printStackTrace(printStream) + System.exit(CLASS_NOT_FOUND_EXIT_STATUS) + } + val mainMethod = mainClass.getMethod("main", new Array[String](0).getClass) + try { mainMethod.invoke(null, childArgs.toArray) } catch { diff --git a/sbin/start-thriftserver.sh b/sbin/start-thriftserver.sh index 2c4452473ccb..c0cc4c885a80 100755 --- a/sbin/start-thriftserver.sh +++ b/sbin/start-thriftserver.sh @@ -27,6 +27,7 @@ set -o posix FWDIR="$(cd `dirname $0`/..; pwd)" CLASS="org.apache.spark.sql.hive.thriftserver.HiveThriftServer2" +CLASS_NOT_FOUND_EXIT_STATUS=1 function usage { echo "Usage: ./sbin/start-thriftserver [options] [thrift server options]" @@ -75,4 +76,13 @@ while (($#)); do esac done -exec "$FWDIR"/bin/spark-submit --class $CLASS "${SUBMISSION_ARGS[@]}" spark-internal "${THRIFT_SERVER_ARGS[@]}" +eval "$FWDIR"/bin/spark-submit --class $CLASS ${SUBMISSION_ARGS[*]} spark-internal ${THRIFT_SERVER_ARGS[*]} +exit_status=$? + +if [[ exit_status -eq CLASS_NOT_FOUND_EXIT_STATUS ]]; then + echo + echo "Failed to load Hive Thrift server main class $CLASS." + echo "You need to build Spark with -Phive." +fi + +exit $exit_status From 9c894d34327e608461bbe9be5ae5fcbb4ac6dd44 Mon Sep 17 00:00:00 2001 From: Cheng Lian Date: Wed, 20 Aug 2014 23:15:47 -0700 Subject: [PATCH 2/4] Fixed bin/spark-sql -S option typo --- bin/spark-sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/spark-sql b/bin/spark-sql index ce5b6d91c710..34bf22fff7a7 100755 --- a/bin/spark-sql +++ b/bin/spark-sql @@ -76,7 +76,7 @@ while (($#)); do CLI_ARGS+=("$1"); shift ;; - -s | --silent) + -S | --silent) CLI_ARGS+=("$1"); shift ;; From a89e66df0085ddac4455654567ee95bc2a4e879a Mon Sep 17 00:00:00 2001 From: Cheng Lian Date: Wed, 20 Aug 2014 23:40:20 -0700 Subject: [PATCH 3/4] Fixed command line options quotation in scripts --- bin/spark-sql | 2 +- sbin/start-thriftserver.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/spark-sql b/bin/spark-sql index 34bf22fff7a7..2e5d8e614175 100755 --- a/bin/spark-sql +++ b/bin/spark-sql @@ -92,7 +92,7 @@ while (($#)); do esac done -eval "$FWDIR"/bin/spark-submit --class $CLASS ${SUBMISSION_ARGS[*]} spark-internal ${CLI_ARGS[*]} +"$FWDIR"/bin/spark-submit --class $CLASS "${SUBMISSION_ARGS[@]}" spark-internal "${CLI_ARGS[@]}" exit_status=$? if [[ exit_status -eq CLASS_NOT_FOUND_EXIT_STATUS ]]; then diff --git a/sbin/start-thriftserver.sh b/sbin/start-thriftserver.sh index c0cc4c885a80..fa50730a3949 100755 --- a/sbin/start-thriftserver.sh +++ b/sbin/start-thriftserver.sh @@ -76,7 +76,7 @@ while (($#)); do esac done -eval "$FWDIR"/bin/spark-submit --class $CLASS ${SUBMISSION_ARGS[*]} spark-internal ${THRIFT_SERVER_ARGS[*]} +"$FWDIR"/bin/spark-submit --class $CLASS "${SUBMISSION_ARGS[@]}" spark-internal "${THRIFT_SERVER_ARGS[@]}" exit_status=$? if [[ exit_status -eq CLASS_NOT_FOUND_EXIT_STATUS ]]; then From 81b43a897b241eca16e75668557ecd81cb25c41a Mon Sep 17 00:00:00 2001 From: Cheng Lian Date: Thu, 21 Aug 2014 00:18:01 -0700 Subject: [PATCH 4/4] Shorten timeout to more reasonable value --- .../spark/sql/hive/thriftserver/HiveThriftServer2Suite.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suite.scala index 326b0a7275b3..0807dcf1e291 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suite.scala @@ -72,11 +72,10 @@ class HiveThriftServer2Suite extends FunSuite with BeforeAndAfterAll with TestUt """.stripMargin.split("\\s+") val pb = new ProcessBuilder(command ++ args: _*) - val environment = pb.environment() process = pb.start() inputReader = new BufferedReader(new InputStreamReader(process.getInputStream)) errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream)) - waitForOutput(inputReader, "ThriftBinaryCLIService listening on", 300000) + waitForOutput(inputReader, "ThriftBinaryCLIService listening on", 30000) // Spawn a thread to read the output from the forked process. // Note that this is necessary since in some configurations, log4j could be blocked