diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh index b54ea496c633b..16598bda87339 100755 --- a/dev/make-distribution.sh +++ b/dev/make-distribution.sh @@ -322,9 +322,11 @@ if [ "$MAKE_TGZ" == "true" ]; then rm -rf "$TARDIR" cp -r "$DISTDIR" "$TARDIR" # Set the Spark Connect system variable in these scripts to enable it by default. + awk 'NR==1{print; print "export SPARK_CONNECT_BEELINE=${SPARK_CONNECT_BEELINE:-1}"; next} {print}' "$TARDIR/bin/beeline" > tmp && cat tmp > "$TARDIR/bin/beeline" awk 'NR==1{print; print "export SPARK_CONNECT_MODE=${SPARK_CONNECT_MODE:-1}"; next} {print}' "$TARDIR/bin/pyspark" > tmp && cat tmp > "$TARDIR/bin/pyspark" awk 'NR==1{print; print "export SPARK_CONNECT_MODE=${SPARK_CONNECT_MODE:-1}"; next} {print}' "$TARDIR/bin/spark-shell" > tmp && cat tmp > "$TARDIR/bin/spark-shell" awk 'NR==1{print; print "export SPARK_CONNECT_MODE=${SPARK_CONNECT_MODE:-1}"; next} {print}' "$TARDIR/bin/spark-submit" > tmp && cat tmp > "$TARDIR/bin/spark-submit" + awk 'NR==1{print; print "if [%SPARK_CONNECT_BEELINE%] == [] set SPARK_CONNECT_BEELINE=1"; next} {print}' "$TARDIR/bin/beeline.cmd" > tmp && cat tmp > "$TARDIR/bin/beeline.cmd" awk 'NR==1{print; print "if [%SPARK_CONNECT_MODE%] == [] set SPARK_CONNECT_MODE=1"; next} {print}' "$TARDIR/bin/pyspark2.cmd" > tmp && cat tmp > "$TARDIR/bin/pyspark2.cmd" awk 'NR==1{print; print "if [%SPARK_CONNECT_MODE%] == [] set SPARK_CONNECT_MODE=1"; next} {print}' "$TARDIR/bin/spark-shell2.cmd" > tmp && cat tmp > "$TARDIR/bin/spark-shell2.cmd" awk 'NR==1{print; print "if [%SPARK_CONNECT_MODE%] == [] set SPARK_CONNECT_MODE=1"; next} {print}' "$TARDIR/bin/spark-submit2.cmd" > tmp && cat tmp > "$TARDIR/bin/spark-submit2.cmd" diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java index caa2d2b5854e1..0214b11023814 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java @@ -66,6 +66,8 @@ abstract class AbstractCommandBuilder { */ protected boolean isRemote = System.getenv().containsKey("SPARK_REMOTE"); + protected boolean isBeeLine = false; + AbstractCommandBuilder() { this.appArgs = new ArrayList<>(); this.childEnv = new HashMap<>(); @@ -195,6 +197,10 @@ List buildClassPath(String appClassPath) throws IOException { if (isRemote && "1".equals(getenv("SPARK_SCALA_SHELL")) && project.equals("sql/core")) { continue; } + if (isBeeLine && "1".equals(getenv("SPARK_CONNECT_BEELINE")) && + project.equals("sql/core")) { + continue; + } // SPARK-49534: The assumption here is that if `spark-hive_xxx.jar` is not in the // classpath, then the `-Phive` profile was not used during package, and therefore // the Hive-related jars should also not be in the classpath. To avoid failure in @@ -241,13 +247,13 @@ List buildClassPath(String appClassPath) throws IOException { } } - if (isRemote) { + if (isRemote || (isBeeLine && "1".equals(getenv("SPARK_CONNECT_BEELINE")))) { for (File f: new File(jarsDir).listFiles()) { - // Exclude Spark Classic SQL and Spark Connect server jars - // if we're in Spark Connect Shell. Also exclude Spark SQL API and - // Spark Connect Common which Spark Connect client shades. - // Then, we add the Spark Connect shell and its dependencies in connect-repl - // See also SPARK-48936. + // Exclude Spark Classic SQL and Spark Connect server jars if we're in + // Spark Connect Shell or BeeLine with Connect JDBC driver. Also exclude + // Spark SQL API and Spark Connect Common which Spark Connect client shades. + // Then, we add the Spark Connect shell and its dependencies in connect-repl. + // See also SPARK-48936, SPARK-54002. if (f.isDirectory() && f.getName().equals("connect-repl")) { addToClassPath(cp, join(File.separator, f.toString(), "*")); } else if ( diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java index d7d10d486e3f5..2dd0bb13dfd3b 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java @@ -38,6 +38,9 @@ class SparkClassCommandBuilder extends AbstractCommandBuilder { SparkClassCommandBuilder(String className, List classArgs) { this.className = className; this.classArgs = classArgs; + if ("org.apache.hive.beeline.BeeLine".equals(className)) { + this.isBeeLine = true; + } } @Override