From 7ac7a713e014e7656713afb99495538856b6545e Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Mon, 6 Dec 2021 14:09:22 +0800 Subject: [PATCH 1/4] [SPARK-37555][SQL] spark-sql should pass last unclosed comment to backend --- .../hive/thriftserver/SparkSQLCLIDriver.scala | 4 +--- .../spark/sql/hive/thriftserver/CliSuite.scala | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index 6b5b412fdaaf1..ed517cc6fbbd7 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -613,9 +613,7 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging { isStatement = statementInProgress(index) } - if (isStatement) { - ret.add(line.substring(beginIndex)) - } + ret.add(line.substring(beginIndex)) ret } } diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index b404d77881137..21db441f5520e 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -620,4 +620,22 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging { |""".stripMargin -> "SELECT 1" ) } + + test("SPARK-37555: spark-sql should pass last unclosed comment to backend") { + runCliWithin(2.minute)( + // Unclosed comment + """ + |/* SELECT /*+ HINT() 4; */; + |""".stripMargin -> "Error in query", + // Unclosed comment with query. + """ + |/* SELECT /*+ HINT() 4; */ + |SELECT 1; + |""".stripMargin -> "1", + // Whole comment. + """ + |/* SELECT /*+ HINT() */ 4; */; + |""".stripMargin -> "" + ) + } } From e874bbd009abd8d1602f6d3f13c791d6a56450cb Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Mon, 6 Dec 2021 14:36:37 +0800 Subject: [PATCH 2/4] Update CliSuite.scala --- .../spark/sql/hive/thriftserver/CliSuite.scala | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index 21db441f5520e..6d2fca543c165 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -623,19 +623,14 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging { test("SPARK-37555: spark-sql should pass last unclosed comment to backend") { runCliWithin(2.minute)( - // Unclosed comment - """ - |/* SELECT /*+ HINT() 4; */; - |""".stripMargin -> "Error in query", + // Only unclosed comment. + "/* SELECT /*+ HINT() 4; */;".stripMargin -> "Error in query", + // Unclosed nested bracketed comment. + "/* SELECT /*+ HINT() 4; */ SELECT 1;".stripMargin -> "1", // Unclosed comment with query. - """ - |/* SELECT /*+ HINT() 4; */ - |SELECT 1; - |""".stripMargin -> "1", + "/* Here is a unclosed bracketed comment SELECT 1;"-> "Unclosed bracketed comment", // Whole comment. - """ - |/* SELECT /*+ HINT() */ 4; */; - |""".stripMargin -> "" + "/* SELECT /*+ HINT() */ 4; */;".stripMargin -> "" ) } } From c67dcaf041f6dd1bfc44642490c939ad2df41c00 Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Mon, 6 Dec 2021 14:57:23 +0800 Subject: [PATCH 3/4] Update CliSuite.scala --- .../scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index 6d2fca543c165..11e6578260190 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -624,7 +624,7 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging { test("SPARK-37555: spark-sql should pass last unclosed comment to backend") { runCliWithin(2.minute)( // Only unclosed comment. - "/* SELECT /*+ HINT() 4; */;".stripMargin -> "Error in query", + "/* SELECT /*+ HINT() 4; */;".stripMargin -> "mismatched input ';'", // Unclosed nested bracketed comment. "/* SELECT /*+ HINT() 4; */ SELECT 1;".stripMargin -> "1", // Unclosed comment with query. From e45b12dd2bf76d1e859af176a553c12260e02fed Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Mon, 6 Dec 2021 17:36:17 +0800 Subject: [PATCH 4/4] Update SparkSQLCLIDriver.scala --- .../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index ed517cc6fbbd7..3c4c4dd0da75c 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -613,7 +613,9 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging { isStatement = statementInProgress(index) } - ret.add(line.substring(beginIndex)) + if (beginIndex < line.length()) { + ret.add(line.substring(beginIndex)) + } ret } }