From b453a2e5b6b3f3f25937a97cfa59ca1c42c032ce Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 29 Oct 2019 14:33:42 +0800 Subject: [PATCH 1/3] [SPARK-29629][SQL] Support typed integer literal expression --- .../sql/catalyst/parser/AstBuilder.scala | 12 ++++++- .../resources/sql-tests/inputs/datetime.sql | 2 ++ .../resources/sql-tests/inputs/literals.sql | 5 +++ .../sql-tests/results/datetime.sql.out | 18 ++++++++++- .../sql-tests/results/literals.sql.out | 32 ++++++++++++++++++- 5 files changed, 66 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 3666eaeaa3d2a..ea07878b8e511 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1769,7 +1769,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging * {{{ * [TYPE] '[VALUE]' * }}} - * Currently Date, Timestamp, Interval and Binary typed literals are supported. + * Currently Date, Timestamp, Interval, Binary and INTEGER typed literals are supported. */ override def visitTypeConstructor(ctx: TypeConstructorContext): Literal = withOrigin(ctx) { val value = string(ctx.STRING) @@ -1799,6 +1799,16 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging case "X" => val padding = if (value.length % 2 != 0) "0" else "" Literal(DatatypeConverter.parseHexBinary(padding + value)) + case "INTEGER" => + val i = try { + Integer.parseInt(value) + } catch { + case e: NumberFormatException => + val ex = new ParseException(s"Cannot parse the Int value: $value, $e", ctx) + ex.setStackTrace(e.getStackTrace) + throw ex + } + Literal(i) case other => throw new ParseException(s"Literals of type '$other' are currently not supported.", ctx) } diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql index 0e22af1fbdf29..a4e5bdc8145d5 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql @@ -36,3 +36,5 @@ select date '2001-10-01' - 7; select date '2001-10-01' - date '2001-09-28'; select date'2020-01-01' - timestamp'2019-10-06 10:11:12.345678'; select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01'; +select date '2001-09-28' + integer '7'; +select date '2001-09-28' - integer '7'; diff --git a/sql/core/src/test/resources/sql-tests/inputs/literals.sql b/sql/core/src/test/resources/sql-tests/inputs/literals.sql index 0f95f85237828..389b9cc53e02e 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/literals.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/literals.sql @@ -125,3 +125,8 @@ select map(1, interval 1 day, 2, interval 3 week); -- typed interval expression select interval 'interval 3 year 1 hour'; select interval '3 year 1 hour'; + +-- typed integer expression +select integer '7'; +select integer'7'; +select integer '2147483648'; diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out index 0f4036cad6125..fc1fcb7b1ab93 100644 --- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 17 +-- Number of queries: 19 -- !query 0 @@ -145,3 +145,19 @@ select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01' struct -- !query 16 output interval -12 weeks -2 days -14 hours -48 minutes -47 seconds -654 milliseconds -322 microseconds + + +-- !query 17 +select date '2001-09-28' + integer '7' +-- !query 17 schema +struct +-- !query 17 output +2001-10-05 + + +-- !query 18 +select date '2001-09-28' - integer '7' +-- !query 18 schema +struct +-- !query 18 output +2001-09-21 diff --git a/sql/core/src/test/resources/sql-tests/results/literals.sql.out b/sql/core/src/test/resources/sql-tests/results/literals.sql.out index fd6e51b2385de..115287821bf4f 100644 --- a/sql/core/src/test/resources/sql-tests/results/literals.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/literals.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 59 +-- Number of queries: 62 -- !query 0 @@ -542,3 +542,33 @@ select interval '3 year 1 hour' struct -- !query 58 output interval 3 years 1 hours + + +-- !query 59 +select integer '7' +-- !query 59 schema +struct<7:int> +-- !query 59 output +7 + + +-- !query 60 +select integer'7' +-- !query 60 schema +struct<7:int> +-- !query 60 output +7 + + +-- !query 61 +select integer '2147483648' +-- !query 61 schema +struct<> +-- !query 61 output +org.apache.spark.sql.catalyst.parser.ParseException + +Cannot parse the Int value: 2147483648, java.lang.NumberFormatException: For input string: "2147483648"(line 1, pos 7) + +== SQL == +select integer '2147483648' +-------^^^ From cdce7956889d7f6fce51cdf37d49ca158300106d Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 29 Oct 2019 15:46:52 +0800 Subject: [PATCH 2/3] avoid pattern matching --- .../scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index ea07878b8e511..4b5eede658024 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1808,7 +1808,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging ex.setStackTrace(e.getStackTrace) throw ex } - Literal(i) + Literal(i, IntegerType) case other => throw new ParseException(s"Literals of type '$other' are currently not supported.", ctx) } From 1960b291519d9ebcdc4356811d7d283ffa3ca3c4 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 29 Oct 2019 16:28:41 +0800 Subject: [PATCH 3/3] resovle review req --- .../spark/sql/catalyst/parser/AstBuilder.scala | 2 +- .../resources/sql-tests/inputs/datetime.sql | 2 -- .../sql-tests/results/datetime.sql.out | 18 +----------------- 3 files changed, 2 insertions(+), 20 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 4b5eede658024..c0f448942aae1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1801,7 +1801,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging Literal(DatatypeConverter.parseHexBinary(padding + value)) case "INTEGER" => val i = try { - Integer.parseInt(value) + value.toInt } catch { case e: NumberFormatException => val ex = new ParseException(s"Cannot parse the Int value: $value, $e", ctx) diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql index a4e5bdc8145d5..0e22af1fbdf29 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql @@ -36,5 +36,3 @@ select date '2001-10-01' - 7; select date '2001-10-01' - date '2001-09-28'; select date'2020-01-01' - timestamp'2019-10-06 10:11:12.345678'; select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01'; -select date '2001-09-28' + integer '7'; -select date '2001-09-28' - integer '7'; diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out index fc1fcb7b1ab93..0f4036cad6125 100644 --- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 19 +-- Number of queries: 17 -- !query 0 @@ -145,19 +145,3 @@ select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01' struct -- !query 16 output interval -12 weeks -2 days -14 hours -48 minutes -47 seconds -654 milliseconds -322 microseconds - - --- !query 17 -select date '2001-09-28' + integer '7' --- !query 17 schema -struct --- !query 17 output -2001-10-05 - - --- !query 18 -select date '2001-09-28' - integer '7' --- !query 18 schema -struct --- !query 18 output -2001-09-21