From b9a25e88a71ba3d288fe48dddabaf65adabfd121 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Fri, 15 Nov 2019 11:40:56 +0800 Subject: [PATCH 1/5] [SPARK-29587][SQL] Support SQL Standard type real as float(4) numeric as decimal --- .../sql/catalyst/parser/AstBuilder.scala | 8 ++-- .../test/resources/sql-tests/inputs/cast.sql | 7 ++++ .../sql-tests/inputs/show-create-table.sql | 5 +++ .../resources/sql-tests/results/cast.sql.out | 42 ++++++++++++++++++- .../results/show-create-table.sql.out | 27 +++++++++++- 5 files changed, 83 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index c623b5c4d36a..cfd4bfeed752 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -2154,7 +2154,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging case ("smallint" | "short", Nil) => ShortType case ("int" | "integer", Nil) => IntegerType case ("bigint" | "long", Nil) => LongType - case ("float", Nil) => FloatType + case ("float" | "real", Nil) => FloatType case ("double", Nil) => DoubleType case ("date", Nil) => DateType case ("timestamp", Nil) => TimestampType @@ -2162,9 +2162,9 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging case ("char", length :: Nil) => CharType(length.getText.toInt) case ("varchar", length :: Nil) => VarcharType(length.getText.toInt) case ("binary", Nil) => BinaryType - case ("decimal", Nil) => DecimalType.USER_DEFAULT - case ("decimal", precision :: Nil) => DecimalType(precision.getText.toInt, 0) - case ("decimal", precision :: scale :: Nil) => + case ("decimal" | "numeric", Nil) => DecimalType.USER_DEFAULT + case ("decimal" | "numeric", precision :: Nil) => DecimalType(precision.getText.toInt, 0) + case ("decimal" | "numeric", precision :: scale :: Nil) => DecimalType(precision.getText.toInt, scale.getText.toInt) case ("interval", Nil) => CalendarIntervalType case (dt, params) => diff --git a/sql/core/src/test/resources/sql-tests/inputs/cast.sql b/sql/core/src/test/resources/sql-tests/inputs/cast.sql index 8a035f594be5..af6356be8739 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/cast.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/cast.sql @@ -60,3 +60,10 @@ DESC FUNCTION EXTENDED boolean; -- cast string to interval and interval to string SELECT CAST('interval 3 month 1 hour' AS interval); SELECT CAST(interval 3 month 1 hour AS string); + +-- real / numeric +select cast(1.0 as real); +select cast('1' as real); +select cast(1.0 as numeric); +select cast(1.0 as numeric(3)); +select cast(1.08 as numeric(3,1)); diff --git a/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql b/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql index 852bfbd63847..f1eef6922ffa 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql @@ -59,3 +59,8 @@ TBLPROPERTIES ('a' = '1'); SHOW CREATE TABLE tbl; DROP TABLE tbl; + +-- new real/numeric type +CREATE TABLE tbl (a REAL, b NUMERIC, c NUMERIC(10), d NUMERIC(10,1)) USING parquet; +SHOW CREATE TABLE tbl; +DROP TABLE tbl; diff --git a/sql/core/src/test/resources/sql-tests/results/cast.sql.out b/sql/core/src/test/resources/sql-tests/results/cast.sql.out index 4884f9d491f2..930d24dc0df6 100644 --- a/sql/core/src/test/resources/sql-tests/results/cast.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/cast.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 35 +-- Number of queries: 40 -- !query 0 @@ -287,3 +287,43 @@ SELECT CAST(interval 3 month 1 hour AS string) struct -- !query 34 output 3 months 1 hours + + +-- !query 35 +select cast(1.0 as real) +-- !query 35 schema +struct +-- !query 35 output +1.0 + + +-- !query 36 +select cast('1' as real) +-- !query 36 schema +struct +-- !query 36 output +1.0 + + +-- !query 37 +select cast(1.0 as numeric) +-- !query 37 schema +struct +-- !query 37 output +1 + + +-- !query 38 +select cast(1.0 as numeric(3)) +-- !query 38 schema +struct +-- !query 38 output +1 + + +-- !query 39 +select cast(1.08 as numeric(3,1)) +-- !query 39 schema +struct +-- !query 39 output +1.1 diff --git a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out index 1faf16cc3050..fc1b62ca6180 100644 --- a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 24 +-- Number of queries: 27 -- !query 0 @@ -220,3 +220,28 @@ DROP TABLE tbl struct<> -- !query 23 output + + +-- !query 24 +CREATE TABLE tbl (a REAL, b NUMERIC, c NUMERIC(10), d NUMERIC(10,1)) USING parquet +-- !query 24 schema +struct<> +-- !query 24 output + + + +-- !query 25 +SHOW CREATE TABLE tbl +-- !query 25 schema +struct +-- !query 25 output +CREATE TABLE `tbl` (`a` FLOAT, `b` DECIMAL(10,0), `c` DECIMAL(10,0), `d` DECIMAL(10,1)) +USING parquet + + +-- !query 26 +DROP TABLE tbl +-- !query 26 schema +struct<> +-- !query 26 output + From e3857e851d8040aedb6ada9d6204dd11069a43bc Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 25 Nov 2019 11:19:55 +0800 Subject: [PATCH 2/5] regen golden file --- .../resources/sql-tests/results/cast.sql.out | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/resources/sql-tests/results/cast.sql.out b/sql/core/src/test/resources/sql-tests/results/cast.sql.out index bf1e873a48d1..28e66c40624b 100644 --- a/sql/core/src/test/resources/sql-tests/results/cast.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/cast.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 43 +-- Number of queries: 48 -- !query 0 @@ -351,3 +351,43 @@ select cast('1.0 ' as DEC) struct -- !query 42 output NULL + + +-- !query 43 +select cast(1.0 as real) +-- !query 43 schema +struct +-- !query 43 output +1.0 + + +-- !query 44 +select cast('1' as real) +-- !query 44 schema +struct +-- !query 44 output +1.0 + + +-- !query 45 +select cast(1.0 as numeric) +-- !query 45 schema +struct +-- !query 45 output +1 + + +-- !query 46 +select cast(1.0 as numeric(3)) +-- !query 46 schema +struct +-- !query 46 output +1 + + +-- !query 47 +select cast(1.08 as numeric(3,1)) +-- !query 47 schema +struct +-- !query 47 output +1.1 From 14265f0eece5e3a73b07deecdb72b7d2d8284bbe Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 25 Nov 2019 13:58:12 +0800 Subject: [PATCH 3/5] regen golden file --- .../resources/sql-tests/results/cast.sql.out | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/resources/sql-tests/results/cast.sql.out b/sql/core/src/test/resources/sql-tests/results/cast.sql.out index 112826a3d315..18c124f50ddb 100644 --- a/sql/core/src/test/resources/sql-tests/results/cast.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/cast.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 43 +-- Number of queries: 48 -- !query 0 @@ -351,3 +351,43 @@ select cast('1.0 ' as DEC) struct -- !query 42 output 1 + + +-- !query 43 +select cast(1.0 as real) +-- !query 43 schema +struct +-- !query 43 output +1.0 + + +-- !query 44 +select cast('1' as real) +-- !query 44 schema +struct +-- !query 44 output +1.0 + + +-- !query 45 +select cast(1.0 as numeric) +-- !query 45 schema +struct +-- !query 45 output +1 + + +-- !query 46 +select cast(1.0 as numeric(3)) +-- !query 46 schema +struct +-- !query 46 output +1 + + +-- !query 47 +select cast(1.08 as numeric(3,1)) +-- !query 47 schema +struct +-- !query 47 output +1.1 From a1f21fc92ed162fd5e5a382f2b68d1f4cec664d7 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 10 Dec 2019 10:09:11 +0800 Subject: [PATCH 4/5] regen gf --- .../resources/sql-tests/results/cast.sql.out | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/resources/sql-tests/results/cast.sql.out b/sql/core/src/test/resources/sql-tests/results/cast.sql.out index 343f9748c5ba..2fcddde55b2c 100644 --- a/sql/core/src/test/resources/sql-tests/results/cast.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/cast.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 46 +-- Number of queries: 51 -- !query 0 @@ -378,3 +378,43 @@ struct -- !query 45 output NULL + + +-- !query 46 +select cast(1.0 as real) +-- !query 46 schema +struct +-- !query 46 output +1.0 + + +-- !query 47 +select cast('1' as real) +-- !query 47 schema +struct +-- !query 47 output +1.0 + + +-- !query 48 +select cast(1.0 as numeric) +-- !query 48 schema +struct +-- !query 48 output +1 + + +-- !query 49 +select cast(1.0 as numeric(3)) +-- !query 49 schema +struct +-- !query 49 output +1 + + +-- !query 50 +select cast(1.08 as numeric(3,1)) +-- !query 50 schema +struct +-- !query 50 output +1.1 From d5d1f8edd9b47ff41d2f50966da9897100b8a38c Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 10 Dec 2019 21:05:22 +0800 Subject: [PATCH 5/5] rm tests in cast --- .../test/resources/sql-tests/inputs/cast.sql | 8 ---- .../sql-tests/inputs/show-create-table.sql | 2 +- .../resources/sql-tests/results/cast.sql.out | 42 +------------------ 3 files changed, 2 insertions(+), 50 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/cast.sql b/sql/core/src/test/resources/sql-tests/inputs/cast.sql index 0f8c26cc5536..972ebdd01f61 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/cast.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/cast.sql @@ -75,11 +75,3 @@ select cast('1.0 ' as DEC); select cast('\t\t true \n\r ' as boolean); select cast('\t\n false \t\r' as boolean); select cast('\t\n xyz \t\r' as boolean); - - --- real / numeric -select cast(1.0 as real); -select cast('1' as real); -select cast(1.0 as numeric); -select cast(1.0 as numeric(3)); -select cast(1.08 as numeric(3,1)); diff --git a/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql b/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql index f1eef6922ffa..3118371af39b 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql @@ -60,7 +60,7 @@ TBLPROPERTIES ('a' = '1'); SHOW CREATE TABLE tbl; DROP TABLE tbl; --- new real/numeric type +-- float alias real and decimal alias numeric CREATE TABLE tbl (a REAL, b NUMERIC, c NUMERIC(10), d NUMERIC(10,1)) USING parquet; SHOW CREATE TABLE tbl; DROP TABLE tbl; diff --git a/sql/core/src/test/resources/sql-tests/results/cast.sql.out b/sql/core/src/test/resources/sql-tests/results/cast.sql.out index 2fcddde55b2c..343f9748c5ba 100644 --- a/sql/core/src/test/resources/sql-tests/results/cast.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/cast.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 51 +-- Number of queries: 46 -- !query 0 @@ -378,43 +378,3 @@ struct -- !query 45 output NULL - - --- !query 46 -select cast(1.0 as real) --- !query 46 schema -struct --- !query 46 output -1.0 - - --- !query 47 -select cast('1' as real) --- !query 47 schema -struct --- !query 47 output -1.0 - - --- !query 48 -select cast(1.0 as numeric) --- !query 48 schema -struct --- !query 48 output -1 - - --- !query 49 -select cast(1.0 as numeric(3)) --- !query 49 schema -struct --- !query 49 output -1 - - --- !query 50 -select cast(1.08 as numeric(3,1)) --- !query 50 schema -struct --- !query 50 output -1.1