From e5abd6406fba19c1557572fda44f29a88c78721c Mon Sep 17 00:00:00 2001 From: PengLei Date: Mon, 17 Jan 2022 16:41:28 +0800 Subject: [PATCH] add draft --- R/pkg/tests/fulltests/test_sparkSQL.R | 8 +-- python/pyspark/sql/functions.py | 8 +-- .../catalyst/expressions/csvExpressions.scala | 2 +- .../expressions/jsonExpressions.scala | 4 +- .../apache/spark/sql/types/StructField.scala | 6 +- .../expressions/CsvExpressionsSuite.scala | 4 +- .../expressions/JsonExpressionsSuite.scala | 8 +-- .../spark/sql/types/StructTypeSuite.scala | 16 +++--- .../sql-tests/results/charvarchar.sql.out | 12 ++-- .../sql-tests/results/csv-functions.sql.out | 2 +- .../sql-tests/results/json-functions.sql.out | 6 +- .../results/show-create-table.sql.out | 56 +++++++++---------- .../apache/spark/sql/CsvFunctionsSuite.scala | 8 +-- .../sql/DataFrameSetOperationsSuite.scala | 50 ++++++++--------- .../apache/spark/sql/JsonFunctionsSuite.scala | 10 ++-- .../command/ShowCreateTableSuiteBase.scala | 22 ++++---- .../command/v1/ShowCreateTableSuite.scala | 18 +++--- .../command/v2/ShowCreateTableSuite.scala | 20 +++---- .../command/ShowCreateTableSuite.scala | 16 +++--- 19 files changed, 138 insertions(+), 138 deletions(-) diff --git a/R/pkg/tests/fulltests/test_sparkSQL.R b/R/pkg/tests/fulltests/test_sparkSQL.R index 0e46324ed5c4..73b9dcc0a572 100644 --- a/R/pkg/tests/fulltests/test_sparkSQL.R +++ b/R/pkg/tests/fulltests/test_sparkSQL.R @@ -1690,9 +1690,9 @@ test_that("column functions", { df <- as.DataFrame(list(list("col" = "1"))) c <- collect(select(df, schema_of_csv("Amsterdam,2018"))) - expect_equal(c[[1]], "STRUCT<`_c0`: STRING, `_c1`: INT>") + expect_equal(c[[1]], "STRUCT<_c0: STRING, _c1: INT>") c <- collect(select(df, schema_of_csv(lit("Amsterdam,2018")))) - expect_equal(c[[1]], "STRUCT<`_c0`: STRING, `_c1`: INT>") + expect_equal(c[[1]], "STRUCT<_c0: STRING, _c1: INT>") # Test to_json(), from_json(), schema_of_json() df <- sql("SELECT array(named_struct('name', 'Bob'), named_struct('name', 'Alice')) as people") @@ -1725,9 +1725,9 @@ test_that("column functions", { df <- as.DataFrame(list(list("col" = "1"))) c <- collect(select(df, schema_of_json('{"name":"Bob"}'))) - expect_equal(c[[1]], "STRUCT<`name`: STRING>") + expect_equal(c[[1]], "STRUCT") c <- collect(select(df, schema_of_json(lit('{"name":"Bob"}')))) - expect_equal(c[[1]], "STRUCT<`name`: STRING>") + expect_equal(c[[1]], "STRUCT") # Test to_json() supports arrays of primitive types and arrays df <- sql("SELECT array(19, 42, 70) as age") diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py index f2bca0b5d050..e69c37d320a3 100644 --- a/python/pyspark/sql/functions.py +++ b/python/pyspark/sql/functions.py @@ -4091,10 +4091,10 @@ def schema_of_json(json: "ColumnOrName", options: Optional[Dict[str, str]] = Non -------- >>> df = spark.range(1) >>> df.select(schema_of_json(lit('{"a": 0}')).alias("json")).collect() - [Row(json='STRUCT<`a`: BIGINT>')] + [Row(json='STRUCT')] >>> schema = schema_of_json('{a: 1}', {'allowUnquotedFieldNames':'true'}) >>> df.select(schema.alias("json")).collect() - [Row(json='STRUCT<`a`: BIGINT>')] + [Row(json='STRUCT')] """ if isinstance(json, str): col = _create_column_from_literal(json) @@ -4127,9 +4127,9 @@ def schema_of_csv(csv: "ColumnOrName", options: Optional[Dict[str, str]] = None) -------- >>> df = spark.range(1) >>> df.select(schema_of_csv(lit('1|a'), {'sep':'|'}).alias("csv")).collect() - [Row(csv='STRUCT<`_c0`: INT, `_c1`: STRING>')] + [Row(csv='STRUCT<_c0: INT, _c1: STRING>')] >>> df.select(schema_of_csv('1|a', {'sep':'|'}).alias("csv")).collect() - [Row(csv='STRUCT<`_c0`: INT, `_c1`: STRING>')] + [Row(csv='STRUCT<_c0: INT, _c1: STRING>')] """ if isinstance(csv, str): col = _create_column_from_literal(csv) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/csvExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/csvExpressions.scala index 79bbc103c92d..30d992a2eea6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/csvExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/csvExpressions.scala @@ -153,7 +153,7 @@ case class CsvToStructs( examples = """ Examples: > SELECT _FUNC_('1,abc'); - STRUCT<`_c0`: INT, `_c1`: STRING> + STRUCT<_c0: INT, _c1: STRING> """, since = "3.0.0", group = "csv_funcs") diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala index 5b058626e222..9f00b7c8b740 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala @@ -766,9 +766,9 @@ case class StructsToJson( examples = """ Examples: > SELECT _FUNC_('[{"col":0}]'); - ARRAY> + ARRAY> > SELECT _FUNC_('[{"col":01}]', map('allowNumericLeadingZeros', 'true')); - ARRAY> + ARRAY> """, group = "json_funcs", since = "2.4.0") diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructField.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructField.scala index 93d57a7fe6f3..f490f8318ef8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructField.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructField.scala @@ -21,7 +21,7 @@ import org.json4s.JsonAST.JValue import org.json4s.JsonDSL._ import org.apache.spark.annotation.Stable -import org.apache.spark.sql.catalyst.util.{escapeSingleQuotedString, quoteIdentifier} +import org.apache.spark.sql.catalyst.util.{escapeSingleQuotedString, quoteIfNeeded} import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat import org.apache.spark.sql.util.SchemaUtils @@ -93,7 +93,7 @@ case class StructField( * Returns a string containing a schema in SQL format. For example the following value: * `StructField("eventId", IntegerType)` will be converted to `eventId`: INT. */ - private[sql] def sql = s"${quoteIdentifier(name)}: ${dataType.sql}$getDDLComment" + private[sql] def sql = s"${quoteIfNeeded(name)}: ${dataType.sql}$getDDLComment" /** * Returns a string containing a schema in DDL format. For example, the following value: @@ -103,6 +103,6 @@ case class StructField( */ def toDDL: String = { val nullString = if (nullable) "" else " NOT NULL" - s"${quoteIdentifier(name)} ${dataType.sql}${nullString}$getDDLComment" + s"${quoteIfNeeded(name)} ${dataType.sql}${nullString}$getDDLComment" } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala index 7945974a1f3d..1d174ed21452 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala @@ -158,13 +158,13 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P } test("infer schema of CSV strings") { - checkEvaluation(new SchemaOfCsv(Literal.create("1,abc")), "STRUCT<`_c0`: INT, `_c1`: STRING>") + checkEvaluation(new SchemaOfCsv(Literal.create("1,abc")), "STRUCT<_c0: INT, _c1: STRING>") } test("infer schema of CSV strings by using options") { checkEvaluation( new SchemaOfCsv(Literal.create("1|abc"), Map("delimiter" -> "|")), - "STRUCT<`_c0`: INT, `_c1`: STRING>") + "STRUCT<_c0: INT, _c1: STRING>") } test("to_csv - struct") { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala index 2ae7c76599e5..af071727b10d 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala @@ -736,17 +736,17 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with test("SPARK-24709: infer schema of json strings") { checkEvaluation(new SchemaOfJson(Literal.create("""{"col":0}""")), - "STRUCT<`col`: BIGINT>") + "STRUCT") checkEvaluation( new SchemaOfJson(Literal.create("""{"col0":["a"], "col1": {"col2": "b"}}""")), - "STRUCT<`col0`: ARRAY, `col1`: STRUCT<`col2`: STRING>>") + "STRUCT, col1: STRUCT>") } test("infer schema of JSON strings by using options") { checkEvaluation( new SchemaOfJson(Literal.create("""{"col":01}"""), CreateMap(Seq(Literal.create("allowNumericLeadingZeros"), Literal.create("true")))), - "STRUCT<`col`: BIGINT>") + "STRUCT") } test("parse date with locale") { @@ -811,7 +811,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with } Seq("en-US", "ko-KR", "ru-RU", "de-DE").foreach { - checkDecimalInfer(_, """STRUCT<`d`: DECIMAL(7,3)>""") + checkDecimalInfer(_, """STRUCT""") } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala index a7e22e940327..16f122334f37 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala @@ -51,7 +51,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper { test("SPARK-24849: toDDL - simple struct") { val struct = StructType(Seq(StructField("a", IntegerType))) - assert(struct.toDDL == "`a` INT") + assert(struct.toDDL == "a INT") } test("SPARK-24849: round trip toDDL - fromDDL") { @@ -61,7 +61,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper { } test("SPARK-24849: round trip fromDDL - toDDL") { - val struct = "`a` MAP,`b` INT" + val struct = "a MAP,b INT" assert(fromDDL(struct).toDDL === struct) } @@ -70,14 +70,14 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper { val struct = new StructType() .add("metaData", new StructType().add("eventId", StringType)) - assert(struct.toDDL == "`metaData` STRUCT<`eventId`: STRING>") + assert(struct.toDDL == "metaData STRUCT") } test("SPARK-24849: toDDL should output field's comment") { val struct = StructType(Seq( StructField("b", BooleanType).withComment("Field's comment"))) - assert(struct.toDDL == """`b` BOOLEAN COMMENT 'Field\'s comment'""") + assert(struct.toDDL == """b BOOLEAN COMMENT 'Field\'s comment'""") } private val nestedStruct = new StructType() @@ -89,7 +89,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper { ).withComment("comment")) test("SPARK-33846: toDDL should output nested field's comment") { - val ddl = "`a` STRUCT<`b`: STRUCT<`c`: STRING COMMENT 'Deep Nested comment'> " + + val ddl = "a STRUCT " + "COMMENT 'Nested comment'> COMMENT 'comment'" assert(nestedStruct.toDDL == ddl) } @@ -153,7 +153,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper { } test("interval keyword in schema string") { - val interval = "`a` INTERVAL" + val interval = "a INTERVAL" assert(fromDDL(interval).toDDL === interval) } @@ -250,10 +250,10 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper { } test("SPARK-35285: ANSI interval types in schema") { - val yearMonthInterval = "`ymi` INTERVAL YEAR TO MONTH" + val yearMonthInterval = "ymi INTERVAL YEAR TO MONTH" assert(fromDDL(yearMonthInterval).toDDL === yearMonthInterval) - val dayTimeInterval = "`dti` INTERVAL DAY TO SECOND" + val dayTimeInterval = "dti INTERVAL DAY TO SECOND" assert(fromDDL(dayTimeInterval).toDDL === dayTimeInterval) } diff --git a/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out b/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out index 5c6b1a727705..de994d67c9f3 100644 --- a/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out @@ -52,8 +52,8 @@ show create table char_tbl struct -- !query output CREATE TABLE default.char_tbl ( - `c` CHAR(5), - `v` VARCHAR(6)) + c CHAR(5), + v VARCHAR(6)) USING parquet @@ -71,8 +71,8 @@ show create table char_tbl2 struct -- !query output CREATE TABLE default.char_tbl2 ( - `c` CHAR(5), - `v` VARCHAR(6)) + c CHAR(5), + v VARCHAR(6)) USING parquet @@ -162,8 +162,8 @@ show create table char_tbl3 struct -- !query output CREATE TABLE default.char_tbl3 ( - `c` CHAR(5), - `v` VARCHAR(6)) + c CHAR(5), + v VARCHAR(6)) USING parquet diff --git a/sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out index 2ca44d51244a..53cae3f93556 100644 --- a/sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out @@ -89,7 +89,7 @@ select schema_of_csv('1|abc', map('delimiter', '|')) -- !query schema struct -- !query output -STRUCT<`_c0`: INT, `_c1`: STRING> +STRUCT<_c0: INT, _c1: STRING> -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out index ff59553e4e9d..e509d4e4cc27 100644 --- a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out @@ -236,7 +236,7 @@ select schema_of_json('{"c1":0, "c2":[1]}') -- !query schema struct -- !query output -STRUCT<`c1`: BIGINT, `c2`: ARRAY> +STRUCT> -- !query @@ -375,7 +375,7 @@ select schema_of_json('{"c1":1}', map('primitivesAsString', 'true')) -- !query schema struct -- !query output -STRUCT<`c1`: STRING> +STRUCT -- !query @@ -383,7 +383,7 @@ select schema_of_json('{"c1":01, "c2":0.1}', map('allowNumericLeadingZeros', 'tr -- !query schema struct -- !query output -STRUCT<`c1`: BIGINT, `c2`: DECIMAL(1,1)> +STRUCT -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out index ffcbb73458aa..4c7f124e72fe 100644 --- a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out @@ -16,9 +16,9 @@ SHOW CREATE TABLE tbl struct -- !query output CREATE TABLE default.tbl ( - `a` INT, - `b` STRING, - `c` INT) + a INT, + b STRING, + c INT) USING parquet @@ -45,9 +45,9 @@ SHOW CREATE TABLE tbl struct -- !query output CREATE TABLE default.tbl ( - `a` INT, - `b` STRING, - `c` INT) + a INT, + b STRING, + c INT) USING parquet OPTIONS ( 'a' = '1') @@ -76,9 +76,9 @@ SHOW CREATE TABLE tbl struct -- !query output CREATE TABLE default.tbl ( - `a` INT, - `b` STRING, - `c` INT) + a INT, + b STRING, + c INT) USING parquet LOCATION 'file:/path/to/table' @@ -106,9 +106,9 @@ SHOW CREATE TABLE tbl struct -- !query output CREATE TABLE default.tbl ( - `a` INT, - `b` STRING, - `c` INT) + a INT, + b STRING, + c INT) USING parquet LOCATION 'file:/path/to/table' @@ -136,9 +136,9 @@ SHOW CREATE TABLE tbl struct -- !query output CREATE TABLE default.tbl ( - `b` STRING, - `c` INT, - `a` INT) + b STRING, + c INT, + a INT) USING parquet PARTITIONED BY (a) @@ -166,9 +166,9 @@ SHOW CREATE TABLE tbl struct -- !query output CREATE TABLE default.tbl ( - `a` INT, - `b` STRING, - `c` INT) + a INT, + b STRING, + c INT) USING parquet CLUSTERED BY (a) SORTED BY (b) @@ -198,9 +198,9 @@ SHOW CREATE TABLE tbl struct -- !query output CREATE TABLE default.tbl ( - `a` INT, - `b` STRING, - `c` INT) + a INT, + b STRING, + c INT) USING parquet COMMENT 'This is a comment' @@ -228,9 +228,9 @@ SHOW CREATE TABLE tbl struct -- !query output CREATE TABLE default.tbl ( - `a` INT, - `b` STRING, - `c` INT) + a INT, + b STRING, + c INT) USING parquet TBLPROPERTIES ( 'a' = '1') @@ -258,10 +258,10 @@ SHOW CREATE TABLE tbl struct -- !query output CREATE TABLE default.tbl ( - `a` FLOAT, - `b` DECIMAL(10,0), - `c` DECIMAL(10,0), - `d` DECIMAL(10,1)) + a FLOAT, + b DECIMAL(10,0), + c DECIMAL(10,0), + d DECIMAL(10,1)) USING parquet diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala index 2808652f2998..461bbd8987ce 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala @@ -82,16 +82,16 @@ class CsvFunctionsSuite extends QueryTest with SharedSparkSession { test("schema_of_csv - infers schemas") { checkAnswer( spark.range(1).select(schema_of_csv(lit("0.1,1"))), - Seq(Row("STRUCT<`_c0`: DOUBLE, `_c1`: INT>"))) + Seq(Row("STRUCT<_c0: DOUBLE, _c1: INT>"))) checkAnswer( spark.range(1).select(schema_of_csv("0.1,1")), - Seq(Row("STRUCT<`_c0`: DOUBLE, `_c1`: INT>"))) + Seq(Row("STRUCT<_c0: DOUBLE, _c1: INT>"))) } test("schema_of_csv - infers schemas using options") { val df = spark.range(1) .select(schema_of_csv(lit("0.1 1"), Map("sep" -> " ").asJava)) - checkAnswer(df, Seq(Row("STRUCT<`_c0`: DOUBLE, `_c1`: INT>"))) + checkAnswer(df, Seq(Row("STRUCT<_c0: DOUBLE, _c1: INT>"))) } test("to_csv - struct") { @@ -220,7 +220,7 @@ class CsvFunctionsSuite extends QueryTest with SharedSparkSession { val input = concat_ws(",", lit(0.1), lit(1)) checkAnswer( spark.range(1).select(schema_of_csv(input)), - Seq(Row("STRUCT<`_c0`: DOUBLE, `_c1`: INT>"))) + Seq(Row("STRUCT<_c0: DOUBLE, _c1: INT>"))) } test("optional datetime parser does not affect csv time formatting") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala index b19e4300b5af..19a62c25f5c5 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala @@ -804,7 +804,7 @@ class DataFrameSetOperationsSuite extends QueryTest with SharedSparkSession { StructType(Seq(StructField("topLevelCol", nestedStructType2)))) val union = df1.unionByName(df2, allowMissingColumns = true) - assert(union.schema.toDDL == "`topLevelCol` STRUCT<`b`: STRING, `a`: STRING>") + assert(union.schema.toDDL == "topLevelCol STRUCT") checkAnswer(union, Row(Row("b", null)) :: Row(Row("b", "a")) :: Nil) } @@ -836,15 +836,15 @@ class DataFrameSetOperationsSuite extends QueryTest with SharedSparkSession { StructType(Seq(StructField("topLevelCol", nestedStructType2)))) var unionDf = df1.unionByName(df2, true) - assert(unionDf.schema.toDDL == "`topLevelCol` " + - "STRUCT<`b`: STRUCT<`ba`: STRING, `bb`: STRING>, `a`: STRUCT<`aa`: STRING>>") + assert(unionDf.schema.toDDL == "topLevelCol " + + "STRUCT, a: STRUCT>") checkAnswer(unionDf, Row(Row(Row("ba", null), null)) :: Row(Row(Row(null, "bb"), Row("aa"))) :: Nil) unionDf = df2.unionByName(df1, true) - assert(unionDf.schema.toDDL == "`topLevelCol` STRUCT<`a`: STRUCT<`aa`: STRING>, " + - "`b`: STRUCT<`bb`: STRING, `ba`: STRING>>") + assert(unionDf.schema.toDDL == "topLevelCol STRUCT, " + + "b: STRUCT>") checkAnswer(unionDf, Row(Row(null, Row(null, "ba"))) :: Row(Row(Row("aa"), Row("bb", null))) :: Nil) @@ -1112,13 +1112,13 @@ class DataFrameSetOperationsSuite extends QueryTest with SharedSparkSession { StructType(Seq(StructField("arr", arrayType2)))) var unionDf = df1.unionByName(df2) - assert(unionDf.schema.toDDL == "`arr` ARRAY>") + assert(unionDf.schema.toDDL == "arr ARRAY>") checkAnswer(unionDf, Row(Seq(Row("ba", "bb"))) :: Row(Seq(Row("ba", "bb"))) :: Nil) unionDf = df2.unionByName(df1) - assert(unionDf.schema.toDDL == "`arr` ARRAY>") + assert(unionDf.schema.toDDL == "arr ARRAY>") checkAnswer(unionDf, Row(Seq(Row("bb", "ba"))) :: Row(Seq(Row("bb", "ba"))) :: Nil) @@ -1150,7 +1150,7 @@ class DataFrameSetOperationsSuite extends QueryTest with SharedSparkSession { } unionDf = df3.unionByName(df4, true) - assert(unionDf.schema.toDDL == "`arr` ARRAY>") + assert(unionDf.schema.toDDL == "arr ARRAY>") checkAnswer(unionDf, Row(Seq(Row("ba", null))) :: Row(Seq(Row(null, "bb"))) :: Nil) @@ -1160,7 +1160,7 @@ class DataFrameSetOperationsSuite extends QueryTest with SharedSparkSession { } unionDf = df4.unionByName(df3, true) - assert(unionDf.schema.toDDL == "`arr` ARRAY>") + assert(unionDf.schema.toDDL == "arr ARRAY>") checkAnswer(unionDf, Row(Seq(Row("bb", null))) :: Row(Seq(Row(null, "ba"))) :: Nil) @@ -1196,15 +1196,15 @@ class DataFrameSetOperationsSuite extends QueryTest with SharedSparkSession { StructType(Seq(StructField("topLevelCol", nestedStructType2)))) var unionDf = df1.unionByName(df2) - assert(unionDf.schema.toDDL == "`topLevelCol` " + - "STRUCT<`b`: ARRAY>>") + assert(unionDf.schema.toDDL == "topLevelCol " + + "STRUCT>>") checkAnswer(unionDf, Row(Row(Seq(Row("ba", "bb")))) :: Row(Row(Seq(Row("ba", "bb")))) :: Nil) unionDf = df2.unionByName(df1) - assert(unionDf.schema.toDDL == "`topLevelCol` STRUCT<" + - "`b`: ARRAY>>") + assert(unionDf.schema.toDDL == "topLevelCol STRUCT<" + + "b: ARRAY>>") checkAnswer(unionDf, Row(Row(Seq(Row("bb", "ba")))) :: Row(Row(Seq(Row("bb", "ba")))) :: Nil) @@ -1240,8 +1240,8 @@ class DataFrameSetOperationsSuite extends QueryTest with SharedSparkSession { } unionDf = df3.unionByName(df4, true) - assert(unionDf.schema.toDDL == "`topLevelCol` " + - "STRUCT<`b`: ARRAY>>") + assert(unionDf.schema.toDDL == "topLevelCol " + + "STRUCT>>") checkAnswer(unionDf, Row(Row(Seq(Row("ba", null)))) :: Row(Row(Seq(Row(null, "bb")))) :: Nil) @@ -1251,8 +1251,8 @@ class DataFrameSetOperationsSuite extends QueryTest with SharedSparkSession { } unionDf = df4.unionByName(df3, true) - assert(unionDf.schema.toDDL == "`topLevelCol` STRUCT<" + - "`b`: ARRAY>>") + assert(unionDf.schema.toDDL == "topLevelCol STRUCT<" + + "b: ARRAY>>") checkAnswer(unionDf, Row(Row(Seq(Row("bb", null)))) :: Row(Row(Seq(Row(null, "ba")))) :: Nil) @@ -1292,15 +1292,15 @@ class DataFrameSetOperationsSuite extends QueryTest with SharedSparkSession { StructType(Seq(StructField("topLevelCol", nestedStructType2)))) var unionDf = df1.unionByName(df2) - assert(unionDf.schema.toDDL == "`topLevelCol` " + - "STRUCT<`b`: ARRAY>>>") + assert(unionDf.schema.toDDL == "topLevelCol " + + "STRUCT>>>") checkAnswer(unionDf, Row(Row(Seq(Seq(Row("ba", "bb"))))) :: Row(Row(Seq(Seq(Row("ba", "bb"))))) :: Nil) unionDf = df2.unionByName(df1) - assert(unionDf.schema.toDDL == "`topLevelCol` STRUCT<" + - "`b`: ARRAY>>>") + assert(unionDf.schema.toDDL == "topLevelCol STRUCT<" + + "b: ARRAY>>>") checkAnswer(unionDf, Row(Row(Seq(Seq(Row("bb", "ba"))))) :: Row(Row(Seq(Seq(Row("bb", "ba"))))) :: Nil) @@ -1340,8 +1340,8 @@ class DataFrameSetOperationsSuite extends QueryTest with SharedSparkSession { } unionDf = df3.unionByName(df4, true) - assert(unionDf.schema.toDDL == "`topLevelCol` " + - "STRUCT<`b`: ARRAY>>>") + assert(unionDf.schema.toDDL == "topLevelCol " + + "STRUCT>>>") checkAnswer(unionDf, Row(Row(Seq(Seq(Row("ba", null))))) :: Row(Row(Seq(Seq(Row(null, "bb"))))) :: Nil) @@ -1351,8 +1351,8 @@ class DataFrameSetOperationsSuite extends QueryTest with SharedSparkSession { } unionDf = df4.unionByName(df3, true) - assert(unionDf.schema.toDDL == "`topLevelCol` STRUCT<" + - "`b`: ARRAY>>>") + assert(unionDf.schema.toDDL == "topLevelCol STRUCT<" + + "b: ARRAY>>>") checkAnswer(unionDf, Row(Row(Seq(Seq(Row("bb", null))))) :: Row(Row(Seq(Seq(Row(null, "ba"))))) :: Nil) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala index 06babab122fd..6661b58b8f52 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala @@ -417,7 +417,7 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession { test("infers schemas using options") { val df = spark.range(1) .select(schema_of_json(lit("{a:1}"), Map("allowUnquotedFieldNames" -> "true").asJava)) - checkAnswer(df, Seq(Row("STRUCT<`a`: BIGINT>"))) + checkAnswer(df, Seq(Row("STRUCT"))) } test("from_json - array of primitive types") { @@ -697,14 +697,14 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession { val input = regexp_replace(lit("""{"item_id": 1, "item_price": 0.1}"""), "item_", "") checkAnswer( spark.range(1).select(schema_of_json(input)), - Seq(Row("STRUCT<`id`: BIGINT, `price`: DOUBLE>"))) + Seq(Row("STRUCT"))) } test("SPARK-31065: schema_of_json - null and empty strings as strings") { Seq("""{"id": null}""", """{"id": ""}""").foreach { input => checkAnswer( spark.range(1).select(schema_of_json(input)), - Seq(Row("STRUCT<`id`: STRING>"))) + Seq(Row("STRUCT"))) } } @@ -716,7 +716,7 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession { schema_of_json( lit("""{"id": "a", "drop": {"drop": null}}"""), options.asJava)), - Seq(Row("STRUCT<`id`: STRING>"))) + Seq(Row("STRUCT"))) // Array of structs checkAnswer( @@ -724,7 +724,7 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession { schema_of_json( lit("""[{"id": "a", "drop": {"drop": null}}]"""), options.asJava)), - Seq(Row("ARRAY>"))) + Seq(Row("ARRAY>"))) // Other types are not affected. checkAnswer( diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowCreateTableSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowCreateTableSuiteBase.scala index 53cdec0d2b6c..7bc076561f44 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowCreateTableSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowCreateTableSuiteBase.scala @@ -51,8 +51,8 @@ trait ShowCreateTableSuiteBase extends QueryTest with DDLCommandTestUtils { """.stripMargin) val showDDL = getShowCreateDDL(t) assert(showDDL(0) == s"CREATE TABLE $fullName (") - assert(showDDL(1) == "`a` BIGINT NOT NULL,") - assert(showDDL(2) == "`b` BIGINT)") + assert(showDDL(1) == "a BIGINT NOT NULL,") + assert(showDDL(2) == "b BIGINT)") assert(showDDL(3) == s"USING ${classOf[SimpleInsertSource].getName}") } } @@ -75,10 +75,10 @@ trait ShowCreateTableSuiteBase extends QueryTest with DDLCommandTestUtils { ) val showDDL = getShowCreateDDL(t) assert(showDDL(0) == s"CREATE TABLE $fullName (") - assert(showDDL(1) == "`a` STRING,") - assert(showDDL(2) == "`b` STRING,") + assert(showDDL(1) == "a STRING,") + assert(showDDL(2) == "b STRING,") assert(showDDL(3) == "`extra col` ARRAY,") - assert(showDDL(4) == "`` STRUCT<`x`: INT, `y`: ARRAY>)") + assert(showDDL(4) == "`` STRUCT>)") assert(showDDL(5) == "USING json") assert(showDDL(6).startsWith("LOCATION 'file:") && showDDL(6).endsWith("sample.json'")) } @@ -95,7 +95,7 @@ trait ShowCreateTableSuiteBase extends QueryTest with DDLCommandTestUtils { """.stripMargin) val showDDL = getShowCreateDDL(t) assert(showDDL(0) == s"CREATE TABLE $fullName (") - assert(showDDL(1) == "`a` STRUCT<`b`: STRING>)") + assert(showDDL(1) == "a STRUCT)") assert(showDDL(2) == "USING json") } } @@ -119,7 +119,7 @@ trait ShowCreateTableSuiteBase extends QueryTest with DDLCommandTestUtils { |) """.stripMargin ) - val expected = s"CREATE TABLE $fullName ( `a` STRING) USING json" + + val expected = s"CREATE TABLE $fullName ( a STRING) USING json" + " OPTIONS ( 'k1' = 'v1', 'k2' = 'v2', 'k3' = 'v3', 'k4' = 'v4', 'k5' = 'v5')" + " TBLPROPERTIES ( 'a' = '2', 'b' = '1')" assert(getShowCreateDDL(t).mkString(" ") == expected) @@ -134,7 +134,7 @@ trait ShowCreateTableSuiteBase extends QueryTest with DDLCommandTestUtils { |AS SELECT 1 AS a, "foo" AS b """.stripMargin ) - val expected = s"CREATE TABLE $fullName ( `a` INT, `b` STRING) USING json" + val expected = s"CREATE TABLE $fullName ( a INT, b STRING) USING json" assert(getShowCreateDDL(t).mkString(" ") == expected) } } @@ -148,7 +148,7 @@ trait ShowCreateTableSuiteBase extends QueryTest with DDLCommandTestUtils { |AS SELECT 1 AS a, "foo" AS b """.stripMargin ) - val expected = s"CREATE TABLE $fullName ( `a` INT, `b` STRING) USING json PARTITIONED BY (b)" + val expected = s"CREATE TABLE $fullName ( a INT, b STRING) USING json PARTITIONED BY (b)" assert(getShowCreateDDL(t).mkString(" ") == expected) } } @@ -162,7 +162,7 @@ trait ShowCreateTableSuiteBase extends QueryTest with DDLCommandTestUtils { |AS SELECT 1 AS a, "foo" AS b, 2.5 AS c """.stripMargin ) - val expected = s"CREATE TABLE $fullName ( `a` INT, `b` STRING, `c` DECIMAL(2,1)) USING json" + + val expected = s"CREATE TABLE $fullName ( a INT, b STRING, c DECIMAL(2,1)) USING json" + s" COMMENT 'This is a comment'" assert(getShowCreateDDL(t).mkString(" ") == expected) } @@ -177,7 +177,7 @@ trait ShowCreateTableSuiteBase extends QueryTest with DDLCommandTestUtils { |AS SELECT 1 AS a, "foo" AS b, 2.5 AS c """.stripMargin ) - val expected = s"CREATE TABLE $fullName ( `a` INT, `b` STRING, `c` DECIMAL(2,1)) USING json" + + val expected = s"CREATE TABLE $fullName ( a INT, b STRING, c DECIMAL(2,1)) USING json" + s" TBLPROPERTIES ( 'a' = '1')" assert(getShowCreateDDL(t).mkString(" ") == expected) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala index 023dfce3ba9c..1dd5e4a5aaa7 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala @@ -58,11 +58,11 @@ trait ShowCreateTableSuiteBase extends command.ShowCreateTableSuiteBase val showDDL = getShowCreateDDL(t) assert(showDDL === Array( s"CREATE TABLE $fullName (", - "`b` BIGINT,", - "`c` BIGINT,", - "`extraCol` ARRAY,", - "`` STRUCT<`x`: INT, `y`: ARRAY>,", - "`a` BIGINT NOT NULL)", + "b BIGINT,", + "c BIGINT,", + "extraCol ARRAY,", + "`` STRUCT>,", + "a BIGINT NOT NULL)", "USING parquet", "OPTIONS (", "'from' = '0',", @@ -89,7 +89,7 @@ trait ShowCreateTableSuiteBase extends command.ShowCreateTableSuiteBase |AS SELECT 1 AS a, "foo" AS b """.stripMargin ) - val expected = s"CREATE TABLE $fullName ( `a` INT, `b` STRING) USING json" + + val expected = s"CREATE TABLE $fullName ( a INT, b STRING) USING json" + s" CLUSTERED BY (a) INTO 2 BUCKETS" assert(getShowCreateDDL(t).mkString(" ") == expected) } @@ -104,7 +104,7 @@ trait ShowCreateTableSuiteBase extends command.ShowCreateTableSuiteBase |AS SELECT 1 AS a, "foo" AS b """.stripMargin ) - val expected = s"CREATE TABLE $fullName ( `a` INT, `b` STRING) USING json" + + val expected = s"CREATE TABLE $fullName ( a INT, b STRING) USING json" + s" CLUSTERED BY (a) SORTED BY (b) INTO 2 BUCKETS" assert(getShowCreateDDL(t).mkString(" ") == expected) } @@ -120,7 +120,7 @@ trait ShowCreateTableSuiteBase extends command.ShowCreateTableSuiteBase |AS SELECT 1 AS a, "foo" AS b, 2.5 AS c """.stripMargin ) - val expected = s"CREATE TABLE $fullName ( `a` INT, `b` STRING, `c` DECIMAL(2,1)) USING json" + + val expected = s"CREATE TABLE $fullName ( a INT, b STRING, c DECIMAL(2,1)) USING json" + s" PARTITIONED BY (c) CLUSTERED BY (a) INTO 2 BUCKETS" assert(getShowCreateDDL(t).mkString(" ") == expected) } @@ -136,7 +136,7 @@ trait ShowCreateTableSuiteBase extends command.ShowCreateTableSuiteBase |AS SELECT 1 AS a, "foo" AS b, 2.5 AS c """.stripMargin ) - val expected = s"CREATE TABLE $fullName ( `a` INT, `b` STRING, `c` DECIMAL(2,1)) USING json" + + val expected = s"CREATE TABLE $fullName ( a INT, b STRING, c DECIMAL(2,1)) USING json" + s" PARTITIONED BY (c) CLUSTERED BY (a) SORTED BY (b) INTO 2 BUCKETS" assert(getShowCreateDDL(t).mkString(" ") == expected) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala index 47e59e965509..7c506812079e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala @@ -51,8 +51,8 @@ class ShowCreateTableSuite extends command.ShowCreateTableSuiteBase with Command val showDDL = getShowCreateDDL(t, false) assert(showDDL === Array( s"CREATE TABLE $t (", - "`a` INT,", - "`b` STRING)", + "a INT,", + "b STRING)", defaultUsing, "PARTITIONED BY (a)", "COMMENT 'This is a comment'", @@ -89,11 +89,11 @@ class ShowCreateTableSuite extends command.ShowCreateTableSuiteBase with Command val showDDL = getShowCreateDDL(t, false) assert(showDDL === Array( s"CREATE TABLE $t (", - "`a` BIGINT NOT NULL,", - "`b` BIGINT,", - "`c` BIGINT,", - "`extraCol` ARRAY,", - "`` STRUCT<`x`: INT, `y`: ARRAY>)", + "a BIGINT NOT NULL,", + "b BIGINT,", + "c BIGINT,", + "extraCol ARRAY,", + "`` STRUCT>)", defaultUsing, "OPTIONS (", "'from' = '0',", @@ -128,9 +128,9 @@ class ShowCreateTableSuite extends command.ShowCreateTableSuiteBase with Command val showDDL = getShowCreateDDL(t, false) assert(showDDL === Array( s"CREATE TABLE $t (", - "`a` INT,", - "`b` STRING,", - "`ts` TIMESTAMP)", + "a INT,", + "b STRING,", + "ts TIMESTAMP)", defaultUsing, "PARTITIONED BY (a, years(ts), months(ts), days(ts), hours(ts))", "CLUSTERED BY (b)", diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowCreateTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowCreateTableSuite.scala index 58145b03fd3c..a7d5e7b08348 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowCreateTableSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowCreateTableSuite.scala @@ -48,7 +48,7 @@ class ShowCreateTableSuite extends v1.ShowCreateTableSuiteBase with CommandSuite |) """.stripMargin ) - val expected = s"CREATE TABLE $fullName ( `c1` INT COMMENT 'bla', `c2` STRING)" + + val expected = s"CREATE TABLE $fullName ( c1 INT COMMENT 'bla', c2 STRING)" + " ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'" + " WITH SERDEPROPERTIES ( 'serialization.format' = '1')" + " STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'" + @@ -73,7 +73,7 @@ class ShowCreateTableSuite extends v1.ShowCreateTableSuiteBase with CommandSuite |) """.stripMargin ) - val expected = s"CREATE EXTERNAL TABLE $fullName ( `c1` INT COMMENT 'bla', `c2` STRING)" + + val expected = s"CREATE EXTERNAL TABLE $fullName ( c1 INT COMMENT 'bla', c2 STRING)" + s" ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'" + s" WITH SERDEPROPERTIES ( 'serialization.format' = '1')" + s" STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'" + @@ -100,8 +100,8 @@ class ShowCreateTableSuite extends v1.ShowCreateTableSuiteBase with CommandSuite |) """.stripMargin ) - val expected = s"CREATE TABLE $fullName ( `c1` INT COMMENT 'bla', `c2` STRING)" + - " COMMENT 'bla' PARTITIONED BY (`p1` BIGINT COMMENT 'bla', `p2` STRING)" + + val expected = s"CREATE TABLE $fullName ( c1 INT COMMENT 'bla', c2 STRING)" + + " COMMENT 'bla' PARTITIONED BY (p1 BIGINT COMMENT 'bla', p2 STRING)" + " ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'" + " WITH SERDEPROPERTIES ( 'serialization.format' = '1')" + " STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'" + @@ -124,7 +124,7 @@ class ShowCreateTableSuite extends v1.ShowCreateTableSuiteBase with CommandSuite |NULL DEFINED AS 'NaN' """.stripMargin ) - val expected = s"CREATE TABLE $fullName ( `c1` INT COMMENT 'bla', `c2` STRING)" + + val expected = s"CREATE TABLE $fullName ( c1 INT COMMENT 'bla', c2 STRING)" + " ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'" + " WITH SERDEPROPERTIES (" + " 'colelction.delim' = '@'," + @@ -148,7 +148,7 @@ class ShowCreateTableSuite extends v1.ShowCreateTableSuiteBase with CommandSuite |STORED AS PARQUET """.stripMargin ) - val expected = s"CREATE TABLE $fullName ( `c1` INT COMMENT 'bla', `c2` STRING)" + + val expected = s"CREATE TABLE $fullName ( c1 INT COMMENT 'bla', c2 STRING)" + " ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'" + " WITH SERDEPROPERTIES ( 'serialization.format' = '1')" + " STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'" + @@ -175,7 +175,7 @@ class ShowCreateTableSuite extends v1.ShowCreateTableSuiteBase with CommandSuite | OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' """.stripMargin ) - val expected = s"CREATE TABLE $fullName ( `c1` INT COMMENT 'bla', `c2` STRING)" + + val expected = s"CREATE TABLE $fullName ( c1 INT COMMENT 'bla', c2 STRING)" + " ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'" + " WITH SERDEPROPERTIES (" + " 'mapkey.delim' = ','," + @@ -197,7 +197,7 @@ class ShowCreateTableSuite extends v1.ShowCreateTableSuiteBase with CommandSuite |INTO 2 BUCKETS """.stripMargin ) - val expected = s"CREATE TABLE $fullName ( `a` INT, `b` STRING)" + + val expected = s"CREATE TABLE $fullName ( a INT, b STRING)" + " CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS" + " ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'" + " WITH SERDEPROPERTIES ( 'serialization.format' = '1')" +