[SPARK-20680][SQL][FOLLOW-UP] Revert NullType.simpleString from 'unknown' to 'null'

python/pyspark/sql/types.py

@@ -116,9 +116,6 @@ class NullType(DataType):
 
     __metaclass__ = DataTypeSingleton
 
-    def simpleString(self):
-        return 'unknown'
-
 
 class AtomicType(DataType):
     """An internal type used to represent everything that is not

sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala

@@ -356,7 +356,7 @@ private[sql] object CatalogV2Util {
     }
     if (containsNullType(dt)) {
       throw new AnalysisException(
-        "Cannot create tables with unknown type.")
+        s"Cannot create tables with ${NullType.simpleString} type.")
     }
   }
 

sql/catalyst/src/main/scala/org/apache/spark/sql/types/NullType.scala

@@ -32,10 +32,6 @@ class NullType private() extends DataType {
   override def defaultSize: Int = 1
 
   private[spark] override def asNullable: NullType = this
-
-  // "null" is mainly used to represent a literal in Spark,
-  // it's better to avoid using it for data types.
-  override def simpleString: String = "unknown"
 }
 
 /**

sql/core/src/test/resources/sql-functions/sql-expression-schema.md

@@ -34,7 +34,7 @@
 | org.apache.spark.sql.catalyst.expressions.Ascii | ascii | SELECT ascii('222') | struct<ascii(222):int> |
 | org.apache.spark.sql.catalyst.expressions.Asin | asin | SELECT asin(0) | struct<ASIN(CAST(0 AS DOUBLE)):double> |
 | org.apache.spark.sql.catalyst.expressions.Asinh | asinh | SELECT asinh(0) | struct<ASINH(CAST(0 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.AssertTrue | assert_true | SELECT assert_true(0 < 1) | struct<assert_true((0 < 1)):unknown> |
+| org.apache.spark.sql.catalyst.expressions.AssertTrue | assert_true | SELECT assert_true(0 < 1) | struct<assert_true((0 < 1)):null> |
 | org.apache.spark.sql.catalyst.expressions.Atan | atan | SELECT atan(0) | struct<ATAN(CAST(0 AS DOUBLE)):double> |
 | org.apache.spark.sql.catalyst.expressions.Atan2 | atan2 | SELECT atan2(0, 0) | struct<ATAN2(CAST(0 AS DOUBLE), CAST(0 AS DOUBLE)):double> |
 | org.apache.spark.sql.catalyst.expressions.Atanh | atanh | SELECT atanh(0) | struct<ATANH(CAST(0 AS DOUBLE)):double> |

sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out

@@ -5,7 +5,7 @@
 -- !query
 select null, Null, nUll
 -- !query schema
-struct<NULL:unknown,NULL:unknown,NULL:unknown>
+struct<NULL:null,NULL:null,NULL:null>
 -- !query output
 NULL	NULL	NULL
 

sql/core/src/test/resources/sql-tests/results/inline-table.sql.out

@@ -49,7 +49,7 @@ two	2
 -- !query
 select * from values ("one", null), ("two", null) as data(a, b)
 -- !query schema
-struct<a:string,b:unknown>
+struct<a:string,b:null>
 -- !query output
 one	NULL
 two	NULL

sql/core/src/test/resources/sql-tests/results/literals.sql.out

@@ -5,7 +5,7 @@
 -- !query
 select null, Null, nUll
 -- !query schema
-struct<NULL:unknown,NULL:unknown,NULL:unknown>
+struct<NULL:null,NULL:null,NULL:null>
 -- !query output
 NULL	NULL	NULL
 

sql/core/src/test/resources/sql-tests/results/misc-functions.sql.out

@@ -7,7 +7,7 @@ select typeof(null)
 -- !query schema
 struct<typeof(NULL):string>
 -- !query output
-unknown
+null
 
 
 -- !query

sql/core/src/test/resources/sql-tests/results/postgreSQL/select.sql.out

@@ -308,15 +308,15 @@ struct<1:int>
 -- !query
 select foo.* from (select null) as foo
 -- !query schema
-struct<NULL:unknown>
+struct<NULL:null>
 -- !query output
 NULL
 
 
 -- !query
 select foo.* from (select 'xyzzy',1,null) as foo
 -- !query schema
-struct<xyzzy:string,1:int,NULL:unknown>
+struct<xyzzy:string,1:int,NULL:null>
 -- !query output
 xyzzy	1	NULL
 

sql/core/src/test/resources/sql-tests/results/sql-compatibility-functions.sql.out

@@ -5,7 +5,7 @@
 -- !query
 SELECT ifnull(null, 'x'), ifnull('y', 'x'), ifnull(null, null)
 -- !query schema
-struct<ifnull(NULL, x):string,ifnull(y, x):string,ifnull(NULL, NULL):unknown>
+struct<ifnull(NULL, x):string,ifnull(y, x):string,ifnull(NULL, NULL):null>
 -- !query output
 x	y	NULL
 
@@ -21,15 +21,15 @@ NULL	x
 -- !query
 SELECT nvl(null, 'x'), nvl('y', 'x'), nvl(null, null)
 -- !query schema
-struct<nvl(NULL, x):string,nvl(y, x):string,nvl(NULL, NULL):unknown>
+struct<nvl(NULL, x):string,nvl(y, x):string,nvl(NULL, NULL):null>
 -- !query output
 x	y	NULL
 
 
 -- !query
 SELECT nvl2(null, 'x', 'y'), nvl2('n', 'x', 'y'), nvl2(null, null, null)
 -- !query schema
-struct<nvl2(NULL, x, y):string,nvl2(n, x, y):string,nvl2(NULL, NULL, NULL):unknown>
+struct<nvl2(NULL, x, y):string,nvl2(n, x, y):string,nvl2(NULL, NULL, NULL):null>
 -- !query output
 y	x	NULL
 

sql/core/src/test/resources/sql-tests/results/udf/udf-inline-table.sql.out

@@ -49,7 +49,7 @@ two	2
 -- !query
 select udf(a), b from values ("one", null), ("two", null) as data(a, b)
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS STRING):string,b:unknown>
+struct<CAST(udf(cast(a as string)) AS STRING):string,b:null>
 -- !query output
 one	NULL
 two	NULL

sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala

@@ -405,7 +405,7 @@ class FileBasedDataSourceSuite extends QueryTest
         ""
       }
       def errorMessage(format: String): String = {
-        s"$format data source does not support unknown data type."
+        s"$format data source does not support null data type."
       }
       withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> useV1List) {
         withTempDir { dir =>

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala

@@ -2310,64 +2310,64 @@ class HiveDDLSuite
     }
   }
 
-  test("SPARK-20680: Spark-sql do not support for unknown column datatype") {
+  test("SPARK-20680: do not support for null column datatype") {
     withTable("t") {
-      withView("tabUnknownType") {
+      withView("tabNullType") {
         hiveClient.runSqlHive("CREATE TABLE t (t1 int)")
         hiveClient.runSqlHive("INSERT INTO t VALUES (3)")
-        hiveClient.runSqlHive("CREATE VIEW tabUnknownType AS SELECT NULL AS col FROM t")
-        checkAnswer(spark.table("tabUnknownType"), Row(null))
+        hiveClient.runSqlHive("CREATE VIEW tabNullType AS SELECT NULL AS col FROM t")
+        checkAnswer(spark.table("tabNullType"), Row(null))
         // No exception shows
-        val desc = spark.sql("DESC tabUnknownType").collect().toSeq
+        val desc = spark.sql("DESC tabNullType").collect().toSeq
         assert(desc.contains(Row("col", NullType.simpleString, null)))
       }
     }
 
-    // Forbid CTAS with unknown type
+    // Forbid CTAS with null type
     withTable("t1", "t2", "t3") {
       val e1 = intercept[AnalysisException] {
         spark.sql("CREATE TABLE t1 USING PARQUET AS SELECT null as null_col")
       }.getMessage
-      assert(e1.contains("Cannot create tables with unknown type"))
+      assert(e1.contains("Cannot create tables with null type"))
 
       val e2 = intercept[AnalysisException] {
         spark.sql("CREATE TABLE t2 AS SELECT null as null_col")
       }.getMessage
-      assert(e2.contains("Cannot create tables with unknown type"))
+      assert(e2.contains("Cannot create tables with null type"))
 
       val e3 = intercept[AnalysisException] {
         spark.sql("CREATE TABLE t3 STORED AS PARQUET AS SELECT null as null_col")
       }.getMessage
-      assert(e3.contains("Cannot create tables with unknown type"))
+      assert(e3.contains("Cannot create tables with null type"))
     }
 
-    // Forbid Replace table AS SELECT with unknown type
+    // Forbid Replace table AS SELECT with null type
     withTable("t") {
       val v2Source = classOf[FakeV2Provider].getName
       val e = intercept[AnalysisException] {
         spark.sql(s"CREATE OR REPLACE TABLE t USING $v2Source AS SELECT null as null_col")
       }.getMessage
-      assert(e.contains("Cannot create tables with unknown type"))
+      assert(e.contains("Cannot create tables with null type"))
     }
 
     // Forbid creating table with VOID type in Spark
     withTable("t1", "t2", "t3", "t4") {
       val e1 = intercept[AnalysisException] {
         spark.sql(s"CREATE TABLE t1 (v VOID) USING PARQUET")
       }.getMessage
-      assert(e1.contains("Cannot create tables with unknown type"))
+      assert(e1.contains("Cannot create tables with null type"))
       val e2 = intercept[AnalysisException] {
         spark.sql(s"CREATE TABLE t2 (v VOID) USING hive")
       }.getMessage
-      assert(e2.contains("Cannot create tables with unknown type"))
+      assert(e2.contains("Cannot create tables with null type"))
       val e3 = intercept[AnalysisException] {
         spark.sql(s"CREATE TABLE t3 (v VOID)")
       }.getMessage
-      assert(e3.contains("Cannot create tables with unknown type"))
+      assert(e3.contains("Cannot create tables with null type"))
       val e4 = intercept[AnalysisException] {
         spark.sql(s"CREATE TABLE t4 (v VOID) STORED AS PARQUET")
       }.getMessage
-      assert(e4.contains("Cannot create tables with unknown type"))
+      assert(e4.contains("Cannot create tables with null type"))
     }
 
     // Forbid Replace table with VOID type
@@ -2376,7 +2376,7 @@ class HiveDDLSuite
       val e = intercept[AnalysisException] {
         spark.sql(s"CREATE OR REPLACE TABLE t (v VOID) USING $v2Source")
       }.getMessage
-      assert(e.contains("Cannot create tables with unknown type"))
+      assert(e.contains("Cannot create tables with null type"))
     }
 
     // Make sure spark.catalog.createTable with null type will fail
@@ -2413,7 +2413,7 @@ class HiveDDLSuite
           schema = schema,
           options = Map("fileFormat" -> "parquet"))
       }.getMessage
-      assert(e.contains("Cannot create tables with unknown type"))
+      assert(e.contains("Cannot create tables with null type"))
     }
   }
 
@@ -2426,7 +2426,7 @@ class HiveDDLSuite
           schema = schema,
           options = Map.empty[String, String])
       }.getMessage
-      assert(e.contains("Cannot create tables with unknown type"))
+      assert(e.contains("Cannot create tables with null type"))
     }
   }
 

sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala

@@ -121,7 +121,7 @@ class HiveOrcSourceSuite extends OrcSuite with TestHiveSingleton {
       msg = intercept[AnalysisException] {
         sql("select null").write.mode("overwrite").orc(orcDir)
       }.getMessage
-      assert(msg.contains("ORC data source does not support unknown data type."))
+      assert(msg.contains("ORC data source does not support null data type."))
 
       msg = intercept[AnalysisException] {
         spark.udf.register("testType", () => new IntervalData())