Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1504,6 +1504,7 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging {
case ("decimal", precision :: Nil) => DecimalType(precision.getText.toInt, 0)
case ("decimal", precision :: scale :: Nil) =>
DecimalType(precision.getText.toInt, scale.getText.toInt)
case ("void", Nil) => NullType
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This change really resolves your issue?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Apparently Hive can have null typed columns. So this should be the location where you'd want to change this.

Copy link
Member

@gatorsmile gatorsmile May 12, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hive 2.x disables it. Could you add some test cases by reading and writing the tables with void types? Thanks!

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1 for the test case.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you add a comment to explain this specific scenario?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure

case (dt, params) =>
val dtStr = if (params.nonEmpty) s"$dt(${params.mkString(",")})" else dt
throw new ParseException(s"DataType $dtStr is not supported.", ctx)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ class NullType private() extends DataType {
override def defaultSize: Int = 1

private[spark] override def asNullable: NullType = this

/**
* Readable string representation for NULL type.
*/
override def simpleString: String = "void"
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ class DataTypeParserSuite extends SparkFunSuite {
checkDataType("varchAr(20)", StringType)
checkDataType("cHaR(27)", StringType)
checkDataType("BINARY", BinaryType)
checkDataType("void", NullType)

checkDataType("array<doublE>", ArrayType(DoubleType, true))
checkDataType("Array<map<int, tinYint>>", ArrayType(MapType(IntegerType, ByteType, true), true))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ two 2
-- !query 5
select * from values ("one", null), ("two", null) as data(a, b)
-- !query 5 schema
struct<a:string,b:null>
struct<a:string,b:void>
-- !query 5 output
one NULL
two NULL
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
-- !query 0
select null, Null, nUll
-- !query 0 schema
struct<NULL:null,NULL:null,NULL:null>
struct<NULL:void,NULL:void,NULL:void>
-- !query 0 output
NULL NULL NULL

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
-- !query 0
SELECT ifnull(null, 'x'), ifnull('y', 'x'), ifnull(null, null)
-- !query 0 schema
struct<ifnull(NULL, 'x'):string,ifnull('y', 'x'):string,ifnull(NULL, NULL):null>
struct<ifnull(NULL, 'x'):string,ifnull('y', 'x'):string,ifnull(NULL, NULL):void>
-- !query 0 output
x y NULL

Expand All @@ -21,15 +21,15 @@ NULL x
-- !query 2
SELECT nvl(null, 'x'), nvl('y', 'x'), nvl(null, null)
-- !query 2 schema
struct<nvl(NULL, 'x'):string,nvl('y', 'x'):string,nvl(NULL, NULL):null>
struct<nvl(NULL, 'x'):string,nvl('y', 'x'):string,nvl(NULL, NULL):void>
-- !query 2 output
x y NULL


-- !query 3
SELECT nvl2(null, 'x', 'y'), nvl2('n', 'x', 'y'), nvl2(null, null, null)
-- !query 3 schema
struct<nvl2(NULL, 'x', 'y'):string,nvl2('n', 'x', 'y'):string,nvl2(NULL, NULL, NULL):null>
struct<nvl2(NULL, 'x', 'y'):string,nvl2('n', 'x', 'y'):string,nvl2(NULL, NULL, NULL):void>
-- !query 3 output
y x NULL

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1928,4 +1928,17 @@ class HiveDDLSuite
}
}
}

test("SPARK-20680: Spark-sql do not support for void column datatype of view") {
withTable("t", "tabNullType") {
val client = spark.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client
client.runSqlHive("CREATE TABLE t (t1 int)")
client.runSqlHive("INSERT INTO t VALUES (3)")
client.runSqlHive("CREATE TABLE tabNullType AS SELECT NULL AS col FROM t")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IIRC, hive 2 does't support this. Let's test with CREATE VIEW AS ... to be safer

checkAnswer(spark.table("tabNullType"), Row(null))
// table description shows "void" representation for NULL type.
val desc = spark.sql("DESC tabNullType").collect().toSeq
assert(desc.contains(Row("col", "void", null)))
}
}
}