From 70459adb2cb3a45a88ab240d7f9e89b141b77200 Mon Sep 17 00:00:00 2001 From: "Jungtaek Lim (HeartSaVioR)" Date: Tue, 29 Oct 2019 09:31:17 +0900 Subject: [PATCH 1/3] [SPARK-28158][SQL][FOLLOWUP] HiveUserDefinedTypeSuite: don't let 'ExamplePoint' in input row be nullable --- .../org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala index bddb7688fe96..278e392e5190 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala @@ -35,7 +35,7 @@ class HiveUserDefinedTypeSuite extends QueryTest with TestHiveSingleton { test("Support UDT in Hive UDF") { val functionName = "get_point_x" try { - val schema = new StructType().add("point", new ExamplePointUDT) + val schema = new StructType().add("point", new ExamplePointUDT, nullable = false) val inputGenerator = RandomDataGenerator.forType(schema, nullable = false).get val input = inputGenerator.apply().asInstanceOf[Row] val df = spark.createDataFrame(Array(input).toList.asJava, schema) From 8d0929b8ed3921039bb510bc0e26358ea567023d Mon Sep 17 00:00:00 2001 From: "Jungtaek Lim (HeartSaVioR)" Date: Tue, 29 Oct 2019 10:13:09 +0900 Subject: [PATCH 2/3] Fix another possible case of UT failure --- .../apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala index 278e392e5190..344e5623cf68 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala @@ -18,26 +18,28 @@ package org.apache.spark.sql.hive import scala.collection.JavaConverters._ +import scala.util.Random import org.apache.hadoop.hive.ql.udf.generic.GenericUDF import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspector, StandardListObjectInspector} import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory -import org.apache.spark.sql.{QueryTest, RandomDataGenerator, Row} +import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.hive.test.TestHiveSingleton import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT} import org.apache.spark.sql.types.StructType + class HiveUserDefinedTypeSuite extends QueryTest with TestHiveSingleton { private val functionClass = classOf[org.apache.spark.sql.hive.TestUDF].getCanonicalName test("Support UDT in Hive UDF") { + val rand = new Random val functionName = "get_point_x" try { val schema = new StructType().add("point", new ExamplePointUDT, nullable = false) - val inputGenerator = RandomDataGenerator.forType(schema, nullable = false).get - val input = inputGenerator.apply().asInstanceOf[Row] + val input = Row.fromSeq(Seq(new ExamplePoint(rand.nextDouble(), rand.nextDouble()))) val df = spark.createDataFrame(Array(input).toList.asJava, schema) df.createOrReplaceTempView("src") spark.sql(s"CREATE FUNCTION $functionName AS '$functionClass'") From d1ba290f01c843fba6648f97274ec2004d2ddff0 Mon Sep 17 00:00:00 2001 From: "Jungtaek Lim (HeartSaVioR)" Date: Tue, 29 Oct 2019 10:29:53 +0900 Subject: [PATCH 3/3] Reflect review comments --- .../org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala index 344e5623cf68..c160ff206a62 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUserDefinedTypeSuite.scala @@ -30,16 +30,14 @@ import org.apache.spark.sql.hive.test.TestHiveSingleton import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT} import org.apache.spark.sql.types.StructType - class HiveUserDefinedTypeSuite extends QueryTest with TestHiveSingleton { private val functionClass = classOf[org.apache.spark.sql.hive.TestUDF].getCanonicalName test("Support UDT in Hive UDF") { - val rand = new Random val functionName = "get_point_x" try { val schema = new StructType().add("point", new ExamplePointUDT, nullable = false) - val input = Row.fromSeq(Seq(new ExamplePoint(rand.nextDouble(), rand.nextDouble()))) + val input = Row.fromSeq(Seq(new ExamplePoint(3.141592d, -3.141592d))) val df = spark.createDataFrame(Array(input).toList.asJava, schema) df.createOrReplaceTempView("src") spark.sql(s"CREATE FUNCTION $functionName AS '$functionClass'")