From f15b7c92129640e1d038c0e47a04f19e14d0c6e7 Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Mon, 8 Nov 2021 18:56:03 +0800 Subject: [PATCH 1/3] [SPARK-37196][SQL] HiveDecimal enforcePrecisionScala failed return null --- .../org/apache/spark/sql/hive/HiveShim.scala | 15 ++++++++++++--- .../org/apache/spark/sql/hive/InsertSuite.scala | 17 +++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala index 3a53a2a8dadd8..351cde58427c6 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala @@ -101,10 +101,19 @@ private[hive] object HiveShim { def toCatalystDecimal(hdoi: HiveDecimalObjectInspector, data: Any): Decimal = { if (hdoi.preferWritable()) { - Decimal(hdoi.getPrimitiveWritableObject(data).getHiveDecimal().bigDecimalValue, - hdoi.precision(), hdoi.scale()) + val value = hdoi.getPrimitiveWritableObject(data) + if (value == null) { + null + } else { + Decimal(value.getHiveDecimal().bigDecimalValue, hdoi.precision(), hdoi.scale()) + } } else { - Decimal(hdoi.getPrimitiveJavaObject(data).bigDecimalValue(), hdoi.precision(), hdoi.scale()) + val value = hdoi.getPrimitiveJavaObject(data) + if (value == null) { + null + } else { + Decimal(value.bigDecimalValue(), hdoi.precision(), hdoi.scale()) + } } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala index 177c227595162..0d40e7834a9f5 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala @@ -905,4 +905,21 @@ class InsertSuite extends QueryTest with TestHiveSingleton with BeforeAndAfter } } } + + test("SPARK-37196: HiveDecimal enforcePrecisionScala failed return null") { + withTempDir { dir => + withSQLConf(HiveUtils.CONVERT_METASTORE_PARQUET.key -> "false") { + withTable("test_precision") { + val df = sql("SELECT 'dummy' AS name, 1000000000000000000010.7000000000000010 AS value") + df.write.mode("Overwrite").parquet(dir.getAbsolutePath) + sql( + s""" + |CREATE EXTERNAL TABLE test_precision(name STRING, value DECIMAL(18,6)) + |STORED AS PARQUET LOCATION '${dir.getAbsolutePath}' + |""".stripMargin) + checkAnswer(sql("SELECT * FROM test_precision"), Row("dummy", null)) + } + } + } + } } From 2da2b49d442fbbab48af40a2df3b545907a3b6a5 Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Mon, 8 Nov 2021 19:02:43 +0800 Subject: [PATCH 2/3] update --- .../org/apache/spark/sql/hive/InsertSuite.scala | 17 ----------------- .../sql/hive/execution/SQLQuerySuite.scala | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala index 0d40e7834a9f5..177c227595162 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala @@ -905,21 +905,4 @@ class InsertSuite extends QueryTest with TestHiveSingleton with BeforeAndAfter } } } - - test("SPARK-37196: HiveDecimal enforcePrecisionScala failed return null") { - withTempDir { dir => - withSQLConf(HiveUtils.CONVERT_METASTORE_PARQUET.key -> "false") { - withTable("test_precision") { - val df = sql("SELECT 'dummy' AS name, 1000000000000000000010.7000000000000010 AS value") - df.write.mode("Overwrite").parquet(dir.getAbsolutePath) - sql( - s""" - |CREATE EXTERNAL TABLE test_precision(name STRING, value DECIMAL(18,6)) - |STORED AS PARQUET LOCATION '${dir.getAbsolutePath}' - |""".stripMargin) - checkAnswer(sql("SELECT * FROM test_precision"), Row("dummy", null)) - } - } - } - } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 5db705036e9f1..3edbce59d6195 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -2642,6 +2642,23 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi } } } + + test("SPARK-37196: HiveDecimal Precision Scale match failed should return null") { + withTempDir { dir => + withSQLConf(HiveUtils.CONVERT_METASTORE_PARQUET.key -> "false") { + withTable("test_precision") { + val df = sql("SELECT 'dummy' AS name, 1000000000000000000010.7000000000000010 AS value") + df.write.mode("Overwrite").parquet(dir.getAbsolutePath) + sql( + s""" + |CREATE EXTERNAL TABLE test_precision(name STRING, value DECIMAL(18,6)) + |STORED AS PARQUET LOCATION '${dir.getAbsolutePath}' + |""".stripMargin) + checkAnswer(sql("SELECT * FROM test_precision"), Row("dummy", null)) + } + } + } + } } @SlowHiveTest From 20048fb399bd4afa53bc7c79f91cddad26f1cdbb Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Mon, 8 Nov 2021 23:30:25 +0800 Subject: [PATCH 3/3] Update SQLQuerySuite.scala --- .../org/apache/spark/sql/hive/execution/SQLQuerySuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 3edbce59d6195..8d248bb387df6 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -2647,7 +2647,7 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi withTempDir { dir => withSQLConf(HiveUtils.CONVERT_METASTORE_PARQUET.key -> "false") { withTable("test_precision") { - val df = sql("SELECT 'dummy' AS name, 1000000000000000000010.7000000000000010 AS value") + val df = sql(s"SELECT 'dummy' AS name, ${"1" * 20}.${"2" * 18} AS value") df.write.mode("Overwrite").parquet(dir.getAbsolutePath) sql( s"""