From 86767d3bc995099c3ba6f5a21eb32863f8c5bb95 Mon Sep 17 00:00:00 2001 From: Harsh Motwani Date: Tue, 15 Oct 2024 08:13:10 +0200 Subject: [PATCH] [SPARK-49451][SQL][FOLLOW-UP] Improve duplicate key exception test ### What changes were proposed in this pull request? This test improves a unit test case where json strings with duplicate keys are prohibited by checking the cause of the exception instead of just the root exception. ### Why are the changes needed? Earlier, the test only checked the top error class but not the cause of the error which should be `VARIANT_DUPLICATE_KEY`. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? ### Was this patch authored or co-authored using generative AI tooling? NA Closes #48464 from harshmotw-db/harshmotw-db/minor_test_fix. Authored-by: Harsh Motwani Signed-off-by: Max Gekk --- .../spark/sql/VariantEndToEndSuite.scala | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/VariantEndToEndSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/VariantEndToEndSuite.scala index 19d4ac23709b6..fe5c6ef004920 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/VariantEndToEndSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/VariantEndToEndSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.spark.SparkThrowable +import org.apache.spark.{SparkException, SparkRuntimeException} import org.apache.spark.sql.QueryTest.sameRows import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Cast, Literal} @@ -359,16 +359,24 @@ class VariantEndToEndSuite extends QueryTest with SharedSparkSession { val expectedMetadata: Array[Byte] = Array(VERSION, 3, 0, 1, 2, 3, 'a', 'b', 'c') assert(actual === new VariantVal(expectedValue, expectedMetadata)) } - withSQLConf(SQLConf.VARIANT_ALLOW_DUPLICATE_KEYS.key -> "false") { - val df = Seq(json).toDF("j") - .selectExpr("from_json(j,'variant')") - checkError( - exception = intercept[SparkThrowable] { + // Check whether the parse_json and from_json expressions throw the correct exception. + Seq("from_json(j, 'variant')", "parse_json(j)").foreach { expr => + withSQLConf(SQLConf.VARIANT_ALLOW_DUPLICATE_KEYS.key -> "false") { + val df = Seq(json).toDF("j").selectExpr(expr) + val exception = intercept[SparkException] { df.collect() - }, - condition = "MALFORMED_RECORD_IN_PARSING.WITHOUT_SUGGESTION", - parameters = Map("badRecord" -> json, "failFastMode" -> "FAILFAST") - ) + } + checkError( + exception = exception, + condition = "MALFORMED_RECORD_IN_PARSING.WITHOUT_SUGGESTION", + parameters = Map("badRecord" -> json, "failFastMode" -> "FAILFAST") + ) + checkError( + exception = exception.getCause.asInstanceOf[SparkRuntimeException], + condition = "VARIANT_DUPLICATE_KEY", + parameters = Map("key" -> "a") + ) + } } } }