From 3426c34352d07445a336854ab92550ac6710ab87 Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Wed, 6 Nov 2024 11:22:48 +0100 Subject: [PATCH 01/19] Throw ARITHMETIC_OVERFLOW in MakeYMInterval, MakeDTInterval --- .../expressions/intervalExpressions.scala | 21 +++++++++++++------ .../sql/catalyst/util/IntervalUtils.scala | 13 ++++++++---- .../IntervalExpressionsSuite.scala | 4 ++-- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala index d18630f54202..37be072cbdf8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala @@ -478,17 +478,26 @@ case class MakeYMInterval(years: Expression, months: Expression) override def dataType: DataType = YearMonthIntervalType() override def nullSafeEval(year: Any, month: Any): Any = { - Math.toIntExact(Math.addExact(month.asInstanceOf[Number].longValue(), - Math.multiplyExact(year.asInstanceOf[Number].longValue(), MONTHS_PER_YEAR))) + try { + Math.toIntExact(Math.addExact(month.asInstanceOf[Number].longValue(), + Math.multiplyExact(year.asInstanceOf[Number].longValue(), MONTHS_PER_YEAR))) + } catch { + case e: ArithmeticException => + throw QueryExecutionErrors.arithmeticOverflowError(e.getMessage) + } } override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - defineCodeGen(ctx, ev, (years, months) => { + nullSafeCodeGen(ctx, ev, (years, months) => { val math = classOf[Math].getName.stripSuffix("$") s""" - |$math.toIntExact(java.lang.Math.addExact($months, - | $math.multiplyExact($years, $MONTHS_PER_YEAR))) - |""".stripMargin + |try { + | ${ev.value} = $math.toIntExact( + | $math.addExact($months, $math.multiplyExact($years, $MONTHS_PER_YEAR))); + |} catch (java.lang.ArithmeticException e) { + | throw QueryExecutionErrors.arithmeticOverflowError(e.getMessage(), "", null); + |} + """.stripMargin }) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala index 90c802b7e28d..ec9df7003b51 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala @@ -785,10 +785,15 @@ object IntervalUtils extends SparkIntervalUtils { secs: Decimal): Long = { assert(secs.scale == 6, "Seconds fractional must have 6 digits for microseconds") var micros = secs.toUnscaledLong - micros = Math.addExact(micros, Math.multiplyExact(days, MICROS_PER_DAY)) - micros = Math.addExact(micros, Math.multiplyExact(hours, MICROS_PER_HOUR)) - micros = Math.addExact(micros, Math.multiplyExact(mins, MICROS_PER_MINUTE)) - micros + try { + micros = Math.addExact(micros, Math.multiplyExact(days, MICROS_PER_DAY)) + micros = Math.addExact(micros, Math.multiplyExact(hours, MICROS_PER_HOUR)) + micros = Math.addExact(micros, Math.multiplyExact(mins, MICROS_PER_MINUTE)) + micros + } catch { + case e: ArithmeticException => + throw QueryExecutionErrors.arithmeticOverflowError(e.getMessage) + } } def intToYearMonthInterval(v: Int, startField: Byte, endField: Byte): Int = { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala index 78bc77b9dc2a..442237734823 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala @@ -316,7 +316,7 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { val secFrac = DateTimeTestUtils.secFrac(seconds, millis, micros) val durationExpr = MakeDTInterval(Literal(days), Literal(hours), Literal(minutes), Literal(Decimal(secFrac, Decimal.MAX_LONG_DIGITS, 6))) - checkExceptionInExpression[ArithmeticException](durationExpr, EmptyRow, "") + checkExceptionInExpression[ArithmeticException](durationExpr, "ARITHMETIC_OVERFLOW") } check(millis = -123) @@ -528,7 +528,7 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { Seq(MakeYMInterval(Literal(178956970), Literal(8)), MakeYMInterval(Literal(-178956970), Literal(-9))) .foreach { ym => - checkExceptionInExpression[ArithmeticException](ym, "integer overflow") + checkExceptionInExpression[ArithmeticException](ym, "ARITHMETIC_OVERFLOW") } def checkImplicitEvaluation(expr: Expression, value: Any): Unit = { From 908e7459afc0a1d961606772f1916f7a5f42206e Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Wed, 6 Nov 2024 20:32:22 +0100 Subject: [PATCH 02/19] Add another type of exception because we can't use current ARITHMETIC_OVERFLOW, it suggests disabling ANSI --- .../resources/error/error-conditions.json | 20 ++++++++++++---- .../expressions/intervalExpressions.scala | 24 ++++++++++++------- .../sql/catalyst/util/IntervalMathUtils.scala | 2 +- .../sql/catalyst/util/IntervalUtils.scala | 4 ++-- .../sql/errors/QueryExecutionErrors.scala | 16 ++++++++++--- .../IntervalExpressionsSuite.scala | 4 ++-- 6 files changed, 49 insertions(+), 21 deletions(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 18603b61d8ae..108f16d20ecc 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -2020,11 +2020,23 @@ ], "sqlState" : "XX000" }, - "INTERVAL_ARITHMETIC_OVERFLOW" : { - "message" : [ - "." + "INTERVAL_ARITHMETIC_OVERFLOW": { + "message": [ + "Integer overflow while creating interval." ], - "sqlState" : "22015" + "subClass": { + "NUMERIC": { + "message": [ + ".." + ] + }, + "DATETIME": { + "message": [ + "Too big interval for datetime, check how you create it." + ] + } + }, + "sqlState": "22015" }, "INTERVAL_DIVIDED_BY_ZERO" : { "message" : [ diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala index 37be072cbdf8..cc7100de02ce 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala @@ -467,7 +467,8 @@ case class MakeDTInterval( group = "datetime_funcs") // scalastyle:on line.size.limit case class MakeYMInterval(years: Expression, months: Expression) - extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant with Serializable { + extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant with Serializable + with SupportQueryContext { def this(years: Expression) = this(years, Literal(0)) def this() = this(Literal(0)) @@ -479,25 +480,26 @@ case class MakeYMInterval(years: Expression, months: Expression) override def nullSafeEval(year: Any, month: Any): Any = { try { - Math.toIntExact(Math.addExact(month.asInstanceOf[Number].longValue(), - Math.multiplyExact(year.asInstanceOf[Number].longValue(), MONTHS_PER_YEAR))) + Math.toIntExact( + Math.addExact(month.asInstanceOf[Int], + Math.multiplyExact(year.asInstanceOf[Int], MONTHS_PER_YEAR))) } catch { - case e: ArithmeticException => - throw QueryExecutionErrors.arithmeticOverflowError(e.getMessage) + case _: ArithmeticException => + throw QueryExecutionErrors.datetimeIntervalArithmeticOverflowError() } } override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { nullSafeCodeGen(ctx, ev, (years, months) => { val math = classOf[Math].getName.stripSuffix("$") + val errorContext = getContextOrNullCode(ctx) s""" |try { | ${ev.value} = $math.toIntExact( | $math.addExact($months, $math.multiplyExact($years, $MONTHS_PER_YEAR))); |} catch (java.lang.ArithmeticException e) { - | throw QueryExecutionErrors.arithmeticOverflowError(e.getMessage(), "", null); - |} - """.stripMargin + | throw QueryExecutionErrors.datetimeIntervalArithmeticOverflowError($errorContext); + |}""".stripMargin }) } @@ -506,6 +508,10 @@ case class MakeYMInterval(years: Expression, months: Expression) override protected def withNewChildrenInternal( newLeft: Expression, newRight: Expression): Expression = copy(years = newLeft, months = newRight) + + override def initQueryContext(): Option[QueryContext] = { + Some(origin.context) + } } // Multiply an year-month interval by a numeric @@ -616,7 +622,7 @@ trait IntervalDivide { context: QueryContext): Unit = { if (value == minValue && num.dataType.isInstanceOf[IntegralType]) { if (numValue.asInstanceOf[Number].longValue() == -1) { - throw QueryExecutionErrors.intervalArithmeticOverflowError( + throw QueryExecutionErrors.numericIntervalArithmeticOverflowError( "Interval value overflows after being divided by -1", "try_divide", context) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala index c935c6057376..b643083ef349 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala @@ -40,7 +40,7 @@ object IntervalMathUtils { f } catch { case e: ArithmeticException => - throw QueryExecutionErrors.intervalArithmeticOverflowError(e.getMessage, hint, null) + throw QueryExecutionErrors.numericIntervalArithmeticOverflowError(e.getMessage, hint, null) } } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala index ec9df7003b51..0b82a9290f6a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala @@ -791,8 +791,8 @@ object IntervalUtils extends SparkIntervalUtils { micros = Math.addExact(micros, Math.multiplyExact(mins, MICROS_PER_MINUTE)) micros } catch { - case e: ArithmeticException => - throw QueryExecutionErrors.arithmeticOverflowError(e.getMessage) + case _: ArithmeticException => + throw QueryExecutionErrors.datetimeIntervalArithmeticOverflowError() } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index 2cc223ba69fa..a5a5e547d33c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -636,15 +636,15 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE summary = "") } - def intervalArithmeticOverflowError( + def numericIntervalArithmeticOverflowError( message: String, hint: String = "", - context: QueryContext): ArithmeticException = { + context: QueryContext = null): ArithmeticException = { val alternative = if (hint.nonEmpty) { s" Use '$hint' to tolerate overflow and return NULL instead." } else "" new SparkArithmeticException( - errorClass = "INTERVAL_ARITHMETIC_OVERFLOW", + errorClass = "INTERVAL_ARITHMETIC_OVERFLOW.NUMERIC", messageParameters = Map( "message" -> message, "alternative" -> alternative), @@ -652,6 +652,16 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE summary = getSummary(context)) } + def datetimeIntervalArithmeticOverflowError( + context: QueryContext = null): SparkArithmeticException = { + new SparkArithmeticException( + errorClass = "INTERVAL_ARITHMETIC_OVERFLOW.DATETIME", + messageParameters = Map(), + context = getQueryContext(context), + summary = getSummary(context)) + } + + def failedToCompileMsg(e: Exception): String = { s"Failed to compile: $e" } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala index 442237734823..b6c41c79cd86 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala @@ -316,7 +316,7 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { val secFrac = DateTimeTestUtils.secFrac(seconds, millis, micros) val durationExpr = MakeDTInterval(Literal(days), Literal(hours), Literal(minutes), Literal(Decimal(secFrac, Decimal.MAX_LONG_DIGITS, 6))) - checkExceptionInExpression[ArithmeticException](durationExpr, "ARITHMETIC_OVERFLOW") + checkExceptionInExpression[ArithmeticException](durationExpr, "ARITHMETIC_OVERFLOW.DATETIME") } check(millis = -123) @@ -528,7 +528,7 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { Seq(MakeYMInterval(Literal(178956970), Literal(8)), MakeYMInterval(Literal(-178956970), Literal(-9))) .foreach { ym => - checkExceptionInExpression[ArithmeticException](ym, "ARITHMETIC_OVERFLOW") + checkExceptionInExpression[ArithmeticException](ym, "ARITHMETIC_OVERFLOW.DATETIME") } def checkImplicitEvaluation(expr: Expression, value: Any): Unit = { From b5cd32aa50024884e6abcfb74c1180111c02a8d9 Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Thu, 7 Nov 2024 12:29:46 +0100 Subject: [PATCH 03/19] better error message --- .../src/main/resources/error/error-conditions.json | 8 ++++---- .../catalyst/expressions/intervalExpressions.scala | 11 ++++++----- .../spark/sql/catalyst/util/IntervalMathUtils.scala | 3 ++- .../spark/sql/catalyst/util/IntervalUtils.scala | 2 +- .../spark/sql/errors/QueryExecutionErrors.scala | 8 ++++---- .../expressions/IntervalExpressionsSuite.scala | 6 ++++-- 6 files changed, 21 insertions(+), 17 deletions(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 108f16d20ecc..05ed287ff946 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -2022,17 +2022,17 @@ }, "INTERVAL_ARITHMETIC_OVERFLOW": { "message": [ - "Integer overflow while creating interval." + "Integer overflow while operating with intervals." ], "subClass": { - "NUMERIC": { + "WITH_TRY_SUGGESTION": { "message": [ ".." ] }, - "DATETIME": { + "WITHOUT_SUGGESTION": { "message": [ - "Too big interval for datetime, check how you create it." + "" ] } }, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala index cc7100de02ce..989b2c92722f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala @@ -485,7 +485,7 @@ case class MakeYMInterval(years: Expression, months: Expression) Math.multiplyExact(year.asInstanceOf[Int], MONTHS_PER_YEAR))) } catch { case _: ArithmeticException => - throw QueryExecutionErrors.datetimeIntervalArithmeticOverflowError() + throw QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError() } } @@ -493,13 +493,14 @@ case class MakeYMInterval(years: Expression, months: Expression) nullSafeCodeGen(ctx, ev, (years, months) => { val math = classOf[Math].getName.stripSuffix("$") val errorContext = getContextOrNullCode(ctx) + // scalastyle:off line.size.limit s""" |try { - | ${ev.value} = $math.toIntExact( - | $math.addExact($months, $math.multiplyExact($years, $MONTHS_PER_YEAR))); + | ${ev.value} = $math.toIntExact($math.addExact($months, $math.multiplyExact($years, $MONTHS_PER_YEAR))); |} catch (java.lang.ArithmeticException e) { - | throw QueryExecutionErrors.datetimeIntervalArithmeticOverflowError($errorContext); + | throw QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError($errorContext); |}""".stripMargin + // scalastyle:on line.size.limit }) } @@ -622,7 +623,7 @@ trait IntervalDivide { context: QueryContext): Unit = { if (value == minValue && num.dataType.isInstanceOf[IntegralType]) { if (numValue.asInstanceOf[Number].longValue() == -1) { - throw QueryExecutionErrors.numericIntervalArithmeticOverflowError( + throw QueryExecutionErrors.withTrySuggestionIntervalArithmeticOverflowError( "Interval value overflows after being divided by -1", "try_divide", context) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala index b643083ef349..0d4296cf6882 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala @@ -40,7 +40,8 @@ object IntervalMathUtils { f } catch { case e: ArithmeticException => - throw QueryExecutionErrors.numericIntervalArithmeticOverflowError(e.getMessage, hint, null) + throw QueryExecutionErrors.withTrySuggestionIntervalArithmeticOverflowError( + e.getMessage, hint, null) } } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala index 0b82a9290f6a..d2c48cd77971 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala @@ -792,7 +792,7 @@ object IntervalUtils extends SparkIntervalUtils { micros } catch { case _: ArithmeticException => - throw QueryExecutionErrors.datetimeIntervalArithmeticOverflowError() + throw QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError() } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index a5a5e547d33c..d40a24faf3d9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -636,7 +636,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE summary = "") } - def numericIntervalArithmeticOverflowError( + def withTrySuggestionIntervalArithmeticOverflowError( message: String, hint: String = "", context: QueryContext = null): ArithmeticException = { @@ -644,7 +644,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE s" Use '$hint' to tolerate overflow and return NULL instead." } else "" new SparkArithmeticException( - errorClass = "INTERVAL_ARITHMETIC_OVERFLOW.NUMERIC", + errorClass = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", messageParameters = Map( "message" -> message, "alternative" -> alternative), @@ -652,10 +652,10 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE summary = getSummary(context)) } - def datetimeIntervalArithmeticOverflowError( + def withoutSuggestionIntervalArithmeticOverflowError( context: QueryContext = null): SparkArithmeticException = { new SparkArithmeticException( - errorClass = "INTERVAL_ARITHMETIC_OVERFLOW.DATETIME", + errorClass = "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION", messageParameters = Map(), context = getQueryContext(context), summary = getSummary(context)) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala index b6c41c79cd86..8fb72ad53062 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala @@ -316,7 +316,8 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { val secFrac = DateTimeTestUtils.secFrac(seconds, millis, micros) val durationExpr = MakeDTInterval(Literal(days), Literal(hours), Literal(minutes), Literal(Decimal(secFrac, Decimal.MAX_LONG_DIGITS, 6))) - checkExceptionInExpression[ArithmeticException](durationExpr, "ARITHMETIC_OVERFLOW.DATETIME") + checkExceptionInExpression[ArithmeticException]( + durationExpr, "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION") } check(millis = -123) @@ -528,7 +529,8 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { Seq(MakeYMInterval(Literal(178956970), Literal(8)), MakeYMInterval(Literal(-178956970), Literal(-9))) .foreach { ym => - checkExceptionInExpression[ArithmeticException](ym, "ARITHMETIC_OVERFLOW.DATETIME") + checkExceptionInExpression[ArithmeticException]( + ym, "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION") } def checkImplicitEvaluation(expr: Expression, value: Any): Unit = { From abd4255005ec28fef87c31065bc160495645e072 Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Thu, 7 Nov 2024 17:06:18 +0100 Subject: [PATCH 04/19] fix styletest for error-conditions.json --- .../resources/error/error-conditions.json | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 88fcf77902d9..3c18dab70960 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -2020,23 +2020,23 @@ ], "sqlState" : "XX000" }, - "INTERVAL_ARITHMETIC_OVERFLOW": { - "message": [ + "INTERVAL_ARITHMETIC_OVERFLOW" : { + "message" : [ "Integer overflow while operating with intervals." ], - "subClass": { - "WITH_TRY_SUGGESTION": { - "message": [ - ".." + "subClass" : { + "WITHOUT_SUGGESTION" : { + "message" : [ + "Try devising appropriate values for the interval parameters." ] }, - "WITHOUT_SUGGESTION": { - "message": [ - "" + "WITH_TRY_SUGGESTION" : { + "message" : [ + ".." ] } }, - "sqlState": "22015" + "sqlState" : "22015" }, "INTERVAL_DIVIDED_BY_ZERO" : { "message" : [ From de44e302e444ee2eedeb922c4175d9a22a53237a Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Thu, 7 Nov 2024 17:11:54 +0100 Subject: [PATCH 05/19] WITH_TRY_SUGGESTION without message --- common/utils/src/main/resources/error/error-conditions.json | 2 +- .../org/apache/spark/sql/errors/QueryExecutionErrors.scala | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 3c18dab70960..b0f908fce16d 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -2032,7 +2032,7 @@ }, "WITH_TRY_SUGGESTION" : { "message" : [ - ".." + "." ] } }, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index c1fd33c9d185..e3b49b16627c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -645,9 +645,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE } else "" new SparkArithmeticException( errorClass = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", - messageParameters = Map( - "message" -> message, - "alternative" -> alternative), + messageParameters = Map("alternative" -> alternative), context = getQueryContext(context), summary = getSummary(context)) } From 3b94f96f6802b0af19aaf234745b3023956af7e8 Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Fri, 8 Nov 2024 11:31:52 +0100 Subject: [PATCH 06/19] fix tests --- .../apache/spark/sql/DataFrameAggregateSuite.scala | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala index 7ebcb280def6..a8f50d9ddce1 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala @@ -1488,12 +1488,14 @@ class DataFrameAggregateSuite extends QueryTest val error = intercept[SparkArithmeticException] { checkAnswer(df2.select(sum($"year-month")), Nil) } - assert(error.getMessage contains "[INTERVAL_ARITHMETIC_OVERFLOW] integer overflow") + assert(error.getMessage contains "[INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION]") + assert(error.getMessage contains "try_add") val error2 = intercept[SparkArithmeticException] { checkAnswer(df2.select(sum($"day")), Nil) } - assert(error2.getMessage contains "[INTERVAL_ARITHMETIC_OVERFLOW] long overflow") + assert(error.getMessage contains "[INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION]") + assert(error.getMessage contains "try_add") } test("SPARK-34837: Support ANSI SQL intervals by the aggregate function `avg`") { @@ -1623,12 +1625,14 @@ class DataFrameAggregateSuite extends QueryTest val error = intercept[SparkArithmeticException] { checkAnswer(df2.select(avg($"year-month")), Nil) } - assert(error.getMessage contains "[INTERVAL_ARITHMETIC_OVERFLOW] integer overflow") + assert(error.getMessage contains "[INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION]") + assert(error.getMessage contains "try_add") val error2 = intercept[SparkArithmeticException] { checkAnswer(df2.select(avg($"day")), Nil) } - assert(error2.getMessage contains "[INTERVAL_ARITHMETIC_OVERFLOW] long overflow") + assert(error.getMessage contains "[INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION]") + assert(error.getMessage contains "try_add") val df3 = intervalData.filter($"class" > 4) val avgDF3 = df3.select(avg($"year-month"), avg($"day")) From d7352a479838a5b92357447ebee98cfe38246a9a Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Fri, 8 Nov 2024 13:50:44 +0100 Subject: [PATCH 07/19] Better tests, return back message field because of 'Divide by -1' part --- .../resources/error/error-conditions.json | 2 +- .../expressions/intervalExpressions.scala | 2 +- .../sql/catalyst/util/IntervalMathUtils.scala | 2 +- .../sql/errors/QueryExecutionErrors.scala | 4 ++- .../spark/sql/DataFrameAggregateSuite.scala | 31 +++++++++++++------ 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index b0f908fce16d..3c18dab70960 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -2032,7 +2032,7 @@ }, "WITH_TRY_SUGGESTION" : { "message" : [ - "." + ".." ] } }, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala index 2435c0f7b565..2c94b4b4066a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala @@ -568,7 +568,7 @@ case class MakeYMInterval(years: Expression, months: Expression) Math.multiplyExact(year.asInstanceOf[Int], MONTHS_PER_YEAR))) } catch { case _: ArithmeticException => - throw QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError() + throw QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError(origin.context) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala index 0d4296cf6882..9b3783ac437b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala @@ -41,7 +41,7 @@ object IntervalMathUtils { } catch { case e: ArithmeticException => throw QueryExecutionErrors.withTrySuggestionIntervalArithmeticOverflowError( - e.getMessage, hint, null) + "", hint, null) } } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index ceda4c6fcfd3..60ae4196c800 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -645,7 +645,9 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE } else "" new SparkArithmeticException( errorClass = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", - messageParameters = Map("alternative" -> alternative), + messageParameters = Map( + "message" -> message, + "alternative" -> alternative), context = getQueryContext(context), summary = getSummary(context)) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala index a8f50d9ddce1..7209df2fa87e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala @@ -1485,17 +1485,28 @@ class DataFrameAggregateSuite extends QueryTest val df2 = Seq((Period.ofMonths(Int.MaxValue), Duration.ofDays(106751991)), (Period.ofMonths(10), Duration.ofDays(10))) .toDF("year-month", "day") - val error = intercept[SparkArithmeticException] { - checkAnswer(df2.select(sum($"year-month")), Nil) - } - assert(error.getMessage contains "[INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION]") - assert(error.getMessage contains "try_add") - val error2 = intercept[SparkArithmeticException] { - checkAnswer(df2.select(sum($"day")), Nil) - } - assert(error.getMessage contains "[INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION]") - assert(error.getMessage contains "try_add") + checkError( + exception = intercept[SparkArithmeticException] { + checkAnswer(df2.select(sum($"year-month")), Nil) + }, + condition = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", + parameters = Map( + "message" -> "", + "alternative" -> " Use 'try_add' to tolerate overflow and return NULL instead." + ) + ) + + checkError( + exception = intercept[SparkArithmeticException] { + checkAnswer(df2.select(sum($"day")), Nil) + }, + condition = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", + parameters = Map( + "message" -> "", + "alternative" -> " Use 'try_add' to tolerate overflow and return NULL instead." + ) + ) } test("SPARK-34837: Support ANSI SQL intervals by the aggregate function `avg`") { From 5508b9c3985e98170446e5182f0c8c4014562dcf Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Fri, 8 Nov 2024 15:24:09 +0100 Subject: [PATCH 08/19] fix tests, regenerate autogenerated files --- .../sql-tests/results/ansi/interval.sql.out | 55 +++++++++++++------ .../sql-tests/results/interval.sql.out | 55 +++++++++++++------ 2 files changed, 78 insertions(+), 32 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out index 766bfba7696f..9d49992f5e2d 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out @@ -936,8 +936,11 @@ select make_dt_interval(2147483647) -- !query schema struct<> -- !query output -java.lang.ArithmeticException -long overflow +org.apache.spark.SparkArithmeticException +{ + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION", + "sqlState" : "22015" +} -- !query @@ -977,8 +980,18 @@ select make_ym_interval(178956970, 8) -- !query schema struct<> -- !query output -java.lang.ArithmeticException -integer overflow +org.apache.spark.SparkArithmeticException +{ + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION", + "sqlState" : "22015", + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 37, + "fragment" : "make_ym_interval(178956970, 8)" + } ] +} -- !query @@ -994,8 +1007,18 @@ select make_ym_interval(-178956970, -9) -- !query schema struct<> -- !query output -java.lang.ArithmeticException -integer overflow +org.apache.spark.SparkArithmeticException +{ + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION", + "sqlState" : "22015", + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 39, + "fragment" : "make_ym_interval(-178956970, -9)" + } ] +} -- !query @@ -2493,11 +2516,11 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { "alternative" : "", - "message" : "integer overflow" + "message" : "" } } @@ -2509,11 +2532,11 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { "alternative" : " Use 'try_subtract' to tolerate overflow and return NULL instead.", - "message" : "integer overflow" + "message" : "" } } @@ -2525,11 +2548,11 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { "alternative" : " Use 'try_add' to tolerate overflow and return NULL instead.", - "message" : "integer overflow" + "message" : "" } } @@ -2838,7 +2861,7 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { "alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.", @@ -2861,7 +2884,7 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { "alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.", @@ -2918,7 +2941,7 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { "alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.", @@ -2941,7 +2964,7 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { "alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.", diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out index 7eed2d42da04..c0a247b20aad 100644 --- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out @@ -823,8 +823,11 @@ select make_dt_interval(2147483647) -- !query schema struct<> -- !query output -java.lang.ArithmeticException -long overflow +org.apache.spark.SparkArithmeticException +{ + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION", + "sqlState" : "22015" +} -- !query @@ -864,8 +867,18 @@ select make_ym_interval(178956970, 8) -- !query schema struct<> -- !query output -java.lang.ArithmeticException -integer overflow +org.apache.spark.SparkArithmeticException +{ + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION", + "sqlState" : "22015", + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 37, + "fragment" : "make_ym_interval(178956970, 8)" + } ] +} -- !query @@ -881,8 +894,18 @@ select make_ym_interval(-178956970, -9) -- !query schema struct<> -- !query output -java.lang.ArithmeticException -integer overflow +org.apache.spark.SparkArithmeticException +{ + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION", + "sqlState" : "22015", + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 39, + "fragment" : "make_ym_interval(-178956970, -9)" + } ] +} -- !query @@ -2316,11 +2339,11 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { "alternative" : "", - "message" : "integer overflow" + "message" : "" } } @@ -2332,11 +2355,11 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { "alternative" : " Use 'try_subtract' to tolerate overflow and return NULL instead.", - "message" : "integer overflow" + "message" : "" } } @@ -2348,11 +2371,11 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { "alternative" : " Use 'try_add' to tolerate overflow and return NULL instead.", - "message" : "integer overflow" + "message" : "" } } @@ -2661,7 +2684,7 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { "alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.", @@ -2684,7 +2707,7 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { "alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.", @@ -2741,7 +2764,7 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { "alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.", @@ -2764,7 +2787,7 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { "alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.", From 6a2ed0bf33c69c4bbf88e55848bebfc5ed2e0b53 Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Fri, 8 Nov 2024 17:14:13 +0100 Subject: [PATCH 09/19] Use another error message types and remove division by -1 text --- .../src/main/resources/error/error-conditions.json | 2 +- .../sql/catalyst/expressions/intervalExpressions.scala | 2 +- .../spark/sql/catalyst/util/IntervalMathUtils.scala | 10 +++++----- .../apache/spark/sql/errors/QueryExecutionErrors.scala | 10 ++-------- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 3c18dab70960..74e205511beb 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -2032,7 +2032,7 @@ }, "WITH_TRY_SUGGESTION" : { "message" : [ - ".." + "Use to tolerate overflow and return NULL instead." ] } }, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala index 2c94b4b4066a..17e0cd7bf407 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala @@ -707,7 +707,7 @@ trait IntervalDivide { if (value == minValue && num.dataType.isInstanceOf[IntegralType]) { if (numValue.asInstanceOf[Number].longValue() == -1) { throw QueryExecutionErrors.withTrySuggestionIntervalArithmeticOverflowError( - "Interval value overflows after being divided by -1", "try_divide", context) + "try_divide", context) } } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala index 9b3783ac437b..2d9f4c43491d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala @@ -31,17 +31,17 @@ object IntervalMathUtils { def subtractExact(a: Long, b: Long): Long = withOverflow(Math.subtractExact(a, b), "try_subtract") - def negateExact(a: Int): Int = withOverflow(Math.negateExact(a)) + def negateExact(a: Int): Int = withOverflow(Math.negateExact(a), "try_multiply") - def negateExact(a: Long): Long = withOverflow(Math.negateExact(a)) + def negateExact(a: Long): Long = withOverflow(Math.negateExact(a), "try_multiply") - private def withOverflow[A](f: => A, hint: String = ""): A = { + private def withOverflow[A](f: => A, suggestedFunc: String): A = { try { f } catch { - case e: ArithmeticException => + case _: ArithmeticException => throw QueryExecutionErrors.withTrySuggestionIntervalArithmeticOverflowError( - "", hint, null) + suggestedFunc, null) } } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index 60ae4196c800..666a316014ca 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -637,17 +637,11 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE } def withTrySuggestionIntervalArithmeticOverflowError( - message: String, - hint: String = "", + suggestedFunc: String, context: QueryContext = null): ArithmeticException = { - val alternative = if (hint.nonEmpty) { - s" Use '$hint' to tolerate overflow and return NULL instead." - } else "" new SparkArithmeticException( errorClass = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", - messageParameters = Map( - "message" -> message, - "alternative" -> alternative), + messageParameters = Map("functionName" -> suggestedFunc), context = getQueryContext(context), summary = getSummary(context)) } From 68594ff38c1914ea638f4dca6b598abb77c00a17 Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Fri, 8 Nov 2024 17:25:28 +0100 Subject: [PATCH 10/19] Regenerate expected sql output --- .../sql-tests/results/ansi/interval.sql.out | 21 +++++++------------ .../sql-tests/results/interval.sql.out | 21 +++++++------------ 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out index 9d49992f5e2d..0e164d44f027 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out @@ -2519,8 +2519,7 @@ org.apache.spark.SparkArithmeticException "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "alternative" : "", - "message" : "" + "functionName" : "try_multiply" } } @@ -2535,8 +2534,7 @@ org.apache.spark.SparkArithmeticException "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "alternative" : " Use 'try_subtract' to tolerate overflow and return NULL instead.", - "message" : "" + "functionName" : "try_subtract" } } @@ -2551,8 +2549,7 @@ org.apache.spark.SparkArithmeticException "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "alternative" : " Use 'try_add' to tolerate overflow and return NULL instead.", - "message" : "" + "functionName" : "try_add" } } @@ -2864,8 +2861,7 @@ org.apache.spark.SparkArithmeticException "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.", - "message" : "Interval value overflows after being divided by -1" + "functionName" : "try_divide" }, "queryContext" : [ { "objectType" : "", @@ -2887,8 +2883,7 @@ org.apache.spark.SparkArithmeticException "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.", - "message" : "Interval value overflows after being divided by -1" + "functionName" : "try_divide" }, "queryContext" : [ { "objectType" : "", @@ -2944,8 +2939,7 @@ org.apache.spark.SparkArithmeticException "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.", - "message" : "Interval value overflows after being divided by -1" + "functionName" : "try_divide" }, "queryContext" : [ { "objectType" : "", @@ -2967,8 +2961,7 @@ org.apache.spark.SparkArithmeticException "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.", - "message" : "Interval value overflows after being divided by -1" + "functionName" : "try_divide" }, "queryContext" : [ { "objectType" : "", diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out index c0a247b20aad..7032571a16ab 100644 --- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out @@ -2342,8 +2342,7 @@ org.apache.spark.SparkArithmeticException "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "alternative" : "", - "message" : "" + "functionName" : "try_multiply" } } @@ -2358,8 +2357,7 @@ org.apache.spark.SparkArithmeticException "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "alternative" : " Use 'try_subtract' to tolerate overflow and return NULL instead.", - "message" : "" + "functionName" : "try_subtract" } } @@ -2374,8 +2372,7 @@ org.apache.spark.SparkArithmeticException "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "alternative" : " Use 'try_add' to tolerate overflow and return NULL instead.", - "message" : "" + "functionName" : "try_add" } } @@ -2687,8 +2684,7 @@ org.apache.spark.SparkArithmeticException "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.", - "message" : "Interval value overflows after being divided by -1" + "functionName" : "try_divide" }, "queryContext" : [ { "objectType" : "", @@ -2710,8 +2706,7 @@ org.apache.spark.SparkArithmeticException "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.", - "message" : "Interval value overflows after being divided by -1" + "functionName" : "try_divide" }, "queryContext" : [ { "objectType" : "", @@ -2767,8 +2762,7 @@ org.apache.spark.SparkArithmeticException "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.", - "message" : "Interval value overflows after being divided by -1" + "functionName" : "try_divide" }, "queryContext" : [ { "objectType" : "", @@ -2790,8 +2784,7 @@ org.apache.spark.SparkArithmeticException "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.", - "message" : "Interval value overflows after being divided by -1" + "functionName" : "try_divide" }, "queryContext" : [ { "objectType" : "", From 30c63f6f806b5060121d16c7519bcb7135d60bc0 Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Fri, 8 Nov 2024 17:38:32 +0100 Subject: [PATCH 11/19] Fix tests --- .../spark/sql/DataFrameAggregateSuite.scala | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala index 7209df2fa87e..54e6756dbf4b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala @@ -1491,10 +1491,7 @@ class DataFrameAggregateSuite extends QueryTest checkAnswer(df2.select(sum($"year-month")), Nil) }, condition = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", - parameters = Map( - "message" -> "", - "alternative" -> " Use 'try_add' to tolerate overflow and return NULL instead." - ) + parameters = Map("functionName" -> "try_add") ) checkError( @@ -1502,10 +1499,7 @@ class DataFrameAggregateSuite extends QueryTest checkAnswer(df2.select(sum($"day")), Nil) }, condition = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", - parameters = Map( - "message" -> "", - "alternative" -> " Use 'try_add' to tolerate overflow and return NULL instead." - ) + parameters = Map("functionName" -> "try_add") ) } @@ -1633,17 +1627,22 @@ class DataFrameAggregateSuite extends QueryTest val df2 = Seq((Period.ofMonths(Int.MaxValue), Duration.ofDays(106751991)), (Period.ofMonths(10), Duration.ofDays(10))) .toDF("year-month", "day") - val error = intercept[SparkArithmeticException] { - checkAnswer(df2.select(avg($"year-month")), Nil) - } - assert(error.getMessage contains "[INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION]") - assert(error.getMessage contains "try_add") - val error2 = intercept[SparkArithmeticException] { - checkAnswer(df2.select(avg($"day")), Nil) - } - assert(error.getMessage contains "[INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION]") - assert(error.getMessage contains "try_add") + checkError( + exception = intercept[SparkArithmeticException] { + checkAnswer(df2.select(avg($"year-month")), Nil) + }, + condition = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", + parameters = Map("functionName" -> "try_add") + ) + + checkError( + exception = intercept[SparkArithmeticException] { + checkAnswer(df2.select(avg($"day")), Nil) + }, + condition = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", + parameters = Map("functionName" -> "try_add") + ) val df3 = intervalData.filter($"class" > 4) val avgDF3 = df3.select(avg($"year-month"), avg($"day")) From 8388de2a276116a3cbe5a856f06dad0dc3e1a6e5 Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Fri, 8 Nov 2024 17:44:39 +0100 Subject: [PATCH 12/19] Enforce setting context --- .../apache/spark/sql/catalyst/util/IntervalMathUtils.scala | 2 +- .../org/apache/spark/sql/catalyst/util/IntervalUtils.scala | 2 +- .../org/apache/spark/sql/errors/QueryExecutionErrors.scala | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala index 2d9f4c43491d..93ac734b17a5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala @@ -41,7 +41,7 @@ object IntervalMathUtils { } catch { case _: ArithmeticException => throw QueryExecutionErrors.withTrySuggestionIntervalArithmeticOverflowError( - suggestedFunc, null) + suggestedFunc, context = null) } } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala index d2c48cd77971..9995789ffd6b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala @@ -792,7 +792,7 @@ object IntervalUtils extends SparkIntervalUtils { micros } catch { case _: ArithmeticException => - throw QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError() + throw QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError(context = null) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index 666a316014ca..f3ec8f869b75 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -638,7 +638,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE def withTrySuggestionIntervalArithmeticOverflowError( suggestedFunc: String, - context: QueryContext = null): ArithmeticException = { + context: QueryContext): ArithmeticException = { new SparkArithmeticException( errorClass = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", messageParameters = Map("functionName" -> suggestedFunc), @@ -647,7 +647,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE } def withoutSuggestionIntervalArithmeticOverflowError( - context: QueryContext = null): SparkArithmeticException = { + context: QueryContext): SparkArithmeticException = { new SparkArithmeticException( errorClass = "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION", messageParameters = Map(), From 4f72f7ee0f4af7803ebfa41359799d596281b016 Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Tue, 12 Nov 2024 12:23:47 +0100 Subject: [PATCH 13/19] remove TRY --- common/utils/src/main/resources/error/error-conditions.json | 2 +- .../spark/sql/catalyst/expressions/intervalExpressions.scala | 2 +- .../apache/spark/sql/catalyst/util/IntervalMathUtils.scala | 2 +- .../org/apache/spark/sql/errors/QueryExecutionErrors.scala | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 74e205511beb..c63ac02d9779 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -2030,7 +2030,7 @@ "Try devising appropriate values for the interval parameters." ] }, - "WITH_TRY_SUGGESTION" : { + "WITH_SUGGESTION" : { "message" : [ "Use to tolerate overflow and return NULL instead." ] diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala index 17e0cd7bf407..5d4544cf09f5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala @@ -706,7 +706,7 @@ trait IntervalDivide { context: QueryContext): Unit = { if (value == minValue && num.dataType.isInstanceOf[IntegralType]) { if (numValue.asInstanceOf[Number].longValue() == -1) { - throw QueryExecutionErrors.withTrySuggestionIntervalArithmeticOverflowError( + throw QueryExecutionErrors.withSuggestionIntervalArithmeticOverflowError( "try_divide", context) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala index 93ac734b17a5..c6f347e72fa0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala @@ -40,7 +40,7 @@ object IntervalMathUtils { f } catch { case _: ArithmeticException => - throw QueryExecutionErrors.withTrySuggestionIntervalArithmeticOverflowError( + throw QueryExecutionErrors.withSuggestionIntervalArithmeticOverflowError( suggestedFunc, context = null) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index f3ec8f869b75..9340fa38d5de 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -636,11 +636,11 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE summary = "") } - def withTrySuggestionIntervalArithmeticOverflowError( + def withSuggestionIntervalArithmeticOverflowError( suggestedFunc: String, context: QueryContext): ArithmeticException = { new SparkArithmeticException( - errorClass = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", + errorClass = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", messageParameters = Map("functionName" -> suggestedFunc), context = getQueryContext(context), summary = getSummary(context)) From 494d6489356df6640f27dbb682a4020a6e913c4f Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Tue, 12 Nov 2024 12:45:54 +0100 Subject: [PATCH 14/19] fix PR issues --- .../spark/sql/catalyst/util/IntervalMathUtils.scala | 10 +++++++--- .../apache/spark/sql/errors/QueryExecutionErrors.scala | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala index c6f347e72fa0..7ccbad137e91 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala @@ -31,15 +31,19 @@ object IntervalMathUtils { def subtractExact(a: Long, b: Long): Long = withOverflow(Math.subtractExact(a, b), "try_subtract") - def negateExact(a: Int): Int = withOverflow(Math.negateExact(a), "try_multiply") + def negateExact(a: Int): Int = withOverflow(Math.negateExact(a)) - def negateExact(a: Long): Long = withOverflow(Math.negateExact(a), "try_multiply") + def negateExact(a: Long): Long = withOverflow(Math.negateExact(a)) - private def withOverflow[A](f: => A, suggestedFunc: String): A = { + private def withOverflow[A](f: => A, suggestedFunc: String = ""): A = { try { f } catch { case _: ArithmeticException => + if (suggestedFunc.isEmpty) { + throw QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError( + context = null) + } throw QueryExecutionErrors.withSuggestionIntervalArithmeticOverflowError( suggestedFunc, context = null) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index 9340fa38d5de..6f79a392e5e7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -641,7 +641,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE context: QueryContext): ArithmeticException = { new SparkArithmeticException( errorClass = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", - messageParameters = Map("functionName" -> suggestedFunc), + messageParameters = Map("functionName" -> toSQLId(suggestedFunc)), context = getQueryContext(context), summary = getSummary(context)) } From b042247f322b088265fe7689020ef573a2582485 Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Tue, 12 Nov 2024 12:50:02 +0100 Subject: [PATCH 15/19] recanonize tests --- .../sql-tests/results/ansi/interval.sql.out | 31 +++++++++---------- .../sql-tests/results/interval.sql.out | 31 +++++++++---------- 2 files changed, 28 insertions(+), 34 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out index 0e164d44f027..dd48ff1a2580 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out @@ -2516,11 +2516,8 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", - "sqlState" : "22015", - "messageParameters" : { - "functionName" : "try_multiply" - } + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION", + "sqlState" : "22015" } @@ -2531,10 +2528,10 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "functionName" : "try_subtract" + "functionName" : "`try_subtract`" } } @@ -2546,10 +2543,10 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "functionName" : "try_add" + "functionName" : "`try_add`" } } @@ -2858,10 +2855,10 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "functionName" : "try_divide" + "functionName" : "`try_divide`" }, "queryContext" : [ { "objectType" : "", @@ -2880,10 +2877,10 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "functionName" : "try_divide" + "functionName" : "`try_divide`" }, "queryContext" : [ { "objectType" : "", @@ -2936,10 +2933,10 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "functionName" : "try_divide" + "functionName" : "`try_divide`" }, "queryContext" : [ { "objectType" : "", @@ -2958,10 +2955,10 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "functionName" : "try_divide" + "functionName" : "`try_divide`" }, "queryContext" : [ { "objectType" : "", diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out index 7032571a16ab..5418597427f5 100644 --- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out @@ -2339,11 +2339,8 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", - "sqlState" : "22015", - "messageParameters" : { - "functionName" : "try_multiply" - } + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION", + "sqlState" : "22015" } @@ -2354,10 +2351,10 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "functionName" : "try_subtract" + "functionName" : "`try_subtract`" } } @@ -2369,10 +2366,10 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "functionName" : "try_add" + "functionName" : "`try_add`" } } @@ -2681,10 +2678,10 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "functionName" : "try_divide" + "functionName" : "`try_divide`" }, "queryContext" : [ { "objectType" : "", @@ -2703,10 +2700,10 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "functionName" : "try_divide" + "functionName" : "`try_divide`" }, "queryContext" : [ { "objectType" : "", @@ -2759,10 +2756,10 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "functionName" : "try_divide" + "functionName" : "`try_divide`" }, "queryContext" : [ { "objectType" : "", @@ -2781,10 +2778,10 @@ struct<> -- !query output org.apache.spark.SparkArithmeticException { - "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", + "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", "sqlState" : "22015", "messageParameters" : { - "functionName" : "try_divide" + "functionName" : "`try_divide`" }, "queryContext" : [ { "objectType" : "", From 0fcd3ccf5037bcf39458d280aa3f822bbe67ce0e Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Tue, 12 Nov 2024 13:05:42 +0100 Subject: [PATCH 16/19] fix tests --- .../spark/sql/DataFrameAggregateSuite.scala | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala index 54e6756dbf4b..6348e5f31539 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala @@ -26,6 +26,7 @@ import org.scalatest.matchers.must.Matchers.the import org.apache.spark.{SparkArithmeticException, SparkRuntimeException} import org.apache.spark.sql.catalyst.plans.logical.Expand import org.apache.spark.sql.catalyst.util.AUTO_GENERATED_ALIAS +import org.apache.spark.sql.errors.DataTypeErrors.toSQLId import org.apache.spark.sql.execution.WholeStageCodegenExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec} @@ -1490,16 +1491,16 @@ class DataFrameAggregateSuite extends QueryTest exception = intercept[SparkArithmeticException] { checkAnswer(df2.select(sum($"year-month")), Nil) }, - condition = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", - parameters = Map("functionName" -> "try_add") + condition = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", + parameters = Map("functionName" -> toSQLId("try_add")) ) checkError( exception = intercept[SparkArithmeticException] { checkAnswer(df2.select(sum($"day")), Nil) }, - condition = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", - parameters = Map("functionName" -> "try_add") + condition = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", + parameters = Map("functionName" -> toSQLId("try_add")) ) } @@ -1632,16 +1633,16 @@ class DataFrameAggregateSuite extends QueryTest exception = intercept[SparkArithmeticException] { checkAnswer(df2.select(avg($"year-month")), Nil) }, - condition = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", - parameters = Map("functionName" -> "try_add") + condition = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", + parameters = Map("functionName" -> toSQLId("try_add")) ) checkError( exception = intercept[SparkArithmeticException] { checkAnswer(df2.select(avg($"day")), Nil) }, - condition = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_TRY_SUGGESTION", - parameters = Map("functionName" -> "try_add") + condition = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION", + parameters = Map("functionName" -> toSQLId("try_add")) ) val df3 = intervalData.filter($"class" > 4) From 29253639ff873c138f0b96a435abd0431c4d8811 Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Wed, 13 Nov 2024 01:01:24 +0100 Subject: [PATCH 17/19] add err context --- .../catalyst/expressions/intervalExpressions.scala | 11 ++++++++--- .../spark/sql/catalyst/util/IntervalUtils.scala | 7 ++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala index 5d4544cf09f5..fd9ec38c7b63 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala @@ -476,7 +476,8 @@ case class MakeDTInterval( hours: Expression, mins: Expression, secs: Expression) - extends QuaternaryExpression with ImplicitCastInputTypes with NullIntolerant { + extends QuaternaryExpression with ImplicitCastInputTypes with NullIntolerant + with SupportQueryContext { def this( days: Expression, @@ -508,13 +509,15 @@ case class MakeDTInterval( day.asInstanceOf[Int], hour.asInstanceOf[Int], min.asInstanceOf[Int], - sec.asInstanceOf[Decimal]) + sec.asInstanceOf[Decimal], + origin.context) } override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { defineCodeGen(ctx, ev, (day, hour, min, sec) => { + val errorContext = getContextOrNullCode(ctx) val iu = IntervalUtils.getClass.getName.stripSuffix("$") - s"$iu.makeDayTimeInterval($day, $hour, $min, $sec)" + s"$iu.makeDayTimeInterval($day, $hour, $min, $sec, $errorContext)" }) } @@ -526,6 +529,8 @@ case class MakeDTInterval( mins: Expression, secs: Expression): MakeDTInterval = copy(days, hours, mins, secs) + + override def initQueryContext(): Option[QueryContext] = Some(origin.context) } @ExpressionDescription( diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala index 9995789ffd6b..39a07990dea3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala @@ -22,7 +22,7 @@ import java.util.concurrent.TimeUnit import scala.util.control.NonFatal -import org.apache.spark.{SparkIllegalArgumentException, SparkThrowable} +import org.apache.spark.{QueryContext, SparkIllegalArgumentException, SparkThrowable} import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.catalyst.parser.CatalystSqlParser import org.apache.spark.sql.catalyst.util.DateTimeConstants._ @@ -782,7 +782,8 @@ object IntervalUtils extends SparkIntervalUtils { days: Int, hours: Int, mins: Int, - secs: Decimal): Long = { + secs: Decimal, + context: QueryContext): Long = { assert(secs.scale == 6, "Seconds fractional must have 6 digits for microseconds") var micros = secs.toUnscaledLong try { @@ -792,7 +793,7 @@ object IntervalUtils extends SparkIntervalUtils { micros } catch { case _: ArithmeticException => - throw QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError(context = null) + throw QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError(context) } } From 2d84c955836da0d061f393f56d762c516d33efd8 Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Wed, 13 Nov 2024 01:03:31 +0100 Subject: [PATCH 18/19] recanonize tests --- .../resources/sql-tests/results/ansi/interval.sql.out | 9 ++++++++- .../test/resources/sql-tests/results/interval.sql.out | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out index dd48ff1a2580..4e012df792de 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out @@ -939,7 +939,14 @@ struct<> org.apache.spark.SparkArithmeticException { "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION", - "sqlState" : "22015" + "sqlState" : "22015", + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 35, + "fragment" : "make_dt_interval(2147483647)" + } ] } diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out index 5418597427f5..a8a0423bdb3e 100644 --- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out @@ -826,7 +826,14 @@ struct<> org.apache.spark.SparkArithmeticException { "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION", - "sqlState" : "22015" + "sqlState" : "22015", + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 35, + "fragment" : "make_dt_interval(2147483647)" + } ] } From 2a367499a5a0877a80a69a83502ce44e97fd452a Mon Sep 17 00:00:00 2001 From: Ruzel Ibragimov Date: Wed, 13 Nov 2024 11:04:36 +0100 Subject: [PATCH 19/19] fix PR issues --- .../apache/spark/sql/catalyst/util/IntervalMathUtils.scala | 6 ++---- .../org/apache/spark/sql/errors/QueryExecutionErrors.scala | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala index 7ccbad137e91..756f2598f13f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalMathUtils.scala @@ -39,11 +39,9 @@ object IntervalMathUtils { try { f } catch { + case _: ArithmeticException if suggestedFunc.isEmpty => + throw QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError(context = null) case _: ArithmeticException => - if (suggestedFunc.isEmpty) { - throw QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError( - context = null) - } throw QueryExecutionErrors.withSuggestionIntervalArithmeticOverflowError( suggestedFunc, context = null) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index 6c7bfdd5a1c2..fb39d3c5d7c6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -655,7 +655,6 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE summary = getSummary(context)) } - def failedToCompileMsg(e: Exception): String = { s"Failed to compile: $e" }