Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
3426c34
Throw ARITHMETIC_OVERFLOW in MakeYMInterval, MakeDTInterval
gotocoding-DB Nov 6, 2024
908e745
Add another type of exception because we can't use current ARITHMETIC…
gotocoding-DB Nov 6, 2024
b5cd32a
better error message
gotocoding-DB Nov 7, 2024
2fa9f9d
Merge branch 'master' into SPARK-50226-overflow-error
gotocoding-DB Nov 7, 2024
abd4255
fix styletest for error-conditions.json
gotocoding-DB Nov 7, 2024
de44e30
WITH_TRY_SUGGESTION without message
gotocoding-DB Nov 7, 2024
87e3c5c
Merge remote-tracking branch 'upstream/master' into SPARK-50226-overf…
gotocoding-DB Nov 8, 2024
3b94f96
fix tests
gotocoding-DB Nov 8, 2024
d7352a4
Better tests, return back message field because of 'Divide by -1' part
gotocoding-DB Nov 8, 2024
5508b9c
fix tests, regenerate autogenerated files
gotocoding-DB Nov 8, 2024
6a2ed0b
Use another error message types and remove division by -1 text
gotocoding-DB Nov 8, 2024
68594ff
Regenerate expected sql output
gotocoding-DB Nov 8, 2024
30c63f6
Fix tests
gotocoding-DB Nov 8, 2024
8388de2
Enforce setting context
gotocoding-DB Nov 8, 2024
4f72f7e
remove TRY
gotocoding-DB Nov 12, 2024
9ed5eb2
Merge remote-tracking branch 'upstream/master' into SPARK-50226-overf…
gotocoding-DB Nov 12, 2024
494d648
fix PR issues
gotocoding-DB Nov 12, 2024
b042247
recanonize tests
gotocoding-DB Nov 12, 2024
0fcd3cc
fix tests
gotocoding-DB Nov 12, 2024
2925363
add err context
gotocoding-DB Nov 13, 2024
2d84c95
recanonize tests
gotocoding-DB Nov 13, 2024
c2d2490
Merge remote-tracking branch 'upstream/master' into SPARK-50226-overf…
gotocoding-DB Nov 13, 2024
2a36749
fix PR issues
gotocoding-DB Nov 13, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion common/utils/src/main/resources/error/error-conditions.json
Original file line number Diff line number Diff line change
Expand Up @@ -2012,8 +2012,20 @@
},
"INTERVAL_ARITHMETIC_OVERFLOW" : {
"message" : [
"<message>.<alternative>"
"Integer overflow while operating with intervals."
],
"subClass" : {
"WITHOUT_SUGGESTION" : {
"message" : [
"Try devising appropriate values for the interval parameters."
]
},
"WITH_SUGGESTION" : {
"message" : [
"Use <functionName> to tolerate overflow and return NULL instead."
]
}
},
"sqlState" : "22015"
},
"INTERVAL_DIVIDED_BY_ZERO" : {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -481,7 +481,7 @@ case class MakeDTInterval(
hours: Expression,
mins: Expression,
secs: Expression)
extends QuaternaryExpression with ImplicitCastInputTypes {
extends QuaternaryExpression with ImplicitCastInputTypes with SupportQueryContext {
override def nullIntolerant: Boolean = true

def this(
Expand Down Expand Up @@ -514,13 +514,15 @@ case class MakeDTInterval(
day.asInstanceOf[Int],
hour.asInstanceOf[Int],
min.asInstanceOf[Int],
sec.asInstanceOf[Decimal])
sec.asInstanceOf[Decimal],
origin.context)
}

override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
defineCodeGen(ctx, ev, (day, hour, min, sec) => {
val errorContext = getContextOrNullCode(ctx)
val iu = IntervalUtils.getClass.getName.stripSuffix("$")
s"$iu.makeDayTimeInterval($day, $hour, $min, $sec)"
s"$iu.makeDayTimeInterval($day, $hour, $min, $sec, $errorContext)"
})
}

Expand All @@ -532,6 +534,8 @@ case class MakeDTInterval(
mins: Expression,
secs: Expression): MakeDTInterval =
copy(days, hours, mins, secs)

override def initQueryContext(): Option[QueryContext] = Some(origin.context)
}

@ExpressionDescription(
Expand All @@ -556,7 +560,7 @@ case class MakeDTInterval(
group = "datetime_funcs")
// scalastyle:on line.size.limit
case class MakeYMInterval(years: Expression, months: Expression)
extends BinaryExpression with ImplicitCastInputTypes with Serializable {
extends BinaryExpression with ImplicitCastInputTypes with Serializable with SupportQueryContext {
override def nullIntolerant: Boolean = true

def this(years: Expression) = this(years, Literal(0))
Expand All @@ -568,17 +572,28 @@ case class MakeYMInterval(years: Expression, months: Expression)
override def dataType: DataType = YearMonthIntervalType()

override def nullSafeEval(year: Any, month: Any): Any = {
Math.toIntExact(Math.addExact(month.asInstanceOf[Number].longValue(),
Math.multiplyExact(year.asInstanceOf[Number].longValue(), MONTHS_PER_YEAR)))
try {
Math.toIntExact(
Math.addExact(month.asInstanceOf[Int],
Math.multiplyExact(year.asInstanceOf[Int], MONTHS_PER_YEAR)))
} catch {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

shall we add IntervalUtils.makeYearMonthInterval so that we duplicate less code between interpreted and codegen?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Off course we should deduplicate it, but I didn't succeed with it: #48773 (comment).
Max suggested to use StaticInvoke, I'll try it in separate PR.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there is already makeDayTimeInterval and we can just follow it. Can you try it again and create a PR? We can help to check the code together.

Copy link
Contributor Author

@gotocoding-DB gotocoding-DB Nov 14, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, I'll do it a bit later.
UPD: #48848

case _: ArithmeticException =>
throw QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError(origin.context)
}
}

override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
defineCodeGen(ctx, ev, (years, months) => {
nullSafeCodeGen(ctx, ev, (years, months) => {
val math = classOf[Math].getName.stripSuffix("$")
val errorContext = getContextOrNullCode(ctx)
// scalastyle:off line.size.limit
s"""
|$math.toIntExact(java.lang.Math.addExact($months,
| $math.multiplyExact($years, $MONTHS_PER_YEAR)))
|""".stripMargin
|try {
| ${ev.value} = $math.toIntExact($math.addExact($months, $math.multiplyExact($years, $MONTHS_PER_YEAR)));
|} catch (java.lang.ArithmeticException e) {
| throw QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError($errorContext);
|}""".stripMargin
// scalastyle:on line.size.limit
})
}

Expand All @@ -587,6 +602,10 @@ case class MakeYMInterval(years: Expression, months: Expression)
override protected def withNewChildrenInternal(
newLeft: Expression, newRight: Expression): Expression =
copy(years = newLeft, months = newRight)

override def initQueryContext(): Option[QueryContext] = {
Some(origin.context)
}
}

// Multiply an year-month interval by a numeric
Expand Down Expand Up @@ -699,8 +718,8 @@ trait IntervalDivide {
context: QueryContext): Unit = {
if (value == minValue && num.dataType.isInstanceOf[IntegralType]) {
if (numValue.asInstanceOf[Number].longValue() == -1) {
throw QueryExecutionErrors.intervalArithmeticOverflowError(
"Interval value overflows after being divided by -1", "try_divide", context)
throw QueryExecutionErrors.withSuggestionIntervalArithmeticOverflowError(
"try_divide", context)
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,15 @@ object IntervalMathUtils {

def negateExact(a: Long): Long = withOverflow(Math.negateExact(a))

private def withOverflow[A](f: => A, hint: String = ""): A = {
private def withOverflow[A](f: => A, suggestedFunc: String = ""): A = {
try {
f
} catch {
case e: ArithmeticException =>
throw QueryExecutionErrors.intervalArithmeticOverflowError(e.getMessage, hint, null)
case _: ArithmeticException if suggestedFunc.isEmpty =>
throw QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError(context = null)
case _: ArithmeticException =>
throw QueryExecutionErrors.withSuggestionIntervalArithmeticOverflowError(
suggestedFunc, context = null)
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import java.util.concurrent.TimeUnit

import scala.util.control.NonFatal

import org.apache.spark.{SparkIllegalArgumentException, SparkThrowable}
import org.apache.spark.{QueryContext, SparkIllegalArgumentException, SparkThrowable}
import org.apache.spark.sql.catalyst.expressions.Literal
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
import org.apache.spark.sql.catalyst.util.DateTimeConstants._
Expand Down Expand Up @@ -782,13 +782,19 @@ object IntervalUtils extends SparkIntervalUtils {
days: Int,
hours: Int,
mins: Int,
secs: Decimal): Long = {
secs: Decimal,
context: QueryContext): Long = {
assert(secs.scale == 6, "Seconds fractional must have 6 digits for microseconds")
var micros = secs.toUnscaledLong
micros = Math.addExact(micros, Math.multiplyExact(days, MICROS_PER_DAY))
micros = Math.addExact(micros, Math.multiplyExact(hours, MICROS_PER_HOUR))
micros = Math.addExact(micros, Math.multiplyExact(mins, MICROS_PER_MINUTE))
micros
try {
micros = Math.addExact(micros, Math.multiplyExact(days, MICROS_PER_DAY))
micros = Math.addExact(micros, Math.multiplyExact(hours, MICROS_PER_HOUR))
micros = Math.addExact(micros, Math.multiplyExact(mins, MICROS_PER_MINUTE))
micros
} catch {
case _: ArithmeticException =>
throw QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError(context)
}
}

def intToYearMonthInterval(v: Int, startField: Byte, endField: Byte): Int = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -636,18 +636,21 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE
summary = "")
}

def intervalArithmeticOverflowError(
message: String,
hint: String = "",
def withSuggestionIntervalArithmeticOverflowError(
suggestedFunc: String,
context: QueryContext): ArithmeticException = {
val alternative = if (hint.nonEmpty) {
s" Use '$hint' to tolerate overflow and return NULL instead."
} else ""
new SparkArithmeticException(
errorClass = "INTERVAL_ARITHMETIC_OVERFLOW",
messageParameters = Map(
"message" -> message,
"alternative" -> alternative),
errorClass = "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION",
messageParameters = Map("functionName" -> toSQLId(suggestedFunc)),
context = getQueryContext(context),
summary = getSummary(context))
}

def withoutSuggestionIntervalArithmeticOverflowError(
context: QueryContext): SparkArithmeticException = {
new SparkArithmeticException(
errorClass = "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
messageParameters = Map(),
context = getQueryContext(context),
summary = getSummary(context))
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,8 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
val secFrac = DateTimeTestUtils.secFrac(seconds, millis, micros)
val durationExpr = MakeDTInterval(Literal(days), Literal(hours), Literal(minutes),
Literal(Decimal(secFrac, Decimal.MAX_LONG_DIGITS, 6)))
checkExceptionInExpression[ArithmeticException](durationExpr, EmptyRow, "")
checkExceptionInExpression[ArithmeticException](
durationExpr, "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION")
}

check(millis = -123)
Expand Down Expand Up @@ -528,7 +529,8 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
Seq(MakeYMInterval(Literal(178956970), Literal(8)),
MakeYMInterval(Literal(-178956970), Literal(-9)))
.foreach { ym =>
checkExceptionInExpression[ArithmeticException](ym, "integer overflow")
checkExceptionInExpression[ArithmeticException](
ym, "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION")
}

def checkImplicitEvaluation(expr: Expression, value: Any): Unit = {
Expand Down
80 changes: 50 additions & 30 deletions sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
Original file line number Diff line number Diff line change
Expand Up @@ -936,8 +936,18 @@ select make_dt_interval(2147483647)
-- !query schema
struct<>
-- !query output
java.lang.ArithmeticException
long overflow
org.apache.spark.SparkArithmeticException
{
"errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
"sqlState" : "22015",
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
"startIndex" : 8,
"stopIndex" : 35,
"fragment" : "make_dt_interval(2147483647)"
} ]
}


-- !query
Expand Down Expand Up @@ -977,8 +987,18 @@ select make_ym_interval(178956970, 8)
-- !query schema
struct<>
-- !query output
java.lang.ArithmeticException
integer overflow
org.apache.spark.SparkArithmeticException
{
"errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
"sqlState" : "22015",
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
"startIndex" : 8,
"stopIndex" : 37,
"fragment" : "make_ym_interval(178956970, 8)"
} ]
}


-- !query
Expand All @@ -994,8 +1014,18 @@ select make_ym_interval(-178956970, -9)
-- !query schema
struct<>
-- !query output
java.lang.ArithmeticException
integer overflow
org.apache.spark.SparkArithmeticException
{
"errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
"sqlState" : "22015",
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
"startIndex" : 8,
"stopIndex" : 39,
"fragment" : "make_ym_interval(-178956970, -9)"
} ]
}


-- !query
Expand Down Expand Up @@ -2493,12 +2523,8 @@ struct<>
-- !query output
org.apache.spark.SparkArithmeticException
{
"errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW",
"sqlState" : "22015",
"messageParameters" : {
"alternative" : "",
"message" : "integer overflow"
}
"errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
"sqlState" : "22015"
}


Expand All @@ -2509,11 +2535,10 @@ struct<>
-- !query output
org.apache.spark.SparkArithmeticException
{
"errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW",
"errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION",
"sqlState" : "22015",
"messageParameters" : {
"alternative" : " Use 'try_subtract' to tolerate overflow and return NULL instead.",
"message" : "integer overflow"
"functionName" : "`try_subtract`"
}
}

Expand All @@ -2525,11 +2550,10 @@ struct<>
-- !query output
org.apache.spark.SparkArithmeticException
{
"errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW",
"errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION",
"sqlState" : "22015",
"messageParameters" : {
"alternative" : " Use 'try_add' to tolerate overflow and return NULL instead.",
"message" : "integer overflow"
"functionName" : "`try_add`"
}
}

Expand Down Expand Up @@ -2838,11 +2862,10 @@ struct<>
-- !query output
org.apache.spark.SparkArithmeticException
{
"errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW",
"errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION",
"sqlState" : "22015",
"messageParameters" : {
"alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.",
"message" : "Interval value overflows after being divided by -1"
"functionName" : "`try_divide`"
},
"queryContext" : [ {
"objectType" : "",
Expand All @@ -2861,11 +2884,10 @@ struct<>
-- !query output
org.apache.spark.SparkArithmeticException
{
"errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW",
"errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION",
"sqlState" : "22015",
"messageParameters" : {
"alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.",
"message" : "Interval value overflows after being divided by -1"
"functionName" : "`try_divide`"
},
"queryContext" : [ {
"objectType" : "",
Expand Down Expand Up @@ -2918,11 +2940,10 @@ struct<>
-- !query output
org.apache.spark.SparkArithmeticException
{
"errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW",
"errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION",
"sqlState" : "22015",
"messageParameters" : {
"alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.",
"message" : "Interval value overflows after being divided by -1"
"functionName" : "`try_divide`"
},
"queryContext" : [ {
"objectType" : "",
Expand All @@ -2941,11 +2962,10 @@ struct<>
-- !query output
org.apache.spark.SparkArithmeticException
{
"errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW",
"errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION",
"sqlState" : "22015",
"messageParameters" : {
"alternative" : " Use 'try_divide' to tolerate overflow and return NULL instead.",
"message" : "Interval value overflows after being divided by -1"
"functionName" : "`try_divide`"
},
"queryContext" : [ {
"objectType" : "",
Expand Down
Loading