Skip to content

[fix](nereids) fix convert to date literal throw exception #48980

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morrySnow merged 10 commits into from
Mar 14, 2025
Merged

[fix](nereids) fix convert to date literal throw exception #48980

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
58e0aae
fix convert to date literal throw exception
yujun777 Mar 12, 2025
74ffb83
add test
yujun777 Mar 12, 2025
7e19312
fix exception
yujun777 Mar 12, 2025
6f022d4
update test
yujun777 Mar 12, 2025
fbf782b
update test
yujun777 Mar 12, 2025
56df640
update
yujun777 Mar 13, 2025
7bad672
update
yujun777 Mar 13, 2025
4591036
update test
yujun777 Mar 13, 2025
40a9e80
update
yujun777 Mar 13, 2025
562a7cf
add test
yujun777 Mar 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 8 additions & 11 deletions ...e/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@
import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.trees.expressions.literal.format.DateTimeChecker;
import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.coercion.DateLikeType;
Expand All @@ -105,7 +106,6 @@
import com.google.common.collect.Lists;
import org.apache.commons.codec.digest.DigestUtils;

import java.time.DateTimeException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
Expand Down Expand Up @@ -493,17 +493,14 @@ public Expression visitCast(Cast cast, ExpressionRewriteContext context) {
if (child.isNullLiteral()) {
return new NullLiteral(dataType);
} else if (child instanceof StringLikeLiteral && dataType instanceof DateLikeType) {
String dateStr = ((StringLikeLiteral) child).getStringValue();
if (!DateTimeChecker.isValidDateTime(dateStr)) {
return cast;
}
try {
return ((DateLikeType) dataType).fromString(((StringLikeLiteral) child).getStringValue());
} catch (AnalysisException t) {
if (cast.isExplicitType()) {
return cast;
} else {
// If cast is from type coercion, we don't use NULL literal and will throw exception.
throw t;
}
} catch (DateTimeException e) {
return new NullLiteral(dataType);
return ((DateLikeType) dataType).fromString(dateStr);
} catch (Exception t) {
return cast;
}
}
try {
Expand Down
5 changes: 2 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,6 @@
import com.google.common.base.Preconditions;
import org.apache.commons.collections.CollectionUtils;

import java.time.DateTimeException;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
Expand Down Expand Up @@ -213,7 +212,7 @@ private ColumnStatistic castMinMax(ColumnStatistic colStats, DataType targetType
long min = dateMinLiteral.getValue();
builder.setMinValue(min);
builder.setMinExpr(dateMinLiteral.toLegacyLiteral());
} catch (AnalysisException | DateTimeException e) {
} catch (AnalysisException e) {
convertSuccess = false;
}
}
Expand All @@ -224,7 +223,7 @@ private ColumnStatistic castMinMax(ColumnStatistic colStats, DataType targetType
long max = dateMaxLiteral.getValue();
builder.setMaxValue(max);
builder.setMaxExpr(dateMaxLiteral.toLegacyLiteral());
} catch (AnalysisException | DateTimeException e) {
} catch (AnalysisException e) {
convertSuccess = false;
}
}
Expand Down
11 changes: 3 additions & 8 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@

import com.google.common.collect.ImmutableSet;

import java.time.DateTimeException;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.Year;
Expand Down Expand Up @@ -274,8 +273,8 @@ static Result<String, AnalysisException> normalize(String s) {
}

/** parseDateLiteral */
public static Result<DateLiteral, ? extends Exception> parseDateLiteral(String s) {
Result<TemporalAccessor, ? extends Exception> parseResult = parseDateTime(s);
public static Result<DateLiteral, AnalysisException> parseDateLiteral(String s) {
Result<TemporalAccessor, AnalysisException> parseResult = parseDateTime(s);
if (parseResult.isError()) {
return parseResult.cast();
}
Expand All @@ -291,7 +290,7 @@ static Result<String, AnalysisException> normalize(String s) {
}

/** parseDateTime */
public static Result<TemporalAccessor, ? extends Exception> parseDateTime(String s) {
public static Result<TemporalAccessor, AnalysisException> parseDateTime(String s) {
String originalString = s;
try {
// fast parse '2022-01-01'
Expand Down Expand Up @@ -354,10 +353,6 @@ static Result<String, AnalysisException> normalize(String s) {
}

return Result.ok(dateTime);
} catch (DateTimeException e) {
return Result.err(() ->
new DateTimeException("date/datetime literal [" + originalString + "] is invalid", e)
);
} catch (Exception ex) {
return Result.err(() -> new AnalysisException("date/datetime literal [" + originalString + "] is invalid"));
}
Expand Down
2 changes: 1 addition & 1 deletion ...ore/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ public static int determineScale(String s) {

/** parseDateTimeLiteral */
public static Result<DateTimeLiteral, AnalysisException> parseDateTimeLiteral(String s, boolean isV2) {
Result<TemporalAccessor, ? extends Exception> parseResult = parseDateTime(s);
Result<TemporalAccessor, AnalysisException> parseResult = parseDateTime(s);
if (parseResult.isError()) {
return parseResult.cast();
}
Expand Down
10 changes: 9 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.doris.catalog.Type;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.format.DateTimeChecker;
import org.apache.doris.nereids.types.coercion.DateLikeType;
import org.apache.doris.nereids.types.coercion.IntegralType;

Expand Down Expand Up @@ -87,7 +88,14 @@ public static DateTimeV2Type forType(DataType dataType) {
* maybe we need to check for validity?
*/
public static DateTimeV2Type forTypeFromString(String s) {
int scale = DateTimeLiteral.determineScale(s);
int scale = MAX_SCALE;
if (DateTimeChecker.isValidDateTime(s)) {
try {
scale = DateTimeLiteral.determineScale(s);
} catch (Exception e) {
// let be to process it
}
}
if (scale > MAX_SCALE) {
scale = MAX_SCALE;
}
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -602,7 +602,7 @@ public static Optional<Expression> characterLiteralTypeCoercion(String value, Da
} else if (dataType.isDateTimeType() && DateTimeChecker.isValidDateTime(value)) {
ret = DateTimeLiteral.parseDateTimeLiteral(value, false).orElse(null);
} else if (dataType.isDateV2Type() && DateTimeChecker.isValidDateTime(value)) {
Result<DateLiteral, ? extends Exception> parseResult = DateV2Literal.parseDateLiteral(value);
Result<DateLiteral, AnalysisException> parseResult = DateV2Literal.parseDateLiteral(value);
if (parseResult.isOk()) {
ret = parseResult.get();
} else {
Expand Down
25 changes: 12 additions & 13 deletions ...a/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateSqlTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,7 @@

package org.apache.doris.nereids.rules.expression.rules;

import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.types.DateTimeV2Type;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.utframe.TestWithFeService;
Expand Down Expand Up @@ -102,7 +101,7 @@ void dateLikeOverflow() {
.rewrite()
.matches(
logicalResultSink(
logicalOneRowRelation().when(p -> p.getProjects().get(0).child(0).equals(new NullLiteral(DateTimeV2Type.of(6))))
logicalOneRowRelation().when(p -> p.getProjects().get(0).child(0) instanceof Cast)
)
);

Expand All @@ -111,7 +110,7 @@ void dateLikeOverflow() {
.rewrite()
.matches(
logicalResultSink(
logicalOneRowRelation().when(p -> p.getProjects().get(0).child(0).equals(new NullLiteral(DateTimeV2Type.of(6))))
logicalOneRowRelation().when(p -> p.getProjects().get(0).child(0) instanceof Cast)
)
);

Expand All @@ -120,7 +119,7 @@ void dateLikeOverflow() {
.rewrite()
.matches(
logicalResultSink(
logicalOneRowRelation().when(p -> p.getProjects().get(0).child(0) instanceof NullLiteral)
logicalOneRowRelation().when(p -> p.getProjects().get(0).child(0).child(0).child(0) instanceof Cast)
)
);

Expand All @@ -129,7 +128,7 @@ void dateLikeOverflow() {
.rewrite()
.matches(
logicalResultSink(
logicalOneRowRelation().when(p -> p.getProjects().get(0).child(0) instanceof NullLiteral)
logicalOneRowRelation().when(p -> p.getProjects().get(0).child(0).child(0) instanceof Cast)
)
);

Expand All @@ -138,7 +137,7 @@ void dateLikeOverflow() {
.rewrite()
.matches(
logicalResultSink(
logicalOneRowRelation().when(p -> p.getProjects().get(0).child(0) instanceof NullLiteral)
logicalOneRowRelation().when(p -> p.getProjects().get(0).child(0).child(0) instanceof Cast)
)
);

Expand All @@ -147,34 +146,34 @@ void dateLikeOverflow() {
.rewrite()
.matches(
logicalResultSink(
logicalOneRowRelation().when(p -> p.getProjects().get(0).child(0) instanceof NullLiteral)
logicalOneRowRelation().when(p -> p.getProjects().get(0).child(0).child(0).child(0) instanceof Cast)
)
);

PlanChecker.from(connectContext)
.analyze("select CAST('2021-01-32 00:00:00' AS DATETIME(6)) = '2021-01-32 00:00:00'")
.rewrite()
.matches(logicalOneRowRelation().when(oneRowRelation ->
oneRowRelation.getExpressions().get(0).child(0) instanceof NullLiteral)
oneRowRelation.getExpressions().get(0).child(0).child(0).child(0).child(0) instanceof Cast)
);

PlanChecker.from(connectContext)
.analyze("select CAST('2021-01-32 00:00:00' AS DATETIME(6)) = '2021-01-32 00:00:00'")
.rewrite()
.matches(logicalOneRowRelation().when(oneRowRelation ->
oneRowRelation.getExpressions().get(0).child(0) instanceof NullLiteral)
oneRowRelation.getExpressions().get(0).child(0).child(0).child(0).child(0) instanceof Cast)
);
PlanChecker.from(connectContext)
.analyze("select CAST('2021-01-32 00:00:00' AS DATETIME(6)) = '2021-01-32 23:00:00'")
.analyze("select CAST('2021-01-32 00:00:00' AS DATETIME(6)) = '2022-01-32 23:00:00'")
.rewrite()
.matches(logicalOneRowRelation().when(oneRowRelation ->
oneRowRelation.getExpressions().get(0).child(0) instanceof NullLiteral)
oneRowRelation.getExpressions().get(0).child(0).child(0) instanceof Cast)
);
PlanChecker.from(connectContext)
.analyze("select CAST('2021-01-32 00:00:00' AS DATETIME(6)) = '1000'")
.rewrite()
.matches(logicalOneRowRelation().when(oneRowRelation ->
oneRowRelation.getExpressions().get(0).child(0) instanceof NullLiteral)
oneRowRelation.getExpressions().get(0).child(0).child(0) instanceof Cast)
);
}
}
7 changes: 3 additions & 4 deletions ...ore/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;

import java.time.DateTimeException;
import java.util.function.Consumer;

class DateLiteralTest {
Expand Down Expand Up @@ -68,7 +67,7 @@ void testDate() {
new DateLiteral("2022-1-1");
new DateLiteral("20220101");

Assertions.assertThrows(DateTimeException.class, () -> new DateLiteral("-01-01"));
Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("-01-01"));
}

@Test
Expand Down Expand Up @@ -129,8 +128,8 @@ void testIrregularDate() {

@Test
void testWrongPunctuationDate() {
Assertions.assertThrows(DateTimeException.class, () -> new DateTimeV2Literal("2020€02€01"));
Assertions.assertThrows(DateTimeException.class, () -> new DateTimeV2Literal("2020【02】01"));
Assertions.assertThrows(AnalysisException.class, () -> new DateTimeV2Literal("2020€02€01"));
Assertions.assertThrows(AnalysisException.class, () -> new DateTimeV2Literal("2020【02】01"));
}

@Test
Expand Down
10 changes: 10 additions & 0 deletions regression-test/data/nereids_syntax_p0/test_cast_datetime.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,13 @@
-- !1 --
1

-- !2 --
1 2000-01-01 2000-01-01 2000-01-01T00:00 2000-01-01T12:12:12
2 \N \N \N \N
3 \N \N \N \N

-- !3 --
1 \N \N \N \N
2 \N \N \N \N
3 \N \N \N \N

Loading
Loading