Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix that the result of expression cast(Real/Decimal)AsTime is inconsistent with TiDB (#5799) #5911

Closed
wants to merge 1 commit into from
Closed

fix that the result of expression cast(Real/Decimal)AsTime is inconsistent with TiDB (#5799) #5911

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 174 additions & 4 deletions dbms/src/Common/MyTime.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -503,7 +503,16 @@ int MyTimeBase::weekDay() const

bool checkTimeValid(Int32 year, Int32 month, Int32 day, Int32 hour, Int32 minute, Int32 second)
{
<<<<<<< HEAD
if (year > 9999 || month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59)
=======
return month != 0 && day != 0 && checkTimeValidAllowMonthAndDayZero(year, month, day, hour, minute, second);
}

bool checkTimeValidAllowMonthAndDayZero(Int32 year, Int32 month, Int32 day, Int32 hour, Int32 minute, Int32 second)
{
if (year > 9999 || month < 0 || month > 12 || day < 0 || day > 31 || hour > 23 || minute > 59 || second > 59)
>>>>>>> 720bfc1787 (fix that the result of expression cast(Real/Decimal)AsTime is inconsistent with TiDB (#5799))
{
return false;
}
Expand All @@ -518,7 +527,123 @@ bool checkTimeValid(Int32 year, Int32 month, Int32 day, Int32 hour, Int32 minute
return day <= (is_leap_year ? 29 : 28);
}

<<<<<<< HEAD
std::pair<Field, bool> parseMyDateTimeAndJudgeIsDate(const String & str, int8_t fsp, bool needCheckTimeValid)
=======
bool noNeedCheckTime(Int32, Int32, Int32, Int32, Int32, Int32)
{
return true;
}

// Return true if the time is invalid.
inline bool getDatetime(const Int64 & num, MyDateTime & result)
{
UInt64 ymd = num / 1000000;
UInt64 hms = num - ymd * 1000000;

UInt64 year = ymd / 10000;
ymd %= 10000;
UInt64 month = ymd / 100;
UInt64 day = ymd % 100;

UInt64 hour = hms / 10000;
hms %= 10000;
UInt64 minute = hms / 100;
UInt64 second = hms % 100;

if (toCoreTimeChecked(year, month, day, hour, minute, second, 0, result))
{
return true;
}
return !result.isValid(true, false);
}

// Convert a integer number to DateTime and return true if the result is NULL.
// If number is invalid(according to SQL_MODE), return NULL and handle the error with DAGContext.
// This function may throw exception.
inline bool numberToDateTime(Int64 number, MyDateTime & result, bool allowZeroDate)
{
MyDateTime datetime(0);
if (number == 0)
{
if (allowZeroDate)
{
result = datetime;
return false;
}
return true;
}

// datetime type
if (number >= 10000101000000)
{
return getDatetime(number, result);
}

// check MMDD
if (number < 101)
{
return true;
}

// check YYMMDD: 2000-2069
if (number <= 69 * 10000 + 1231)
{
number = (number + 20000000) * 1000000;
return getDatetime(number, result);
}

if (number < 70 * 10000 + 101)
{
return true;
}

// check YYMMDD
if (number <= 991231)
{
number = (number + 19000000) * 1000000;
return getDatetime(number, result);
}

// check hour/min/second
if (number <= 99991231)
{
number *= 1000000;
return getDatetime(number, result);
}

// check MMDDHHMMSS
if (number < 101000000)
{
return true;
}

// check YYMMDDhhmmss: 2000-2069
if (number <= 69 * 10000000000 + 1231235959)
{
number += 20000000000000;
return getDatetime(number, result);
}

// check YYYYMMDDhhmmss
if (number < 70 * 10000000000 + 101000000)
{
return true;
}

// check YYMMDDHHMMSS
if (number <= 991231235959)
{
number += 19000000000000;
return getDatetime(number, result);
}

return getDatetime(number, result);
}

// isFloat is true means that the input string is float format like "1212.111"
std::pair<Field, bool> parseMyDateTimeAndJudgeIsDate(const String & str, int8_t fsp, CheckTimeFunc checkTimeFunc, bool isFloat)
>>>>>>> 720bfc1787 (fix that the result of expression cast(Real/Decimal)AsTime is inconsistent with TiDB (#5799))
{
// Since we only use DateLUTImpl as parameter placeholder of AddSecondsImpl::execute
// and it's costly to construct a DateLUTImpl, a shared static instance is enough.
Expand All @@ -543,7 +668,7 @@ std::pair<Field, bool> parseMyDateTimeAndJudgeIsDate(const String & str, int8_t
return seps.size() > 5 || (seps.size() == 1 && seps[0].size() > 4);
};

if (!frac_str.empty())
if (!frac_str.empty() && !isFloat)
{
if (!no_absorb(seps))
{
Expand Down Expand Up @@ -577,6 +702,24 @@ std::pair<Field, bool> parseMyDateTimeAndJudgeIsDate(const String & str, int8_t
case 1:
{
size_t l = seps[0].size();
if (isFloat)
{
MyDateTime date_time(0);
if (seps[0] == "0")
{
return {date_time.toPackedUInt(), is_date};
}
if (numberToDateTime(std::stoll(seps[0]), date_time))
{
return {Field(), is_date};
}
std::tie(year, month, day, hour, minute, second) = std::tuple(date_time.year, date_time.month, date_time.day, date_time.hour, date_time.minute, date_time.second);
if (l >= 9 && l <= 14)
{
hhmmss = true;
}
break;
}
switch (l)
{
case 14: // YYYYMMDDHHMMSS
Expand Down Expand Up @@ -720,7 +863,7 @@ std::pair<Field, bool> parseMyDateTimeAndJudgeIsDate(const String & str, int8_t
// If str is sepereated by delimiters, the first one is year, and if the year is 2 digit,
// we should adjust it.
// TODO: adjust year is very complex, now we only consider the simplest way.
if (seps[0].size() == 2)
if (seps[0].size() <= 2 && !isFloat)
{
if (year == 0 && month == 0 && day == 0 && hour == 0 && minute == 0 && second == 0 && frac_str.empty())
{
Expand Down Expand Up @@ -768,7 +911,11 @@ std::pair<Field, bool> parseMyDateTimeAndJudgeIsDate(const String & str, int8_t
}
}

<<<<<<< HEAD
if (needCheckTimeValid && !checkTimeValid(year, month, day, hour, minute, second))
=======
if (!checkTimeFunc(year, month, day, hour, minute, second))
>>>>>>> 720bfc1787 (fix that the result of expression cast(Real/Decimal)AsTime is inconsistent with TiDB (#5799))
{
throw Exception("Wrong datetime format");
}
Expand Down Expand Up @@ -811,9 +958,20 @@ std::pair<Field, bool> parseMyDateTimeAndJudgeIsDate(const String & str, int8_t
}

// TODO: support parse time from float string
<<<<<<< HEAD
Field parseMyDateTime(const String & str, int8_t fsp, bool needCheckTimeValid)
{
return parseMyDateTimeAndJudgeIsDate(str, fsp, needCheckTimeValid).first;
=======
Field parseMyDateTime(const String & str, int8_t fsp, CheckTimeFunc checkTimeFunc)
{
return parseMyDateTimeAndJudgeIsDate(str, fsp, checkTimeFunc).first;
}

Field parseMyDateTimeFromFloat(const String & str, int8_t fsp, CheckTimeFunc checkTimeFunc)
{
return parseMyDateTimeAndJudgeIsDate(str, fsp, checkTimeFunc, true).first;
>>>>>>> 720bfc1787 (fix that the result of expression cast(Real/Decimal)AsTime is inconsistent with TiDB (#5799))
}

String MyDateTime::toString(int fsp) const
Expand Down Expand Up @@ -1037,7 +1195,7 @@ void MyTimeBase::check(bool allow_zero_in_date, bool allow_invalid_date) const
}
}

if (year >= 9999 || month > 12)
if (year > 9999 || month > 12)
{
throw TiFlashException("Incorrect time value", Errors::Types::WrongValue);
}
Expand All @@ -1049,10 +1207,22 @@ void MyTimeBase::check(bool allow_zero_in_date, bool allow_invalid_date) const
static auto is_leap_year = [](UInt16 _year) {
return ((_year % 4 == 0) && (_year % 100 != 0)) || (_year % 400 == 0);
};
<<<<<<< HEAD
max_day = max_days_in_month[month - 1]; // NOLINT
if (month == 2 && is_leap_year(year))
=======
if (allow_zero_in_date && month == 0)
>>>>>>> 720bfc1787 (fix that the result of expression cast(Real/Decimal)AsTime is inconsistent with TiDB (#5799))
{
max_day = 29;
max_day = 31;
}
else
{
max_day = max_days_in_month[month - 1]; // NOLINT
if (month == 2 && is_leap_year(year))
{
max_day = 29;
}
}
}
if (day > max_day)
Expand Down
18 changes: 18 additions & 0 deletions dbms/src/Common/MyTime.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,8 @@ struct MyDate : public MyTimeBase
}
};

bool numberToDateTime(Int64 number, MyDateTime & result, bool allowZeroDate = true);

struct MyDateTimeFormatter
{
std::vector<std::function<void(const MyTimeBase & datetime, String & result)>> formatters;
Expand Down Expand Up @@ -159,8 +161,24 @@ struct MyDateTimeParser
std::vector<ParserCallback> parsers;
};

<<<<<<< HEAD
Field parseMyDateTime(const String & str, int8_t fsp = 6, bool needCheckTimeValid = false);
std::pair<Field, bool> parseMyDateTimeAndJudgeIsDate(const String & str, int8_t fsp = 6, bool needCheckTimeValid = false);
=======
bool checkTimeValid(Int32 year, Int32 month, Int32 day, Int32 hour, Int32 minute, Int32 second);
bool checkTimeValidAllowMonthAndDayZero(Int32 year, Int32 month, Int32 day, Int32 hour, Int32 minute, Int32 second);
bool noNeedCheckTime(Int32, Int32, Int32, Int32, Int32, Int32);

using CheckTimeFunc = std::function<bool(Int32, Int32, Int32, Int32, Int32, Int32)>;

static const int8_t DefaultFsp = 6;
static bool DefaultIsFloat = false;
static CheckTimeFunc DefaultCheckTimeFunc = noNeedCheckTime;

Field parseMyDateTime(const String & str, int8_t fsp = DefaultFsp, CheckTimeFunc checkTimeFunc = DefaultCheckTimeFunc);
Field parseMyDateTimeFromFloat(const String & str, int8_t fsp = DefaultFsp, CheckTimeFunc checkTimeFunc = DefaultCheckTimeFunc);
std::pair<Field, bool> parseMyDateTimeAndJudgeIsDate(const String & str, int8_t fsp = DefaultFsp, CheckTimeFunc checkTimeFunc = DefaultCheckTimeFunc, bool isFloat = DefaultIsFloat);
>>>>>>> 720bfc1787 (fix that the result of expression cast(Real/Decimal)AsTime is inconsistent with TiDB (#5799))

void convertTimeZone(UInt64 from_time, UInt64 & to_time, const DateLUTImpl & time_zone_from, const DateLUTImpl & time_zone_to, bool throw_exception = false);

Expand Down
8 changes: 6 additions & 2 deletions dbms/src/Common/tests/gtest_mytime.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,19 +57,23 @@ class TestMyTime : public testing::Test
}
}

static void checkNumberToMyDateTime(const Int64 & input, const MyDateTime & expected, bool expect_error, DAGContext * ctx)
static void checkNumberToMyDateTime(const Int64 & input, const MyDateTime & expected, bool expect_error, DAGContext *)
{
if (expect_error)
{
MyDateTime datetime(0, 0, 0, 0, 0, 0, 0);
<<<<<<< HEAD
EXPECT_THROW({ numberToDateTime(input, datetime, ctx); }, TiFlashException) << "Original time number: " << input;
=======
EXPECT_TRUE(numberToDateTime(input, datetime));
>>>>>>> 720bfc1787 (fix that the result of expression cast(Real/Decimal)AsTime is inconsistent with TiDB (#5799))
return;
}

try
{
MyDateTime source(0, 0, 0, 0, 0, 0, 0);
numberToDateTime(input, source, ctx);
numberToDateTime(input, source);
EXPECT_EQ(source.year, expected.year) << "Original time number: " << input;
EXPECT_EQ(source.month, expected.month) << "Original time number: " << input;
EXPECT_EQ(source.day, expected.day) << "Original time number: " << input;
Expand Down
12 changes: 12 additions & 0 deletions dbms/src/Functions/FunctionsDateTime.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1007,7 +1007,11 @@ struct AddSecondsImpl
// TODO: need do these in vector mode in the future
static inline String execute(String str, Int64 delta, const DateLUTImpl & time_zone)
{
<<<<<<< HEAD
Field packed_uint_value = parseMyDateTime(str);
=======
Field packed_uint_value = parseMyDateTime(str, 6, checkTimeValid);
>>>>>>> 720bfc1787 (fix that the result of expression cast(Real/Decimal)AsTime is inconsistent with TiDB (#5799))
UInt64 packed_uint = packed_uint_value.template safeGet<UInt64>();
UInt64 result = AddSecondsImpl::execute(packed_uint, delta, time_zone);
MyDateTime myDateTime(result);
Expand Down Expand Up @@ -1091,7 +1095,11 @@ struct AddDaysImpl

static inline String execute(String str, Int64 delta, const DateLUTImpl & time_zone)
{
<<<<<<< HEAD
auto value_and_is_date = parseMyDateTimeAndJudgeIsDate(str, 6, true);
=======
auto value_and_is_date = parseMyDateTimeAndJudgeIsDate(str, 6, checkTimeValid);
>>>>>>> 720bfc1787 (fix that the result of expression cast(Real/Decimal)AsTime is inconsistent with TiDB (#5799))
Field packed_uint_value = value_and_is_date.first;
bool is_date = value_and_is_date.second;
UInt64 packed_uint = packed_uint_value.template safeGet<UInt64>();
Expand Down Expand Up @@ -1159,7 +1167,11 @@ struct AddMonthsImpl

static inline String execute(String str, Int64 delta, const DateLUTImpl & time_zone)
{
<<<<<<< HEAD
auto value_and_is_date = parseMyDateTimeAndJudgeIsDate(str, 6, true);
=======
auto value_and_is_date = parseMyDateTimeAndJudgeIsDate(str, 6, checkTimeValid);
>>>>>>> 720bfc1787 (fix that the result of expression cast(Real/Decimal)AsTime is inconsistent with TiDB (#5799))
Field packed_uint_value = value_and_is_date.first;
bool is_date = value_and_is_date.second;
UInt64 packed_uint = packed_uint_value.template safeGet<UInt64>();
Expand Down
Loading