diff --git a/dbms/src/Common/MyTime.cpp b/dbms/src/Common/MyTime.cpp index b8e752fe4b4..f4831a69e59 100644 --- a/dbms/src/Common/MyTime.cpp +++ b/dbms/src/Common/MyTime.cpp @@ -930,95 +930,69 @@ size_t maxFormattedDateTimeStringLength(const String & format) return std::max(result, 1); } -MyDateTime numberToDateTime(Int64 number) +void MyTimeBase::check(bool allow_zero_in_date, bool allow_invalid_date) const { - MyDateTime datetime(0); - - auto get_datetime = [](const Int64 & num) { - auto ymd = num / 1000000; - auto hms = num - ymd * 1000000; - - UInt16 year = ymd / 10000; - ymd %= 10000; - UInt8 month = ymd / 100; - UInt8 day = ymd % 100; - - UInt16 hour = hms / 10000; - hms %= 10000; - UInt8 minute = hms / 100; - UInt8 second = hms % 100; - - return MyDateTime(year, month, day, hour, minute, second, 0); - }; - - if (number == 0) - return datetime; - - // datetime type - if (number >= 10000101000000) - { - return get_datetime(number); - } - - // check MMDD - if (number < 101) - { - throw TiFlashException("Cannot convert " + std::to_string(number) + " to Datetime", Errors::Types::WrongValue); - } - - // check YYMMDD: 2000-2069 - if (number <= 69 * 10000 + 1231) + if (!(year == 0 && month == 0 && day == 0)) { - number = (number + 200000000) * 1000000; - return get_datetime(number); + if (!allow_zero_in_date && (month == 0 || day == 0)) + { + char buff[] = "0000-00-00"; + std::sprintf(buff, "%04d-%02d-%02d", year, month, day); + throw TiFlashException("Incorrect datetime value: " + String(buff), Errors::Types::WrongValue); + } } - // check YYMMDD - if (number <= 991231) + if (year >= 9999 || month > 12) { - number = (number + 19000000) * 1000000; - return get_datetime(number); + throw TiFlashException("Incorrect time value", Errors::Types::WrongValue); } - // check YYYYMMDD - if (number <= 10000101) + UInt8 max_day = 31; + if (!allow_invalid_date) { - throw TiFlashException("Cannot convert " + std::to_string(number) + " to Datetime", Errors::Types::WrongValue); + constexpr static UInt8 max_days_in_month[12] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + static auto is_leap_year = [](UInt16 _year) { return ((_year % 4 == 0) && (_year % 100 != 0)) || (_year % 400 == 0); }; + max_day = max_days_in_month[month - 1]; + if (month == 2 && is_leap_year(year)) + { + max_day = 29; + } } - - // check hhmmss - if (number <= 99991231) + if (day > max_day) { - number = number * 1000000; - return get_datetime(number); + char buff[] = "0000-00-00"; + std::sprintf(buff, "%04d-%02d-%02d", year, month, day); + throw TiFlashException("Incorrect datetime value: " + String(buff), Errors::Types::WrongValue); } - // check MMDDhhmmss - if (number < 101000000) + if (hour < 0 || hour >= 24) { - throw TiFlashException("Cannot convert " + std::to_string(number) + " to Datetime", Errors::Types::WrongValue); + throw TiFlashException("Incorrect datetime value", Errors::Types::WrongValue); } - - // check YYMMDDhhmmss: 2000-2069 - if (number <= 69 * 10000000000 + 1231235959) + if (minute >= 60) { - number += 20000000000000; - return get_datetime(number); + throw TiFlashException("Incorrect datetime value", Errors::Types::WrongValue); } - - // check YYMMDDhhmmss - if (number < 70 * 10000000000 + 101000000) + if (second >= 60) { - throw TiFlashException("Cannot convert " + std::to_string(number) + " to Datetime", Errors::Types::WrongValue); + throw TiFlashException("Incorrect datetime value", Errors::Types::WrongValue); } + return; +} - if (number <= 991231235959) +bool toCoreTimeChecked(const UInt64 & year, const UInt64 & month, const UInt64 & day, const UInt64 & hour, const UInt64 & minute, + const UInt64 & second, const UInt64 & microsecond, MyDateTime & result) +{ + if (year >= (1 << MyTimeBase::YEAR_BIT_FIELD_WIDTH) || month >= (1 << MyTimeBase::MONTH_BIT_FIELD_WIDTH) + || day >= (1 << MyTimeBase::DAY_BIT_FIELD_WIDTH) || hour >= (1 << MyTimeBase::HOUR_BIT_FIELD_WIDTH) + || minute >= (1 << MyTimeBase::MINUTE_BIT_FIELD_WIDTH) || second >= (1 << MyTimeBase::SECOND_BIT_FIELD_WIDTH) + || microsecond >= (1 << MyTimeBase::MICROSECOND_BIT_FIELD_WIDTH)) { - number += 19000000000000; - return get_datetime(number); + result = MyDateTime(0, 0, 0, 0, 0, 0, 0); + return true; } - - return get_datetime(number); + result = MyDateTime(year, month, day, hour, minute, second, microsecond); + return false; } MyDateTimeFormatter::MyDateTimeFormatter(const String & layout) diff --git a/dbms/src/Common/MyTime.h b/dbms/src/Common/MyTime.h index 46b1aef2e43..7dc8001b025 100644 --- a/dbms/src/Common/MyTime.h +++ b/dbms/src/Common/MyTime.h @@ -89,6 +89,10 @@ struct MyTimeBase int week(UInt32 mode) const; std::tuple calcWeek(UInt32 mode) const; + + // Check validity of time under specified SQL_MODE. + // May throw exception. + void check(bool allow_zero_in_date, bool allow_invalid_date) const; }; struct MyDateTime : public MyTimeBase @@ -139,11 +143,15 @@ int calcDayNum(int year, int month, int day); size_t maxFormattedDateTimeStringLength(const String & format); -MyDateTime numberToDateTime(Int64 number); + bool isPunctuation(char c); bool isValidSeperator(char c, int previous_parts); +// Build CoreTime value with checking overflow of internal bit fields, return true if input is invalid. +// Note that this function will not check if the input is logically a valid datetime value. +bool toCoreTimeChecked(const UInt64 & year, const UInt64 & month, const UInt64 & day, const UInt64 & hour, const UInt64 & minute, + const UInt64 & second, const UInt64 & microsecond, MyDateTime & result); } // namespace DB diff --git a/dbms/src/Common/tests/gtest_mytime.cpp b/dbms/src/Common/tests/gtest_mytime.cpp index cc844dbc2b5..537e74fc522 100644 --- a/dbms/src/Common/tests/gtest_mytime.cpp +++ b/dbms/src/Common/tests/gtest_mytime.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -36,7 +37,7 @@ class TestMyTime : public testing::Test } } - static void checkParseMyDateTime(const std::string & str, MyDateTime & expected, const DataTypeMyDateTime & type) + static void checkParseMyDateTime(const std::string & str, const MyDateTime & expected, const DataTypeMyDateTime & type) { try { @@ -56,6 +57,34 @@ class TestMyTime : public testing::Test throw; } } + + static void checkNumberToMyDateTime(const Int64 & input, const MyDateTime & expected, bool expect_error, DAGContext * ctx) + { + if (expect_error) + { + MyDateTime datetime(0, 0, 0, 0, 0, 0, 0); + EXPECT_THROW({ numberToDateTime(input, datetime, ctx); }, TiFlashException) << "Original time number: " << input; + return; + } + + try + { + MyDateTime source(0, 0, 0, 0, 0, 0, 0); + numberToDateTime(input, source, ctx); + EXPECT_EQ(source.year, expected.year) << "Original time number: " << input; + EXPECT_EQ(source.month, expected.month) << "Original time number: " << input; + EXPECT_EQ(source.day, expected.day) << "Original time number: " << input; + EXPECT_EQ(source.hour, expected.hour) << "Original time number: " << input; + EXPECT_EQ(source.minute, expected.minute) << "Original time number: " << input; + EXPECT_EQ(source.second, expected.second) << "Original time number: " << input; + EXPECT_EQ(source.micro_second, expected.micro_second) << "Original time number: " << input; + } + catch (...) + { + std::cerr << "Error occurs when parsing: \"" << input << "\"" << std::endl; + throw; + } + } }; TEST_F(TestMyTime, ParseMyDateTimeWithFraction) @@ -153,6 +182,46 @@ catch (Exception & e) GTEST_FAIL(); } -} // namespace tests +TEST_F(TestMyTime, NumberToDateTime) +try +{ + std::vector> cases{ + {20101010111111, false, MyDateTime(2010, 10, 10, 11, 11, 11, 0)}, + {2010101011111, false, MyDateTime(201, 1, 1, 1, 11, 11, 0)}, + {201010101111, false, MyDateTime(2020, 10, 10, 10, 11, 11, 0)}, + {20101010111, false, MyDateTime(2002, 1, 1, 1, 1, 11, 0)}, + {2010101011, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)}, + {201010101, false, MyDateTime(2000, 2, 1, 1, 1, 1, 0)}, + {20101010, false, MyDateTime(2010, 10, 10, 0, 0, 0, 0)}, + {2010101, false, MyDateTime(201, 1, 1, 0, 0, 0, 0)}, + {201010, false, MyDateTime(2020, 10, 10, 0, 0, 0, 0)}, + {20101, false, MyDateTime(2002, 1, 1, 0, 0, 0, 0)}, + {2010, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)}, + {201, false, MyDateTime(2000, 2, 1, 0, 0, 0, 0)}, + {20, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)}, + {2, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)}, + {0, false, MyDateTime(0, 0, 0, 0, 0, 0, 0)}, + {-1, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)}, + {99999999999999, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)}, + {100000000000000, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)}, + {10000102000000, false, MyDateTime(1000, 1, 2, 0, 0, 0, 0)}, + {19690101000000, false, MyDateTime(1969, 1, 1, 0, 0, 0, 0)}, + {991231235959, false, MyDateTime(1999, 12, 31, 23, 59, 59, 0)}, + {691231235959, false, MyDateTime(2069, 12, 31, 23, 59, 59, 0)}, + {370119031407, false, MyDateTime(2037, 1, 19, 3, 14, 7, 0)}, + {380120031407, false, MyDateTime(2038, 1, 20, 3, 14, 7, 0)}, + {11111111111, false, MyDateTime(2001, 11, 11, 11, 11, 11, 0)}, + }; + for (auto & [input, expect_error, expected] : cases) + { + checkNumberToMyDateTime(input, expected, expect_error, nullptr); + } +} +catch (Exception & e) +{ + std::cerr << e.displayText() << std::endl; + GTEST_FAIL(); +} +} // namespace tests } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/DAGContext.cpp b/dbms/src/Flash/Coprocessor/DAGContext.cpp index 12471d66a18..9e364c7cd1f 100644 --- a/dbms/src/Flash/Coprocessor/DAGContext.cpp +++ b/dbms/src/Flash/Coprocessor/DAGContext.cpp @@ -11,6 +11,8 @@ extern const int DIVIDED_BY_ZERO; extern const int INVALID_TIME; } // namespace ErrorCodes +namespace +{ enum Flag { IGNORE_TRUNCATE = 1, @@ -20,22 +22,58 @@ enum Flag IN_UPDATE_OR_DELETE_STMT = 1u << 4u, IN_SELECT_STMT = 1u << 5u, OVERFLOW_AS_WARNING = 1u << 6u, + IGNORE_ZERO_IN_DATE = 1u << 7u, DIVIDED_BY_ZERO_AS_WARNING = 1u << 8u, IN_LOAD_DATA_STMT = 1u << 10u, }; enum SqlMode { - STRICT_TRANS_TABLES = 1u << 22u, - STRICT_ALL_TABLES = 1u << 23u, - NO_ZERO_IN_DATE = 1u << 24u, - NO_ZERO_DATE = 1u << 25u, - INVALID_DATES = 1u << 26u, - ERROR_FOR_DIVISION_BY_ZERO = 1u << 27u, + REAL_AS_FLOAT = 1ul, + PIPES_AS_CONCAT = 1ul << 1ul, + ANSI_QUOTES = 1ul << 2ul, + IGNORE_SPACE = 1ul << 3ul, + NOT_USED = 1ul << 4ul, + ONLY_FULL_GROUP_BY = 1ul << 5ul, + NO_UNSIGNED_SUBTRACTION = 1ul << 6ul, + NO_DIR_IN_CREATE = 1ul << 7ul, + POSTGRESQL = 1ul << 8ul, + ORACLE = 1ul << 9ul, + MSSQL = 1ul << 10ul, + DB2 = 1ul << 11ul, + MAXDB = 1ul << 12ul, + NO_KEY_OPTIONS = 1ul << 13ul, + NO_TABLE_OPTIONS = 1ul << 14ul, + NO_FIELD_OPTIONS = 1ul << 15ul, + MYSQL323 = 1ul << 16ul, + MYSQL40 = 1ul << 17ul, + ANSI = 1ul << 18ul, + NO_AUTO_VALUE_ON_ZERO = 1ul << 19ul, + NO_BACK_SLASH_ESCAPES = 1ul << 20ul, + STRICT_TRANS_TABLES = 1ul << 21ul, + STRICT_ALL_TABLES = 1ul << 22ul, + NO_ZERO_IN_DATE = 1ul << 23ul, + NO_ZERO_DATE = 1ul << 24ul, + INVALID_DATES = 1ul << 25ul, + ERROR_FOR_DIVISION_BY_ZERO = 1ul << 26ul, + TRADITIONAL = 1ul << 27ul, + NO_AUTO_CREATE_USER = 1ul << 28ul, + HIGH_NOT_PRECEDENCE = 1ul << 29ul, + NO_ENGINE_SUBSTITUTION = 1ul << 30ul, + + // Duplicated with Flag::PAD_CHAR_TO_FULL_LENGTH + // PAD_CHAR_TO_FULL_LENGTH = 1ul << 31ul, + + ALLOW_INVALID_DATES = 1ul << 32ul, }; +} // namespace bool strictSqlMode(UInt64 sql_mode) { return sql_mode & SqlMode::STRICT_ALL_TABLES || sql_mode & SqlMode::STRICT_TRANS_TABLES; } +bool DAGContext::allowZeroInDate() const { return flags & Flag::IGNORE_ZERO_IN_DATE; } + +bool DAGContext::allowInvalidDate() const { return sql_mode & SqlMode::ALLOW_INVALID_DATES; } + std::map & DAGContext::getProfileStreamsMap() { return profile_streams_map; } std::unordered_map & DAGContext::getProfileStreamsMapForJoinBuildSide() diff --git a/dbms/src/Flash/Coprocessor/DAGContext.h b/dbms/src/Flash/Coprocessor/DAGContext.h index da39fc8eb2b..67ed7c1b48f 100644 --- a/dbms/src/Flash/Coprocessor/DAGContext.h +++ b/dbms/src/Flash/Coprocessor/DAGContext.h @@ -60,6 +60,8 @@ class DAGContext void handleOverflowError(const String & msg); void handleDivisionByZero(const String & msg); void handleInvalidTime(const String & msg); + bool allowZeroInDate() const; + bool allowInvalidDate() const; bool shouldClipToZero(); const std::vector> & getWarnings() const { return warnings; } const mpp::TaskMeta & getMPPTaskMeta() const { return mpp_task_meta; } diff --git a/dbms/src/Functions/FunctionsTiDBConversion.h b/dbms/src/Functions/FunctionsTiDBConversion.h index cf74e5424be..788644435bb 100644 --- a/dbms/src/Functions/FunctionsTiDBConversion.h +++ b/dbms/src/Functions/FunctionsTiDBConversion.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -1294,7 +1295,8 @@ struct TiDBConvertToTime { try { - MyDateTime datetime = numberToDateTime(vec_from[i]); + MyDateTime datetime(0, 0, 0, 0, 0, 0, 0); + bool is_null = numberToDateTime(vec_from[i], datetime, context.getDAGContext()); if constexpr (std::is_same_v) { MyDate date(datetime.year, datetime.month, datetime.day); @@ -1304,8 +1306,9 @@ struct TiDBConvertToTime { vec_to[i] = datetime.toPackedUInt(); } + (*vec_null_map_to)[i] = is_null; } - catch (const Exception &) + catch (const TiFlashException & e) { // Cannot cast, fill with NULL (*vec_null_map_to)[i] = 1; @@ -1403,6 +1406,115 @@ struct TiDBConvertToTime } }; +inline bool getDatetime(const Int64 & num, MyDateTime & result, DAGContext * ctx) +{ + UInt64 ymd = num / 1000000; + UInt64 hms = num - ymd * 1000000; + + UInt64 year = ymd / 10000; + ymd %= 10000; + UInt64 month = ymd / 100; + UInt64 day = ymd % 100; + + UInt64 hour = hms / 10000; + hms %= 10000; + UInt64 minute = hms / 100; + UInt64 second = hms % 100; + + if (toCoreTimeChecked(year, month, day, hour, minute, second, 0, result)) + { + throw TiFlashException("Incorrect time value", Errors::Types::WrongValue); + } + if (ctx) + { + result.check(ctx->allowZeroInDate(), ctx->allowInvalidDate()); + } + else + { + result.check(false, false); + } + return false; +} + +// Convert a integer number to DateTime and return true if the result is NULL. +// If number is invalid(according to SQL_MODE), return NULL and handle the error with DAGContext. +// This function may throw exception. +inline bool numberToDateTime(Int64 number, MyDateTime & result, DAGContext * ctx) +{ + MyDateTime datetime(0); + if (number == 0) + { + result = datetime; + return false; + } + + // datetime type + if (number >= 10000101000000) + { + return getDatetime(number, result, ctx); + } + + // check MMDD + if (number < 101) + { + throw TiFlashException("Incorrect time value", Errors::Types::WrongValue); + } + + // check YYMMDD: 2000-2069 + if (number <= 69 * 10000 + 1231) + { + number = (number + 20000000) * 1000000; + return getDatetime(number, result, ctx); + } + + if (number < 70 * 10000 + 101) + { + throw TiFlashException("Incorrect time value", Errors::Types::WrongValue); + } + + // check YYMMDD + if (number <= 991231) + { + number = (number + 19000000) * 1000000; + return getDatetime(number, result, ctx); + } + + // check hour/min/second + if (number <= 99991231) + { + number *= 1000000; + return getDatetime(number, result, ctx); + } + + // check MMDDHHMMSS + if (number < 101000000) + { + throw TiFlashException("Incorrect time value", Errors::Types::WrongValue); + } + + // check YYMMDDhhmmss: 2000-2069 + if (number <= 69 * 10000000000 + 1231235959) + { + number += 20000000000000; + return getDatetime(number, result, ctx); + } + + // check YYYYMMDDhhmmss + if (number < 70 * 10000000000 + 101000000) + { + throw TiFlashException("Incorrect time value", Errors::Types::WrongValue); + } + + // check YYMMDDHHMMSS + if (number <= 991231235959) + { + number += 19000000000000; + return getDatetime(number, result, ctx); + } + + return getDatetime(number, result, ctx); +} + class PreparedFunctionTiDBCast : public PreparedFunctionImpl { public: