Skip to content

Commit

Permalink
Fix cast int as time (#1788) (#1892)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ti-srebot
ti-srebot authored Jul 7, 2021
1 parent 858e93e commit 3e1dd3c
Show file tree
Hide file tree
Showing 6 changed files with 282 additions and 79 deletions.
110 changes: 42 additions & 68 deletions dbms/src/Common/MyTime.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -930,95 +930,69 @@ size_t maxFormattedDateTimeStringLength(const String & format)
return std::max<size_t>(result, 1);
}

MyDateTime numberToDateTime(Int64 number)
void MyTimeBase::check(bool allow_zero_in_date, bool allow_invalid_date) const
{
MyDateTime datetime(0);

auto get_datetime = [](const Int64 & num) {
auto ymd = num / 1000000;
auto hms = num - ymd * 1000000;

UInt16 year = ymd / 10000;
ymd %= 10000;
UInt8 month = ymd / 100;
UInt8 day = ymd % 100;

UInt16 hour = hms / 10000;
hms %= 10000;
UInt8 minute = hms / 100;
UInt8 second = hms % 100;

return MyDateTime(year, month, day, hour, minute, second, 0);
};

if (number == 0)
return datetime;

// datetime type
if (number >= 10000101000000)
{
return get_datetime(number);
}

// check MMDD
if (number < 101)
{
throw TiFlashException("Cannot convert " + std::to_string(number) + " to Datetime", Errors::Types::WrongValue);
}

// check YYMMDD: 2000-2069
if (number <= 69 * 10000 + 1231)
if (!(year == 0 && month == 0 && day == 0))
{
number = (number + 200000000) * 1000000;
return get_datetime(number);
if (!allow_zero_in_date && (month == 0 || day == 0))
{
char buff[] = "0000-00-00";
std::sprintf(buff, "%04d-%02d-%02d", year, month, day);
throw TiFlashException("Incorrect datetime value: " + String(buff), Errors::Types::WrongValue);
}
}

// check YYMMDD
if (number <= 991231)
if (year >= 9999 || month > 12)
{
number = (number + 19000000) * 1000000;
return get_datetime(number);
throw TiFlashException("Incorrect time value", Errors::Types::WrongValue);
}

// check YYYYMMDD
if (number <= 10000101)
UInt8 max_day = 31;
if (!allow_invalid_date)
{
throw TiFlashException("Cannot convert " + std::to_string(number) + " to Datetime", Errors::Types::WrongValue);
constexpr static UInt8 max_days_in_month[12] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
static auto is_leap_year = [](UInt16 _year) { return ((_year % 4 == 0) && (_year % 100 != 0)) || (_year % 400 == 0); };
max_day = max_days_in_month[month - 1];
if (month == 2 && is_leap_year(year))
{
max_day = 29;
}
}

// check hhmmss
if (number <= 99991231)
if (day > max_day)
{
number = number * 1000000;
return get_datetime(number);
char buff[] = "0000-00-00";
std::sprintf(buff, "%04d-%02d-%02d", year, month, day);
throw TiFlashException("Incorrect datetime value: " + String(buff), Errors::Types::WrongValue);
}

// check MMDDhhmmss
if (number < 101000000)
if (hour < 0 || hour >= 24)
{
throw TiFlashException("Cannot convert " + std::to_string(number) + " to Datetime", Errors::Types::WrongValue);
throw TiFlashException("Incorrect datetime value", Errors::Types::WrongValue);
}

// check YYMMDDhhmmss: 2000-2069
if (number <= 69 * 10000000000 + 1231235959)
if (minute >= 60)
{
number += 20000000000000;
return get_datetime(number);
throw TiFlashException("Incorrect datetime value", Errors::Types::WrongValue);
}

// check YYMMDDhhmmss
if (number < 70 * 10000000000 + 101000000)
if (second >= 60)
{
throw TiFlashException("Cannot convert " + std::to_string(number) + " to Datetime", Errors::Types::WrongValue);
throw TiFlashException("Incorrect datetime value", Errors::Types::WrongValue);
}
return;
}

if (number <= 991231235959)
bool toCoreTimeChecked(const UInt64 & year, const UInt64 & month, const UInt64 & day, const UInt64 & hour, const UInt64 & minute,
const UInt64 & second, const UInt64 & microsecond, MyDateTime & result)
{
if (year >= (1 << MyTimeBase::YEAR_BIT_FIELD_WIDTH) || month >= (1 << MyTimeBase::MONTH_BIT_FIELD_WIDTH)
|| day >= (1 << MyTimeBase::DAY_BIT_FIELD_WIDTH) || hour >= (1 << MyTimeBase::HOUR_BIT_FIELD_WIDTH)
|| minute >= (1 << MyTimeBase::MINUTE_BIT_FIELD_WIDTH) || second >= (1 << MyTimeBase::SECOND_BIT_FIELD_WIDTH)
|| microsecond >= (1 << MyTimeBase::MICROSECOND_BIT_FIELD_WIDTH))
{
number += 19000000000000;
return get_datetime(number);
result = MyDateTime(0, 0, 0, 0, 0, 0, 0);
return true;
}

return get_datetime(number);
result = MyDateTime(year, month, day, hour, minute, second, microsecond);
return false;
}

MyDateTimeFormatter::MyDateTimeFormatter(const String & layout)
Expand Down
10 changes: 9 additions & 1 deletion dbms/src/Common/MyTime.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,10 @@ struct MyTimeBase
int week(UInt32 mode) const;

std::tuple<int, int> calcWeek(UInt32 mode) const;

// Check validity of time under specified SQL_MODE.
// May throw exception.
void check(bool allow_zero_in_date, bool allow_invalid_date) const;
};

struct MyDateTime : public MyTimeBase
Expand Down Expand Up @@ -139,11 +143,15 @@ int calcDayNum(int year, int month, int day);

size_t maxFormattedDateTimeStringLength(const String & format);

MyDateTime numberToDateTime(Int64 number);


bool isPunctuation(char c);

bool isValidSeperator(char c, int previous_parts);

// Build CoreTime value with checking overflow of internal bit fields, return true if input is invalid.
// Note that this function will not check if the input is logically a valid datetime value.
bool toCoreTimeChecked(const UInt64 & year, const UInt64 & month, const UInt64 & day, const UInt64 & hour, const UInt64 & minute,
const UInt64 & second, const UInt64 & microsecond, MyDateTime & result);

} // namespace DB
73 changes: 71 additions & 2 deletions dbms/src/Common/tests/gtest_mytime.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#include <Common/Exception.h>
#include <Common/MyTime.h>
#include <DataTypes/DataTypeMyDateTime.h>
#include <Functions/FunctionsTiDBConversion.h>
#include <gtest/gtest.h>

#include <iostream>
Expand Down Expand Up @@ -36,7 +37,7 @@ class TestMyTime : public testing::Test
}
}

static void checkParseMyDateTime(const std::string & str, MyDateTime & expected, const DataTypeMyDateTime & type)
static void checkParseMyDateTime(const std::string & str, const MyDateTime & expected, const DataTypeMyDateTime & type)
{
try
{
Expand All @@ -56,6 +57,34 @@ class TestMyTime : public testing::Test
throw;
}
}

static void checkNumberToMyDateTime(const Int64 & input, const MyDateTime & expected, bool expect_error, DAGContext * ctx)
{
if (expect_error)
{
MyDateTime datetime(0, 0, 0, 0, 0, 0, 0);
EXPECT_THROW({ numberToDateTime(input, datetime, ctx); }, TiFlashException) << "Original time number: " << input;
return;
}

try
{
MyDateTime source(0, 0, 0, 0, 0, 0, 0);
numberToDateTime(input, source, ctx);
EXPECT_EQ(source.year, expected.year) << "Original time number: " << input;
EXPECT_EQ(source.month, expected.month) << "Original time number: " << input;
EXPECT_EQ(source.day, expected.day) << "Original time number: " << input;
EXPECT_EQ(source.hour, expected.hour) << "Original time number: " << input;
EXPECT_EQ(source.minute, expected.minute) << "Original time number: " << input;
EXPECT_EQ(source.second, expected.second) << "Original time number: " << input;
EXPECT_EQ(source.micro_second, expected.micro_second) << "Original time number: " << input;
}
catch (...)
{
std::cerr << "Error occurs when parsing: \"" << input << "\"" << std::endl;
throw;
}
}
};

TEST_F(TestMyTime, ParseMyDateTimeWithFraction)
Expand Down Expand Up @@ -153,6 +182,46 @@ catch (Exception & e)
GTEST_FAIL();
}

} // namespace tests
TEST_F(TestMyTime, NumberToDateTime)
try
{
std::vector<std::tuple<Int64, bool /* ExpectError */, MyDateTime>> cases{
{20101010111111, false, MyDateTime(2010, 10, 10, 11, 11, 11, 0)},
{2010101011111, false, MyDateTime(201, 1, 1, 1, 11, 11, 0)},
{201010101111, false, MyDateTime(2020, 10, 10, 10, 11, 11, 0)},
{20101010111, false, MyDateTime(2002, 1, 1, 1, 1, 11, 0)},
{2010101011, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)},
{201010101, false, MyDateTime(2000, 2, 1, 1, 1, 1, 0)},
{20101010, false, MyDateTime(2010, 10, 10, 0, 0, 0, 0)},
{2010101, false, MyDateTime(201, 1, 1, 0, 0, 0, 0)},
{201010, false, MyDateTime(2020, 10, 10, 0, 0, 0, 0)},
{20101, false, MyDateTime(2002, 1, 1, 0, 0, 0, 0)},
{2010, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)},
{201, false, MyDateTime(2000, 2, 1, 0, 0, 0, 0)},
{20, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)},
{2, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)},
{0, false, MyDateTime(0, 0, 0, 0, 0, 0, 0)},
{-1, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)},
{99999999999999, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)},
{100000000000000, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)},
{10000102000000, false, MyDateTime(1000, 1, 2, 0, 0, 0, 0)},
{19690101000000, false, MyDateTime(1969, 1, 1, 0, 0, 0, 0)},
{991231235959, false, MyDateTime(1999, 12, 31, 23, 59, 59, 0)},
{691231235959, false, MyDateTime(2069, 12, 31, 23, 59, 59, 0)},
{370119031407, false, MyDateTime(2037, 1, 19, 3, 14, 7, 0)},
{380120031407, false, MyDateTime(2038, 1, 20, 3, 14, 7, 0)},
{11111111111, false, MyDateTime(2001, 11, 11, 11, 11, 11, 0)},
};
for (auto & [input, expect_error, expected] : cases)
{
checkNumberToMyDateTime(input, expected, expect_error, nullptr);
}
}
catch (Exception & e)
{
std::cerr << e.displayText() << std::endl;
GTEST_FAIL();
}

} // namespace tests
} // namespace DB
50 changes: 44 additions & 6 deletions dbms/src/Flash/Coprocessor/DAGContext.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ extern const int DIVIDED_BY_ZERO;
extern const int INVALID_TIME;
} // namespace ErrorCodes

namespace
{
enum Flag
{
IGNORE_TRUNCATE = 1,
Expand All @@ -20,22 +22,58 @@ enum Flag
IN_UPDATE_OR_DELETE_STMT = 1u << 4u,
IN_SELECT_STMT = 1u << 5u,
OVERFLOW_AS_WARNING = 1u << 6u,
IGNORE_ZERO_IN_DATE = 1u << 7u,
DIVIDED_BY_ZERO_AS_WARNING = 1u << 8u,
IN_LOAD_DATA_STMT = 1u << 10u,
};

enum SqlMode
{
STRICT_TRANS_TABLES = 1u << 22u,
STRICT_ALL_TABLES = 1u << 23u,
NO_ZERO_IN_DATE = 1u << 24u,
NO_ZERO_DATE = 1u << 25u,
INVALID_DATES = 1u << 26u,
ERROR_FOR_DIVISION_BY_ZERO = 1u << 27u,
REAL_AS_FLOAT = 1ul,
PIPES_AS_CONCAT = 1ul << 1ul,
ANSI_QUOTES = 1ul << 2ul,
IGNORE_SPACE = 1ul << 3ul,
NOT_USED = 1ul << 4ul,
ONLY_FULL_GROUP_BY = 1ul << 5ul,
NO_UNSIGNED_SUBTRACTION = 1ul << 6ul,
NO_DIR_IN_CREATE = 1ul << 7ul,
POSTGRESQL = 1ul << 8ul,
ORACLE = 1ul << 9ul,
MSSQL = 1ul << 10ul,
DB2 = 1ul << 11ul,
MAXDB = 1ul << 12ul,
NO_KEY_OPTIONS = 1ul << 13ul,
NO_TABLE_OPTIONS = 1ul << 14ul,
NO_FIELD_OPTIONS = 1ul << 15ul,
MYSQL323 = 1ul << 16ul,
MYSQL40 = 1ul << 17ul,
ANSI = 1ul << 18ul,
NO_AUTO_VALUE_ON_ZERO = 1ul << 19ul,
NO_BACK_SLASH_ESCAPES = 1ul << 20ul,
STRICT_TRANS_TABLES = 1ul << 21ul,
STRICT_ALL_TABLES = 1ul << 22ul,
NO_ZERO_IN_DATE = 1ul << 23ul,
NO_ZERO_DATE = 1ul << 24ul,
INVALID_DATES = 1ul << 25ul,
ERROR_FOR_DIVISION_BY_ZERO = 1ul << 26ul,
TRADITIONAL = 1ul << 27ul,
NO_AUTO_CREATE_USER = 1ul << 28ul,
HIGH_NOT_PRECEDENCE = 1ul << 29ul,
NO_ENGINE_SUBSTITUTION = 1ul << 30ul,

// Duplicated with Flag::PAD_CHAR_TO_FULL_LENGTH
// PAD_CHAR_TO_FULL_LENGTH = 1ul << 31ul,

ALLOW_INVALID_DATES = 1ul << 32ul,
};
} // namespace

bool strictSqlMode(UInt64 sql_mode) { return sql_mode & SqlMode::STRICT_ALL_TABLES || sql_mode & SqlMode::STRICT_TRANS_TABLES; }

bool DAGContext::allowZeroInDate() const { return flags & Flag::IGNORE_ZERO_IN_DATE; }

bool DAGContext::allowInvalidDate() const { return sql_mode & SqlMode::ALLOW_INVALID_DATES; }

std::map<String, ProfileStreamsInfo> & DAGContext::getProfileStreamsMap() { return profile_streams_map; }

std::unordered_map<String, BlockInputStreams> & DAGContext::getProfileStreamsMapForJoinBuildSide()
Expand Down
2 changes: 2 additions & 0 deletions dbms/src/Flash/Coprocessor/DAGContext.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ class DAGContext
void handleOverflowError(const String & msg);
void handleDivisionByZero(const String & msg);
void handleInvalidTime(const String & msg);
bool allowZeroInDate() const;
bool allowInvalidDate() const;
bool shouldClipToZero();
const std::vector<std::pair<Int32, String>> & getWarnings() const { return warnings; }

Expand Down
Loading

0 comments on commit 3e1dd3c

Please sign in to comment.