Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix cast int as time #1788

Merged
merged 12 commits into from
May 12, 2021
Merged
Prev Previous commit
Next Next commit
refine
Signed-off-by: leiysky <leiysky@outlook.com>
leiysky committed Apr 26, 2021
commit 8528e37600f99802ae7f7744b4de7ff1413b0aff
150 changes: 36 additions & 114 deletions dbms/src/Common/MyTime.cpp
Original file line number Diff line number Diff line change
@@ -930,147 +930,69 @@ size_t maxFormattedDateTimeStringLength(const String & format)
return std::max<size_t>(result, 1);
}

void checkDate(const UInt64 & year, const UInt64 & month, const UInt64 & day)
void MyTimeBase::check(bool allow_zero_in_date, bool allow_invalid_date) const
{
if (year == 0 && month == 0 && day == 0)
if (!(year == 0 && month == 0 && day == 0))
{
return;
}

if (month == 0 || day == 0)
{
throw TiFlashException("Invalid Datetime value", Errors::Types::WrongValue);
if (!allow_zero_in_date && (month == 0 || day == 0))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why year == 0 is allowed?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks weird but MySQL does allow this.

Reference from https://dev.mysql.com/doc/refman/5.7/en/sql-mode.html#sqlmode_no_zero_in_date:

The NO_ZERO_IN_DATE mode affects whether the server permits dates in which the year part is nonzero but the month or day part is 0.

image

{
char buff[] = "0000-00-00";
std::sprintf(buff, "%04d-%02d-%02d", year, month, day);
throw TiFlashException("Incorrect datetime value: " + String(buff), Errors::Types::WrongValue);
}
}

if (year >= 9999 || month > 12)
{
throw TiFlashException("Invalid Datetime value", Errors::Types::WrongValue);
throw TiFlashException("Incorrect time value", Errors::Types::WrongValue);
}

UInt8 max_day = 31;
if (!allow_invalid_date)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So day should never be bigger than 31 even if allow_invalid_date is true?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. 🤣
image

{
constexpr static UInt8 max_days_in_month[12] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
UInt8 max_day = max_days_in_month[month - 1];
static auto is_leap_year = [](UInt16 year) { return ((year % 4 == 0) && (year % 100 != 0)) || (year % 400 == 0); };
static auto is_leap_year = [](UInt16 _year) { return ((_year % 4 == 0) && (_year % 100 != 0)) || (_year % 400 == 0); };
max_day = max_days_in_month[month - 1];
if (month == 2 && is_leap_year(year))
{
max_day = 29;
}

if (day > max_day)
{
throw TiFlashException("Invalid Datetime value", Errors::Types::WrongValue);
}
}
return;
}

MyDateTime checkedDateTime(const UInt64 & year, const UInt64 & month, const UInt64 & day, const UInt64 & hour, const UInt64 & minute,
const UInt64 & second, const UInt64 & microsecond)
{
if (year >= (1 << MyTimeBase::YEAR_BIT_FIELD_WIDTH) || month >= (1 << MyTimeBase::MONTH_BIT_FIELD_WIDTH)
|| day >= (1 << MyTimeBase::DAY_BIT_FIELD_WIDTH) || hour >= (1 << MyTimeBase::HOUR_BIT_FIELD_WIDTH)
|| minute >= (1 << MyTimeBase::MINUTE_BIT_FIELD_WIDTH) || second >= (1 << MyTimeBase::SECOND_BIT_FIELD_WIDTH)
|| microsecond >= (1 << MyTimeBase::MICROSECOND_BIT_FIELD_WIDTH))
{
throw TiFlashException("Datetime value overflow", Errors::Types::WrongValue);
}
checkDate(year, month, day);
if (hour >= 24 || minute >= 60 || second >= 60)
{
throw TiFlashException("Invalid Datetime value", Errors::Types::WrongValue);
}
return MyDateTime(year, month, day, hour, minute, second, microsecond);
}

MyDateTime numberToDateTime(Int64 number)
{
MyDateTime datetime(0);

auto get_datetime = [](const Int64 & num) {
UInt64 ymd = num / 1000000;
UInt64 hms = num - ymd * 1000000;

UInt64 year = ymd / 10000;
ymd %= 10000;
UInt64 month = ymd / 100;
UInt64 day = ymd % 100;

UInt64 hour = hms / 10000;
hms %= 10000;
UInt64 minute = hms / 100;
UInt64 second = hms % 100;

return checkedDateTime(year, month, day, hour, minute, second, 0);
};

if (number == 0)
return datetime;

// datetime type
if (number >= 10000101000000)
{
return get_datetime(number);
}

// check MMDD
if (number < 101)
if (day > max_day)
{
throw TiFlashException("Cannot convert " + std::to_string(number) + " to Datetime", Errors::Types::WrongValue);
char buff[] = "0000-00-00";
std::sprintf(buff, "%04d-%02d-%02d", year, month, day);
throw TiFlashException("Incorrect datetime value: " + String(buff), Errors::Types::WrongValue);
}

// check YYMMDD: 2000-2069
if (number <= 69 * 10000 + 1231)
if (hour < 0 || hour >= 24)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Theoretically speaking, for a MyDate value, hour,minute,second is meaningless, they can be any value, we should not throw Incorrect datetime value for a MyDate if its hour is invalid.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, but MySQL doesn't really distinguish them in this case.
image

{
number = (number + 20000000) * 1000000;
return get_datetime(number);
throw TiFlashException("Incorrect datetime value", Errors::Types::WrongValue);
}

if (number < 70 * 10000 + 101)
if (minute >= 60)
{
throw TiFlashException("Cannot convert " + std::to_string(number) + " to Datetime", Errors::Types::WrongValue);
throw TiFlashException("Incorrect datetime value", Errors::Types::WrongValue);
}

// check YYMMDD
if (number <= 991231)
if (second >= 60)
{
number = (number + 19000000) * 1000000;
return get_datetime(number);
}

// check hour/min/second
if (number <= 99991231)
{
number *= 1000000;
return get_datetime(number);
}

// check MMDDHHMMSS
if (number < 101000000)
{
throw TiFlashException("Cannot convert " + std::to_string(number) + " to Datetime", Errors::Types::WrongValue);
}

// check YYMMDDhhmmss: 2000-2069
if (number <= 69 * 10000000000 + 1231235959)
{
number += 20000000000000;
return get_datetime(number);
}

// check YYYYMMDDhhmmss
if (number < 70 * 10000000000 + 101000000)
{
throw TiFlashException("Cannot convert " + std::to_string(number) + " to Datetime", Errors::Types::WrongValue);
throw TiFlashException("Incorrect datetime value", Errors::Types::WrongValue);
}
return;
}

// check YYMMDDHHMMSS
if (number <= 991231235959)
bool checkedDateTime(const UInt64 & year, const UInt64 & month, const UInt64 & day, const UInt64 & hour, const UInt64 & minute,
const UInt64 & second, const UInt64 & microsecond, MyDateTime & result)
{
if (year >= (1 << MyTimeBase::YEAR_BIT_FIELD_WIDTH) || month >= (1 << MyTimeBase::MONTH_BIT_FIELD_WIDTH)
|| day >= (1 << MyTimeBase::DAY_BIT_FIELD_WIDTH) || hour >= (1 << MyTimeBase::HOUR_BIT_FIELD_WIDTH)
|| minute >= (1 << MyTimeBase::MINUTE_BIT_FIELD_WIDTH) || second >= (1 << MyTimeBase::SECOND_BIT_FIELD_WIDTH)
|| microsecond >= (1 << MyTimeBase::MICROSECOND_BIT_FIELD_WIDTH))
{
number += 19000000000000;
return get_datetime(number);
result = MyDateTime(0, 0, 0, 0, 0, 0, 0);
return true;
}

return get_datetime(number);
result = MyDateTime(year, month, day, hour, minute, second, microsecond);
return false;
}

MyDateTimeFormatter::MyDateTimeFormatter(const String & layout)
9 changes: 8 additions & 1 deletion dbms/src/Common/MyTime.h
Original file line number Diff line number Diff line change
@@ -89,6 +89,10 @@ struct MyTimeBase
int week(UInt32 mode) const;

std::tuple<int, int> calcWeek(UInt32 mode) const;

// Check validity of time under specified SQL_MODE.
// May throw exception.
void check(bool allow_zero_in_date, bool allow_invalid_date) const;
};

struct MyDateTime : public MyTimeBase
@@ -139,11 +143,14 @@ int calcDayNum(int year, int month, int day);

size_t maxFormattedDateTimeStringLength(const String & format);

MyDateTime numberToDateTime(Int64 number);


bool isPunctuation(char c);

bool isValidSeperator(char c, int previous_parts);

// Build MyDateTime value with checking overflow of internal fields, return true if input is invalid.
bool checkedDateTime(const UInt64 & year, const UInt64 & month, const UInt64 & day, const UInt64 & hour, const UInt64 & minute,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The function name is misleading, better to use a more appropriate name.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed

const UInt64 & second, const UInt64 & microsecond, MyDateTime & result);

} // namespace DB
16 changes: 11 additions & 5 deletions dbms/src/Common/tests/gtest_mytime.cpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#include <Common/Exception.h>
#include <Common/MyTime.h>
#include <DataTypes/DataTypeMyDateTime.h>
#include <Functions/FunctionsTiDBConversion.h>
#include <gtest/gtest.h>

#include <iostream>
@@ -57,17 +58,19 @@ class TestMyTime : public testing::Test
}
}

static void checkNumberToMyDateTime(const Int64 & input, const MyDateTime & expected, bool expect_error)
static void checkNumberToMyDateTime(const Int64 & input, const MyDateTime & expected, bool expect_error, DAGContext * ctx)
{
if (expect_error)
{
EXPECT_THROW({ numberToDateTime(input); }, TiFlashException) << "Original time number: " << input;
MyDateTime datetime(0, 0, 0, 0, 0, 0, 0);
EXPECT_THROW({ numberToDateTime(input, datetime, ctx); }, TiFlashException) << "Original time number: " << input;
return;
}

try
{
MyDateTime source = numberToDateTime(input);
MyDateTime source(0, 0, 0, 0, 0, 0, 0);
numberToDateTime(input, source, ctx);
EXPECT_EQ(source.year, expected.year) << "Original time number: " << input;
EXPECT_EQ(source.month, expected.month) << "Original time number: " << input;
EXPECT_EQ(source.day, expected.day) << "Original time number: " << input;
@@ -197,17 +200,21 @@ try
{201, false, MyDateTime(2000, 2, 1, 0, 0, 0, 0)},
{20, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)},
{2, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)},
{0, false, MyDateTime(0, 0, 0, 0, 0, 0, 0)},
{-1, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)},
{99999999999999, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)},
{100000000000000, true, MyDateTime(0, 0, 0, 0, 0, 0, 0)},
{10000102000000, false, MyDateTime(1000, 1, 2, 0, 0, 0, 0)},
{19690101000000, false, MyDateTime(1969, 1, 1, 0, 0, 0, 0)},
{991231235959, false, MyDateTime(1999, 12, 31, 23, 59, 59, 0)},
{691231235959, false, MyDateTime(2069, 12, 31, 23, 59, 59, 0)},
{370119031407, false, MyDateTime(2037, 1, 19, 3, 14, 7, 0)},
{380120031407, false, MyDateTime(2038, 1, 20, 3, 14, 7, 0)},
{11111111111, false, MyDateTime(2001, 11, 11, 11, 11, 11, 0)},
};
for (auto & [input, expect_error, expected] : cases)
{
checkNumberToMyDateTime(input, expected, expect_error);
checkNumberToMyDateTime(input, expected, expect_error, nullptr);
}
}
catch (Exception & e)
@@ -217,5 +224,4 @@ catch (Exception & e)
}

} // namespace tests

} // namespace DB
50 changes: 44 additions & 6 deletions dbms/src/Flash/Coprocessor/DAGContext.cpp
Original file line number Diff line number Diff line change
@@ -11,6 +11,8 @@ extern const int DIVIDED_BY_ZERO;
extern const int INVALID_TIME;
} // namespace ErrorCodes

namespace
{
enum Flag
{
IGNORE_TRUNCATE = 1,
@@ -20,22 +22,58 @@ enum Flag
IN_UPDATE_OR_DELETE_STMT = 1u << 4u,
IN_SELECT_STMT = 1u << 5u,
OVERFLOW_AS_WARNING = 1u << 6u,
IGNORE_ZERO_IN_DATE = 1u << 7u,
DIVIDED_BY_ZERO_AS_WARNING = 1u << 8u,
IN_LOAD_DATA_STMT = 1u << 10u,
};

enum SqlMode
{
STRICT_TRANS_TABLES = 1u << 22u,
STRICT_ALL_TABLES = 1u << 23u,
NO_ZERO_IN_DATE = 1u << 24u,
NO_ZERO_DATE = 1u << 25u,
INVALID_DATES = 1u << 26u,
ERROR_FOR_DIVISION_BY_ZERO = 1u << 27u,
REAL_AS_FLOAT = 1ul,
PIPES_AS_CONCAT = 1ul << 1ul,
ANSI_QUOTES = 1ul << 2ul,
IGNORE_SPACE = 1ul << 3ul,
NOT_USED = 1ul << 4ul,
ONLY_FULL_GROUP_BY = 1ul << 5ul,
NO_UNSIGNED_SUBTRACTION = 1ul << 6ul,
NO_DIR_IN_CREATE = 1ul << 7ul,
POSTGRESQL = 1ul << 8ul,
ORACLE = 1ul << 9ul,
MSSQL = 1ul << 10ul,
DB2 = 1ul << 11ul,
MAXDB = 1ul << 12ul,
NO_KEY_OPTIONS = 1ul << 13ul,
NO_TABLE_OPTIONS = 1ul << 14ul,
NO_FIELD_OPTIONS = 1ul << 15ul,
MYSQL323 = 1ul << 16ul,
MYSQL40 = 1ul << 17ul,
ANSI = 1ul << 18ul,
NO_AUTO_VALUE_ON_ZERO = 1ul << 19ul,
NO_BACK_SLASH_ESCAPES = 1ul << 20ul,
STRICT_TRANS_TABLES = 1ul << 21ul,
STRICT_ALL_TABLES = 1ul << 22ul,
NO_ZERO_IN_DATE = 1ul << 23ul,
NO_ZERO_DATE = 1ul << 24ul,
INVALID_DATES = 1ul << 25ul,
ERROR_FOR_DIVISION_BY_ZERO = 1ul << 26ul,
TRADITIONAL = 1ul << 27ul,
NO_AUTO_CREATE_USER = 1ul << 28ul,
HIGH_NOT_PRECEDENCE = 1ul << 29ul,
NO_ENGINE_SUBSTITUTION = 1ul << 30ul,

// Duplicated with Flag::PAD_CHAR_TO_FULL_LENGTH
// PAD_CHAR_TO_FULL_LENGTH = 1ul << 31ul,

ALLOW_INVALID_DATES = 1ul << 32ul,
};
} // namespace

bool strictSqlMode(UInt64 sql_mode) { return sql_mode & SqlMode::STRICT_ALL_TABLES || sql_mode & SqlMode::STRICT_TRANS_TABLES; }

bool DAGContext::allowZeroInDate() const { return flags & Flag::IGNORE_ZERO_IN_DATE; }

bool DAGContext::allowInvalidDate() const { return sql_mode & SqlMode::ALLOW_INVALID_DATES; }

std::map<String, ProfileStreamsInfo> & DAGContext::getProfileStreamsMap() { return profile_streams_map; }

std::unordered_map<String, BlockInputStreams> & DAGContext::getProfileStreamsMapForJoinBuildSide()
2 changes: 2 additions & 0 deletions dbms/src/Flash/Coprocessor/DAGContext.h
Original file line number Diff line number Diff line change
@@ -59,6 +59,8 @@ class DAGContext
void handleOverflowError(const String & msg);
void handleDivisionByZero(const String & msg);
void handleInvalidTime(const String & msg);
bool allowZeroInDate() const;
bool allowInvalidDate() const;
bool shouldClipToZero();
const std::vector<std::pair<Int32, String>> & getWarnings() const { return warnings; }
const mpp::TaskMeta & getMPPTaskMeta() const { return mpp_task_meta; }
Loading