diff --git a/dbms/src/Common/MyTime.cpp b/dbms/src/Common/MyTime.cpp new file mode 100644 index 00000000000..8a9f9b219ac --- /dev/null +++ b/dbms/src/Common/MyTime.cpp @@ -0,0 +1,468 @@ +#include + +#include +#include +#include + +#include + +namespace DB +{ + +using std::sprintf; + +int adjustYear(int year) +{ + if (year >= 0 && year <= 69) + return 2000 + year; + if (year >= 70 && year <= 99) + return 1900 + year; + return year; +} + +void scanTimeArgs(const std::vector & seps, std::initializer_list && list) +{ + int i = 0; + for (auto * ptr : list) + { + *ptr = std::stoi(seps[i]); + i++; + } +} + +// find index of fractional point. +int getFracIndex(const String & format) +{ + int idx = -1; + for (int i = int(format.size()) - 1; i >= 0; i--) + { + if (std::ispunct(format[i])) + { + if (format[i] == '.') + { + idx = i; + } + break; + } + } + return idx; +} + +std::vector parseDateFormat(String format) +{ + format = Poco::trimInPlace(format); + + std::vector seps; + size_t start = 0; + for (size_t i = 0; i < format.size(); i++) + { + if (i == 0 || i + 1 == format.size()) + { + if (!std::isdigit(format[i])) + return {}; + continue; + } + + if (!std::isdigit(format[i])) + { + if (!std::isdigit(format[i - 1])) + return {}; + seps.push_back(format.substr(start, i - start)); + start = i + 1; + } + } + seps.push_back(format.substr(start)); + return seps; +} + +std::pair, String> splitDatetime(String format) +{ + int idx = getFracIndex(format); + String frac; + if (idx > 0) + { + frac = format.substr(idx + 1); + format = format.substr(0, idx); + } + return std::make_pair(parseDateFormat(format), std::move(frac)); +} + +MyTimeBase::MyTimeBase(UInt64 packed, UInt8 fsp_) +{ + fsp = fsp_; + + UInt64 ymdhms = packed >> 24; + UInt64 ymd = ymdhms >> 17; + day = UInt8(ymd & ((1 << 5) - 1)); + UInt64 ym = ymd >> 5; + month = UInt8(ym % 13); + year = UInt16(ym / 13); + + UInt64 hms = ymdhms & ((1 << 17) - 1); + second = UInt8(hms & ((1 << 6) - 1)); + minute = UInt8((hms >> 6) & ((1 << 6) - 1)); + hour = UInt16(hms >> 12); + + micro_second = packed % (1 << 24); +} + +MyTimeBase::MyTimeBase(UInt16 year_, UInt8 month_, UInt8 day_, UInt16 hour_, UInt8 minute_, UInt8 second_, UInt32 micro_second_, UInt8 fsp_) + : year(year_), month(month_), day(day_), hour(hour_), minute(minute_), second(second_), micro_second(micro_second_), fsp(fsp_) +{} + +UInt64 MyTimeBase::toPackedUInt() const +{ + UInt64 ymd = ((year * 13 + month) << 5) | day; + UInt64 hms = (hour << 12) | (minute << 6) | second; + return (ymd << 17 | hms) << 24 | micro_second; +} + +String MyTimeBase::dateFormat(const String & layout) const +{ + String result; + bool in_pattern_match = false; + for (size_t i = 0; i < layout.size(); i++) + { + char x = layout[i]; + if (in_pattern_match) + { + convertDateFormat(x, result); + in_pattern_match = false; + continue; + } + + if (x == '%') + in_pattern_match = true; + else + result.push_back(x); + } + return result; +} + +void MyTimeBase::convertDateFormat(char c, String & result) const +{ + // TODO:: Implement other formats. + switch (c) + { + //case 'b': + //{ + // if (month == 0 || month > 12) + // { + // throw Exception("invalid time format"); + // } + // result.append(String(MonthNames[month-1], 3)); + // break; + //} + //case 'M': + //{ + // if (month == 0 || month > 12) + // { + // throw Exception("invalid time format"); + // } + // result.append(String(MonthNames[month-1], 3)); + // break; + //} + case 'm': + { + char buf[16]; + sprintf(buf, "%02d", month); + result.append(String(buf)); + break; + } + case 'c': + { + char buf[16]; + sprintf(buf, "%d", month); + result.append(String(buf)); + break; + } + //case 'D': + //{ + // char buf[16]; + // sprintf(buf, "%d", month); + // result.append(String(buf)); + // result.append(abbrDayOfMonth(day)); + // break; + //} + case 'd': + { + char buf[16]; + sprintf(buf, "%02d", day); + result.append(String(buf)); + break; + } + case 'e': + { + char buf[16]; + sprintf(buf, "%d", day); + result.append(String(buf)); + break; + } + //case 'j': + //{ + // char buf[16]; + // sprintf(buf, "%03d", yearDay()); + // result.append(String(buf)); + // break; + //} + case 'H': + { + char buf[16]; + sprintf(buf, "%02d", hour); + result.append(String(buf)); + break; + } + case 'k': + { + char buf[16]; + sprintf(buf, "%d", hour); + result.append(String(buf)); + break; + } + case 'h': + case 'I': + { + char buf[16]; + if (hour % 12 == 0) + sprintf(buf, "%d", 12); + else + sprintf(buf, "%02d", hour); + result.append(String(buf)); + break; + } + case 'l': + { + char buf[16]; + if (hour % 12 == 0) + sprintf(buf, "%d", 12); + else + sprintf(buf, "%d", hour); + result.append(String(buf)); + break; + } + case 'i': + { + char buf[16]; + sprintf(buf, "%02d", minute); + result.append(String(buf)); + break; + } + case 'p': + { + if ((hour / 12) % 2) + result.append(String("PM")); + else + result.append(String("AM")); + break; + } + case 'r': + { + char buf[24]; + auto h = hour % 24; + if (h == 0) + sprintf(buf, "%02d:%02d:%02d AM", 12, minute, second); + else if (h == 12) + sprintf(buf, "%02d:%02d:%02d PM", 12, minute, second); + else if (h < 12) + sprintf(buf, "%02d:%02d:%02d AM", h, minute, second); + else + sprintf(buf, "%02d:%02d:%02d PM", h - 12, minute, second); + result.append(String(buf)); + break; + } + case 'T': + { + char buf[24]; + sprintf(buf, "%02d:%02d:%02d", hour, minute, second); + result.append(String(buf)); + break; + } + case 'S': + case 's': + { + char buf[16]; + sprintf(buf, "%02d", second); + result.append(String(buf)); + break; + } + case 'f': + { + char buf[16]; + sprintf(buf, "%06d", micro_second); + result.append(String(buf)); + break; + } + //case 'U': + //{ + // char buf[16]; + // auto w = week(0); + // sprintf(buf, "%02d", w); + // result.append(String(buf)); + // break; + //} + //case 'u': + //{ + // char buf[16]; + // auto w = week(1); + // sprintf(buf, "%02d", w); + // result.append(String(buf)); + // break; + //} + //case 'V': + //{ + // char buf[16]; + // auto w = week(2); + // sprintf(buf, "%02d", w); + // result.append(String(buf)); + // break; + //} + //case 'v': + //{ + // char buf[16]; + // auto w = yearWeek(2); + // sprintf(buf, "%02d", w); + // result.append(String(buf)); + // break; + //} + //case 'a': + //{ + // auto weekDay = weekDay(); + // result.append(String(abbrevWeekdayName[weekDay])); + // break; + //} + //case 'w': + //{ + // auto weekDay = weekDay(); + // result.append(std::to_string(weekDay)); + // break; + //} + case 'Y': + { + char buf[16]; + sprintf(buf, "%04d", year); + result.append(String(buf)); + break; + } + case 'y': + { + char buf[16]; + sprintf(buf, "%02d", year % 100); + result.append(String(buf)); + break; + } + default: + { + result.push_back(c); + } + } +} + +Field parseMyDateTime(const String & str) +{ + Int32 year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0; + + auto [seps, frac_str] = splitDatetime(str); + + switch (seps.size()) + { + // No delimiter + case 1: + { + size_t l = seps[0].size(); + switch (l) + { + case 14: + // YYYYMMDDHHMMSS + { + std::sscanf(seps[0].c_str(), "%4d%2d%2d%2d%2d%2d", &year, &month, &day, &hour, &minute, &second); + break; + } + case 12: + { + std::sscanf(seps[0].c_str(), "%2d%2d%2d%2d%2d%2d", &year, &month, &day, &hour, &minute, &second); + year = adjustYear(year); + break; + } + case 11: + { + std::sscanf(seps[0].c_str(), "%2d%2d%2d%2d%2d%1d", &year, &month, &day, &hour, &minute, &second); + year = adjustYear(year); + break; + } + case 10: + { + std::sscanf(seps[0].c_str(), "%2d%2d%2d%2d%2d", &year, &month, &day, &hour, &minute); + year = adjustYear(year); + break; + } + case 9: + { + std::sscanf(seps[0].c_str(), "%2d%2d%2d%2d%1d", &year, &month, &day, &hour, &minute); + year = adjustYear(year); + break; + } + case 8: + { + std::sscanf(seps[0].c_str(), "%4d%2d%2d", &year, &month, &day); + break; + } + case 6: + case 5: + { + std::sscanf(seps[0].c_str(), "%2d%2d%2d", &year, &month, &day); + year = adjustYear(year); + break; + } + default: + { + throw Exception("Wrong datetime format"); + } + } + break; + } + case 3: + { + scanTimeArgs(seps, {&year, &month, &day}); + break; + } + case 6: + { + scanTimeArgs(seps, {&year, &month, &day, &hour, &minute, &second}); + break; + } + default: + { + throw Exception("Wrong datetime format"); + } + } + + UInt32 micro_second = 0; + // TODO This is a simple implement, without processing overflow. + if (frac_str.size() > 6) + { + frac_str = frac_str.substr(0, 6); + } + + if (frac_str.size() > 0) + { + micro_second = std::stoul(frac_str); + for (size_t i = frac_str.size(); i < 6; i++) + micro_second *= 10; + } + + return MyDateTime(year, month, day, hour, minute, second, micro_second, 0).toPackedUInt(); +} + +String MyDateTime::toString() const +{ + String result = dateFormat("%Y-%m-%d %H:%i:%s"); + if (fsp > 0) + { + char buf[16]; + sprintf(buf, ".%06d", micro_second); + result.append(String(buf, fsp + 1)); + } + return result; +} + +} // namespace DB diff --git a/dbms/src/Common/MyTime.h b/dbms/src/Common/MyTime.h new file mode 100644 index 00000000000..9cac825c4ef --- /dev/null +++ b/dbms/src/Common/MyTime.h @@ -0,0 +1,67 @@ +#pragma once + +#include + +namespace DB +{ + +struct MyTimeBase +{ + + enum MyTimeType : UInt8 + { + TypeDate = 0, + TypeDateTime, + TypeTimeStamp, + TypeDuration + }; + + UInt16 year; // year <= 9999 + UInt8 month; // month <= 12 + UInt8 day; // day <= 31 + // When it's type is Time, HH:MM:SS may be 839:59:59 to -839:59:59, so use int16 to avoid overflow + Int16 hour; + UInt8 minute; + UInt8 second; + UInt32 micro_second; // ms second <= 999999 + + UInt8 fsp; + + MyTimeBase() = default; + MyTimeBase(UInt64 packed, UInt8 fsp_ = 0); + MyTimeBase(UInt16 year_, UInt8 month_, UInt8 day_, UInt16 hour_, UInt8 minute_, UInt8 second_, UInt32 micro_second_, UInt8 fsp_); + + UInt64 toPackedUInt() const; + + // DateFormat returns a textual representation of the time value formatted + // according to layout + // See http://dev.mysql.com/doc/refman/5.7/en/date-and-time-functions.html#function_date-format + String dateFormat(const String & layout) const; + +protected: + void convertDateFormat(char c, String & result) const; +}; + +struct MyDateTime : public MyTimeBase +{ + MyDateTime(UInt64 packed, UInt8 fsp) : MyTimeBase(packed, fsp) {} + + MyDateTime(UInt16 year_, UInt8 month_, UInt8 day_, UInt16 hour_, UInt8 minute_, UInt8 second_, UInt32 micro_second_, UInt8 fsp_) + : MyTimeBase(year_, month_, day_, hour_, minute_, second_, micro_second_, fsp_) + {} + + String toString() const; +}; + +struct MyDate : public MyTimeBase +{ + MyDate(UInt64 packed) : MyTimeBase(packed) {} + + MyDate(UInt16 year_, UInt8 month_, UInt8 day_) : MyTimeBase(year_, month_, day_, 0, 0, 0, 0, 0) {} + + String toString() const { return dateFormat("%Y-%m-%d"); } +}; + +Field parseMyDateTime(const String & str); + +} // namespace DB diff --git a/dbms/src/Core/Types.h b/dbms/src/Core/Types.h index 88ebeb078b4..5f5eb918180 100644 --- a/dbms/src/Core/Types.h +++ b/dbms/src/Core/Types.h @@ -133,6 +133,10 @@ enum class TypeIndex Function, AggregateFunction, LowCardinality, + MyDate, + MyDateTime, + MyTimeStamp, + MyTime }; template struct TypeId; diff --git a/dbms/src/DataTypes/DataTypeDate.cpp b/dbms/src/DataTypes/DataTypeDate.cpp index e6c3033e9a2..27b31b0f3db 100644 --- a/dbms/src/DataTypes/DataTypeDate.cpp +++ b/dbms/src/DataTypes/DataTypeDate.cpp @@ -11,14 +11,14 @@ namespace DB void DataTypeDate::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const { - writeDateText(DayNum_t(static_cast(column).getData()[row_num]), ostr); + writeDateText(DayNum_t(static_cast(column).getData()[row_num]), ostr); } static void deserializeText(IColumn & column, ReadBuffer & istr) { DayNum_t x; readDateText(x, istr); - static_cast(column).getData().push_back(x); + static_cast(column).getData().push_back(x); } void DataTypeDate::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const @@ -44,7 +44,7 @@ void DataTypeDate::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) co assertChar('\'', istr); readDateText(x, istr); assertChar('\'', istr); - static_cast(column).getData().push_back(x); /// It's important to do this at the end - for exception safety. + static_cast(column).getData().push_back(x); /// It's important to do this at the end - for exception safety. } void DataTypeDate::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const @@ -60,7 +60,7 @@ void DataTypeDate::deserializeTextJSON(IColumn & column, ReadBuffer & istr) cons assertChar('"', istr); readDateText(x, istr); assertChar('"', istr); - static_cast(column).getData().push_back(x); + static_cast(column).getData().push_back(x); } void DataTypeDate::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const @@ -74,7 +74,7 @@ void DataTypeDate::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const { LocalDate value; readCSV(value, istr); - static_cast(column).getData().push_back(value.getDayNum()); + static_cast(column).getData().push_back(value.getDayNum()); } bool DataTypeDate::equals(const IDataType & rhs) const diff --git a/dbms/src/DataTypes/DataTypeDate.h b/dbms/src/DataTypes/DataTypeDate.h index 5c328e8199a..3da530bcd6d 100644 --- a/dbms/src/DataTypes/DataTypeDate.h +++ b/dbms/src/DataTypes/DataTypeDate.h @@ -6,7 +6,7 @@ namespace DB { -class DataTypeDate final : public DataTypeNumberBase +class DataTypeDate final : public DataTypeNumberBase { public: const char * getFamilyName() const override { return "Date"; } diff --git a/dbms/src/DataTypes/DataTypeDateTime.cpp b/dbms/src/DataTypes/DataTypeDateTime.cpp index 806ad43de38..3be6b61e408 100644 --- a/dbms/src/DataTypes/DataTypeDateTime.cpp +++ b/dbms/src/DataTypes/DataTypeDateTime.cpp @@ -34,7 +34,7 @@ std::string DataTypeDateTime::getName() const void DataTypeDateTime::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const { - writeDateTimeText(static_cast(column).getData()[row_num], ostr, time_zone); + writeDateTimeText(static_cast(column).getData()[row_num], ostr, time_zone); } void DataTypeDateTime::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const @@ -46,7 +46,7 @@ void DataTypeDateTime::deserializeTextEscaped(IColumn & column, ReadBuffer & ist { time_t x; readDateTimeText(x, istr, time_zone); - static_cast(column).getData().push_back(x); + static_cast(column).getData().push_back(x); } void DataTypeDateTime::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const @@ -68,7 +68,7 @@ void DataTypeDateTime::deserializeTextQuoted(IColumn & column, ReadBuffer & istr { readIntText(x, istr); } - static_cast(column).getData().push_back(x); /// It's important to do this at the end - for exception safety. + static_cast(column).getData().push_back(x); /// It's important to do this at the end - for exception safety. } void DataTypeDateTime::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const @@ -90,7 +90,7 @@ void DataTypeDateTime::deserializeTextJSON(IColumn & column, ReadBuffer & istr) { readIntText(x, istr); } - static_cast(column).getData().push_back(x); + static_cast(column).getData().push_back(x); } void DataTypeDateTime::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const @@ -104,7 +104,7 @@ void DataTypeDateTime::deserializeTextCSV(IColumn & column, ReadBuffer & istr, c { time_t x; readDateTimeCSV(x, istr, time_zone); - static_cast(column).getData().push_back(x); + static_cast(column).getData().push_back(x); } bool DataTypeDateTime::equals(const IDataType & rhs) const diff --git a/dbms/src/DataTypes/DataTypeDateTime.h b/dbms/src/DataTypes/DataTypeDateTime.h index f1e3a6c53f2..103472b9e28 100644 --- a/dbms/src/DataTypes/DataTypeDateTime.h +++ b/dbms/src/DataTypes/DataTypeDateTime.h @@ -28,7 +28,7 @@ namespace DB * Server time zone is the time zone specified in 'timezone' parameter in configuration file, * or system time zone at the moment of server startup. */ -class DataTypeDateTime final : public DataTypeNumberBase +class DataTypeDateTime final : public DataTypeNumberBase { public: DataTypeDateTime(const std::string & time_zone_name = ""); diff --git a/dbms/src/DataTypes/DataTypeFactory.cpp b/dbms/src/DataTypes/DataTypeFactory.cpp index 4f6d8b17ce9..da7dbd23250 100644 --- a/dbms/src/DataTypes/DataTypeFactory.cpp +++ b/dbms/src/DataTypes/DataTypeFactory.cpp @@ -107,6 +107,8 @@ void DataTypeFactory::registerSimpleDataType(const String & name, SimpleCreator void registerDataTypeNumbers(DataTypeFactory & factory); void registerDataTypeDate(DataTypeFactory & factory); void registerDataTypeDateTime(DataTypeFactory & factory); +void registerDataTypeMyDateTime(DataTypeFactory & factory); +void registerDataTypeMyDate(DataTypeFactory & factory); void registerDataTypeString(DataTypeFactory & factory); void registerDataTypeFixedString(DataTypeFactory & factory); void registerDataTypeDecimal(DataTypeFactory & factory); @@ -126,6 +128,7 @@ DataTypeFactory::DataTypeFactory() registerDataTypeNumbers(*this); registerDataTypeDate(*this); registerDataTypeDateTime(*this); + registerDataTypeMyDateTime(*this); registerDataTypeString(*this); registerDataTypeFixedString(*this); registerDataTypeDecimal(*this); @@ -138,6 +141,7 @@ DataTypeFactory::DataTypeFactory() registerDataTypeAggregateFunction(*this); registerDataTypeNested(*this); registerDataTypeInterval(*this); + registerDataTypeMyDate(*this); } } diff --git a/dbms/src/DataTypes/DataTypeMyDate.cpp b/dbms/src/DataTypes/DataTypeMyDate.cpp new file mode 100644 index 00000000000..47dad778f74 --- /dev/null +++ b/dbms/src/DataTypes/DataTypeMyDate.cpp @@ -0,0 +1,87 @@ + +#include +#include + +#include +#include +#include + + +namespace DB +{ + +void DataTypeMyDate::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const +{ + writeMyDateText(static_cast(column).getData()[row_num], ostr); +} + +static void deserializeText(IColumn & column, ReadBuffer & istr) +{ + UInt64 x = 0; + readMyDateText(x, istr); + static_cast(column).getData().push_back(x); +} + +void DataTypeMyDate::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const +{ + serializeText(column, row_num, ostr); +} + +void DataTypeMyDate::deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const { deserializeText(column, istr); } + +void DataTypeMyDate::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const +{ + writeChar('\'', ostr); + serializeText(column, row_num, ostr); + writeChar('\'', ostr); +} + +void DataTypeMyDate::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const +{ + UInt64 x = 0; + assertChar('\'', istr); + readMyDateText(x, istr); + assertChar('\'', istr); + static_cast(column).getData().push_back(x); /// It's important to do this at the end - for exception safety. +} + +void DataTypeMyDate::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const +{ + writeChar('"', ostr); + serializeText(column, row_num, ostr); + writeChar('"', ostr); +} + +void DataTypeMyDate::deserializeTextJSON(IColumn & column, ReadBuffer & istr) const +{ + UInt64 x = 0; + assertChar('"', istr); + readMyDateText(x, istr); + assertChar('"', istr); + static_cast(column).getData().push_back(x); +} + +void DataTypeMyDate::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const +{ + writeChar('"', ostr); + serializeText(column, row_num, ostr); + writeChar('"', ostr); +} + +void DataTypeMyDate::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char /*delimiter*/) const +{ + UInt64 value = 0; + readCSV(value, istr); + static_cast(column).getData().push_back(value); +} + +bool DataTypeMyDate::equals(const IDataType & rhs) const { return typeid(rhs) == typeid(*this); } + + +void registerDataTypeMyDate(DataTypeFactory & factory) +{ + factory.registerSimpleDataType( + "MyDate", [] { return DataTypePtr(std::make_shared()); }, DataTypeFactory::CaseInsensitive); +} + +} // namespace DB diff --git a/dbms/src/DataTypes/DataTypeMyDate.h b/dbms/src/DataTypes/DataTypeMyDate.h new file mode 100644 index 00000000000..2a67fce1d77 --- /dev/null +++ b/dbms/src/DataTypes/DataTypeMyDate.h @@ -0,0 +1,33 @@ +#pragma once + +#include + + +namespace DB +{ + +class DataTypeMyDate final : public DataTypeMyTimeBase +{ +public: + const char * getFamilyName() const override { return "MyDate"; } + + TypeIndex getTypeId() const override { return TypeIndex::MyDate; } + + void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; + void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; + void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override; + void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; + void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override; + void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const override; + void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override; + void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; + void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override; + + bool canBeUsedAsVersion() const override { return true; } + bool isDateOrDateTime() const override { return true; } + bool canBeInsideNullable() const override { return true; } + + bool equals(const IDataType & rhs) const override; +}; + +} // namespace DB diff --git a/dbms/src/DataTypes/DataTypeMyDateTime.cpp b/dbms/src/DataTypes/DataTypeMyDateTime.cpp new file mode 100644 index 00000000000..752931dc3fa --- /dev/null +++ b/dbms/src/DataTypes/DataTypeMyDateTime.cpp @@ -0,0 +1,136 @@ +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include + + +namespace DB +{ + +DataTypeMyDateTime::DataTypeMyDateTime(int fraction_) +{ + fraction = fraction_; + if (fraction < 0 || fraction > 6) + throw Exception("fraction must >= 0 and < 6", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); +} + +void DataTypeMyDateTime::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const +{ + writeMyDateTimeText(static_cast(column).getData()[row_num], fraction, ostr); +} + +void DataTypeMyDateTime::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const +{ + serializeText(column, row_num, ostr); +} + +void DataTypeMyDateTime::deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const +{ + UInt64 x; + readMyDateTimeText(x, fraction, istr); + static_cast(column).getData().push_back(x); +} + +void DataTypeMyDateTime::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const +{ + writeChar('\'', ostr); + serializeText(column, row_num, ostr); + writeChar('\'', ostr); +} + +void DataTypeMyDateTime::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const +{ + UInt64 x; + if (checkChar('\'', istr)) /// Cases: '2017-08-31 18:36:48' or '1504193808' + { + readMyDateTimeText(x, fraction, istr); + assertChar('\'', istr); + } + else /// Just 1504193808 or 01504193808 + { + readIntText(x, istr); + } + static_cast(column).getData().push_back(x); /// It's important to do this at the end - for exception safety. +} + +void DataTypeMyDateTime::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const +{ + writeChar('"', ostr); + serializeText(column, row_num, ostr); + writeChar('"', ostr); +} + +void DataTypeMyDateTime::deserializeTextJSON(IColumn & column, ReadBuffer & istr) const +{ + UInt64 x; + if (checkChar('"', istr)) + { + readMyDateTimeText(x, fraction, istr); + assertChar('"', istr); + } + else + { + readIntText(x, istr); + } + static_cast(column).getData().push_back(x); +} + +void DataTypeMyDateTime::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const +{ + writeChar('"', ostr); + serializeText(column, row_num, ostr); + writeChar('"', ostr); +} + +void DataTypeMyDateTime::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char /*delimiter*/) const +{ + UInt64 x; + readMyDateTimeCSV(x, fraction, istr); + static_cast(column).getData().push_back(x); +} + +bool DataTypeMyDateTime::equals(const IDataType & rhs) const +{ + /// DateTime with different timezones are equal, because: + /// "all types with different time zones are equivalent and may be used interchangingly." + return typeid(rhs) == typeid(*this); +} + + +namespace ErrorCodes +{ +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} // namespace ErrorCodes + +static DataTypePtr create(const ASTPtr & arguments) +{ + if (!arguments) + return std::make_shared(0); + + if (arguments->children.size() != 1) + throw Exception( + "MyDateTime data type can optionally have only one argument - fractional", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + const ASTLiteral * arg = typeid_cast(arguments->children[0].get()); + if (!arg || arg->value.getType() != Field::Types::UInt64) + throw Exception("Parameter for MyDateTime data type must be uint literal", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(arg->value.get()); +} + +void registerDataTypeMyDateTime(DataTypeFactory & factory) +{ + factory.registerDataType("MyDateTime", create, DataTypeFactory::CaseInsensitive); +} + +} // namespace DB diff --git a/dbms/src/DataTypes/DataTypeMyDateTime.h b/dbms/src/DataTypes/DataTypeMyDateTime.h new file mode 100644 index 00000000000..70cb4d10150 --- /dev/null +++ b/dbms/src/DataTypes/DataTypeMyDateTime.h @@ -0,0 +1,37 @@ +#pragma once + +#include + + +namespace DB +{ + +class DataTypeMyDateTime final : public DataTypeMyTimeBase +{ + int fraction; + +public: + DataTypeMyDateTime(int fraction_ = 0); + + const char * getFamilyName() const override { return "MyDateTime"; } + + String getName() const override { return "MyDateTime(" + std::to_string(fraction) + ")"; } + + TypeIndex getTypeId() const override { return TypeIndex::MyDateTime; } + + void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; + void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; + void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override; + void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; + void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override; + void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const override; + void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override; + void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; + void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override; + + bool equals(const IDataType & rhs) const override; + + int getFraction() const { return fraction; } +}; + +} // namespace DB diff --git a/dbms/src/DataTypes/DataTypeMyTimeBase.h b/dbms/src/DataTypes/DataTypeMyTimeBase.h new file mode 100644 index 00000000000..51bac428a62 --- /dev/null +++ b/dbms/src/DataTypes/DataTypeMyTimeBase.h @@ -0,0 +1,18 @@ +#pragma once + +#include + + +namespace DB +{ + +// It stands for mysql datetime/ date/ time types. +class DataTypeMyTimeBase : public DataTypeNumberBase +{ +public: + bool canBeUsedAsVersion() const override { return true; } + bool isDateOrDateTime() const override { return true; } + bool canBeInsideNullable() const override { return true; } +}; + +} // namespace DB diff --git a/dbms/src/Debug/MockTiDB.cpp b/dbms/src/Debug/MockTiDB.cpp index 7853fb5ae5e..e7c8056123c 100644 --- a/dbms/src/Debug/MockTiDB.cpp +++ b/dbms/src/Debug/MockTiDB.cpp @@ -1,10 +1,10 @@ #include #include -#include -#include #include #include +#include +#include #include #include #include @@ -177,6 +177,8 @@ ColumnInfo getColumnInfoFromColumn(const NameAndTypePair & column, ColumnID id, else if (checkDataType(nested_type)) column_info.tp = TiDB::TypeLong; + if (auto type = checkAndGetDataType(nested_type)) + column_info.decimal = type->getFraction(); // UInt64 is hijacked by the macro expansion, we check it again. if (checkDataType(nested_type)) column_info.tp = TiDB::TypeLongLong; diff --git a/dbms/src/Debug/dbgTools.cpp b/dbms/src/Debug/dbgTools.cpp index e4a0f2ac5ee..921caa234e7 100644 --- a/dbms/src/Debug/dbgTools.cpp +++ b/dbms/src/Debug/dbgTools.cpp @@ -121,15 +121,23 @@ void addRequestsToRaftCmd(enginepb::CommandRequest * cmd, RegionID region_id, co } } +bool isDateTimeType(TiDB::TP tp) { return tp == TiDB::TypeTimestamp || tp == TiDB::TypeDate || tp == TiDB::TypeDatetime; } + void insert(const TiDB::TableInfo & table_info, RegionID region_id, HandleID handle_id, ASTs::const_iterator begin, ASTs::const_iterator end, Context & context, const std::optional> & tso_del) { std::vector fields; ASTs::const_iterator it; + int idx = 0; while ((it = begin++) != end) { auto field = typeid_cast((*it).get())->value; + if (isDateTimeType(table_info.columns[idx].tp)) + { + field = parseMyDateTime(field.safeGet()); + } fields.emplace_back(field); + idx++; } if (fields.size() != table_info.columns.size()) throw Exception("Number of insert values and columns do not match.", ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Functions/FunctionsConversion.h b/dbms/src/Functions/FunctionsConversion.h index ec3ad00c074..131324411f1 100644 --- a/dbms/src/Functions/FunctionsConversion.h +++ b/dbms/src/Functions/FunctionsConversion.h @@ -241,7 +241,7 @@ struct ToDateTimeImpl }; template struct ConvertImpl - : DateTimeTransformImpl {}; + : DateTimeTransformImpl {}; /// Implementation of toDate function. @@ -260,7 +260,7 @@ struct ToDateTransform32Or64 /** Conversion of DateTime to Date: throw off time component. */ template struct ConvertImpl - : DateTimeTransformImpl {}; + : DateTimeTransformImpl {}; /** Special case of converting (U)Int32 or (U)Int64 (and also, for convenience, Float32, Float64) to Date. * If number is less than 65536, then it is treated as DayNum, and if greater or equals, then as unix timestamp. diff --git a/dbms/src/Functions/FunctionsDateTime.h b/dbms/src/Functions/FunctionsDateTime.h index 777aaf28e96..6644c5aa94f 100644 --- a/dbms/src/Functions/FunctionsDateTime.h +++ b/dbms/src/Functions/FunctionsDateTime.h @@ -45,7 +45,7 @@ namespace ErrorCodes * (toDate - located in FunctionConversion.h file) * * Return types: - * toYear -> UInt32 + * toYear -> UInt16 * toMonth, toDayOfMonth, toDayOfWeek, toHour, toMinute, toSecond -> UInt8 * toMonday, toStartOfMonth, toStartOfYear -> Date * toStartOfMinute, toStartOfHour, toTime, now -> DateTime @@ -72,7 +72,7 @@ const DateLUTImpl & extractTimeZoneFromFunctionArguments(Block & block, const Co #define TIME_SLOT_SIZE 1800 /** Transformations. - * Represents two functions - from datetime (Int64) and from date (UInt32). + * Represents two functions - from datetime (UInt32) and from date (UInt16). * * Also, the "factor transformation" F is defined for the T transformation. * This is a transformation of F such that its value identifies the region of monotonicity for T @@ -87,19 +87,19 @@ const DateLUTImpl & extractTimeZoneFromFunctionArguments(Block & block, const Co /// This factor transformation will say that the function is monotone everywhere. struct ZeroTransform { - static inline UInt32 execute(Int64, const DateLUTImpl &) { return 0; } - static inline UInt32 execute(UInt32, const DateLUTImpl &) { return 0; } + static inline UInt16 execute(UInt32, const DateLUTImpl &) { return 0; } + static inline UInt16 execute(UInt16, const DateLUTImpl &) { return 0; } }; struct ToDateImpl { static constexpr auto name = "toDate"; - static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { - return UInt32(time_zone.toDayNum(t)); + return UInt16(time_zone.toDayNum(t)); } - static inline UInt32 execute(UInt32 d, const DateLUTImpl &) + static inline UInt16 execute(UInt16 d, const DateLUTImpl &) { return d; } @@ -111,11 +111,11 @@ struct ToStartOfDayImpl { static constexpr auto name = "toStartOfDay"; - static inline Int64 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toDate(t); } - static inline Int64 execute(UInt32, const DateLUTImpl &) + static inline UInt32 execute(UInt16, const DateLUTImpl &) { throw Exception("Illegal type Date of argument for function toStartOfDay", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } @@ -127,11 +127,11 @@ struct ToMondayImpl { static constexpr auto name = "toMonday"; - static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t)); } - static inline UInt32 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfWeek(DayNum_t(d)); } @@ -143,11 +143,11 @@ struct ToStartOfMonthImpl { static constexpr auto name = "toStartOfMonth"; - static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfMonth(time_zone.toDayNum(t)); } - static inline UInt32 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfMonth(DayNum_t(d)); } @@ -159,11 +159,11 @@ struct ToStartOfQuarterImpl { static constexpr auto name = "toStartOfQuarter"; - static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfQuarter(time_zone.toDayNum(t)); } - static inline UInt32 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfQuarter(DayNum_t(d)); } @@ -175,11 +175,11 @@ struct ToStartOfYearImpl { static constexpr auto name = "toStartOfYear"; - static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfYear(time_zone.toDayNum(t)); } - static inline UInt32 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfYear(DayNum_t(d)); } @@ -193,12 +193,12 @@ struct ToTimeImpl static constexpr auto name = "toTime"; /// When transforming to time, the date will be equated to 1970-01-02. - static inline Int64 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toTime(t) + 86400; } - static inline Int64 execute(UInt32, const DateLUTImpl &) + static inline UInt32 execute(UInt16, const DateLUTImpl &) { throw Exception("Illegal type Date of argument for function toTime", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } @@ -210,11 +210,11 @@ struct ToStartOfMinuteImpl { static constexpr auto name = "toStartOfMinute"; - static inline Int64 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfMinute(t); } - static inline Int64 execute(UInt32, const DateLUTImpl &) + static inline UInt32 execute(UInt16, const DateLUTImpl &) { throw Exception("Illegal type Date of argument for function toStartOfMinute", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } @@ -226,11 +226,11 @@ struct ToStartOfFiveMinuteImpl { static constexpr auto name = "toStartOfFiveMinute"; - static inline Int64 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfFiveMinute(t); } - static inline Int64 execute(UInt32, const DateLUTImpl &) + static inline UInt32 execute(UInt16, const DateLUTImpl &) { throw Exception("Illegal type Date of argument for function toStartOfFiveMinute", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } @@ -242,11 +242,11 @@ struct ToStartOfFifteenMinutesImpl { static constexpr auto name = "toStartOfFifteenMinutes"; - static inline Int64 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfFifteenMinutes(t); } - static inline Int64 execute(UInt32, const DateLUTImpl &) + static inline UInt32 execute(UInt16, const DateLUTImpl &) { throw Exception("Illegal type Date of argument for function toStartOfFifteenMinutes", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } @@ -258,12 +258,12 @@ struct ToStartOfHourImpl { static constexpr auto name = "toStartOfHour"; - static inline Int64 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfHour(t); } - static inline Int64 execute(UInt32, const DateLUTImpl &) + static inline UInt32 execute(UInt16, const DateLUTImpl &) { throw Exception("Illegal type Date of argument for function toStartOfHour", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } @@ -275,11 +275,11 @@ struct ToYearImpl { static constexpr auto name = "toYear"; - static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toYear(t); } - static inline UInt32 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toYear(DayNum_t(d)); } @@ -291,11 +291,11 @@ struct ToQuarterImpl { static constexpr auto name = "toQuarter"; - static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toQuarter(t); } - static inline UInt8 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toQuarter(DayNum_t(d)); } @@ -307,11 +307,11 @@ struct ToMonthImpl { static constexpr auto name = "toMonth"; - static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toMonth(t); } - static inline UInt8 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toMonth(DayNum_t(d)); } @@ -323,11 +323,11 @@ struct ToDayOfMonthImpl { static constexpr auto name = "toDayOfMonth"; - static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toDayOfMonth(t); } - static inline UInt8 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toDayOfMonth(DayNum_t(d)); } @@ -339,11 +339,11 @@ struct ToDayOfWeekImpl { static constexpr auto name = "toDayOfWeek"; - static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toDayOfWeek(t); } - static inline UInt8 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toDayOfWeek(DayNum_t(d)); } @@ -355,12 +355,12 @@ struct ToHourImpl { static constexpr auto name = "toHour"; - static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toHour(t); } - static inline UInt8 execute(UInt32, const DateLUTImpl &) + static inline UInt8 execute(UInt16, const DateLUTImpl &) { throw Exception("Illegal type Date of argument for function toHour", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } @@ -372,11 +372,11 @@ struct ToMinuteImpl { static constexpr auto name = "toMinute"; - static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toMinute(t); } - static inline UInt8 execute(UInt32, const DateLUTImpl &) + static inline UInt8 execute(UInt16, const DateLUTImpl &) { throw Exception("Illegal type Date of argument for function toMinute", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } @@ -388,11 +388,11 @@ struct ToSecondImpl { static constexpr auto name = "toSecond"; - static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toSecond(t); } - static inline UInt8 execute(UInt32, const DateLUTImpl &) + static inline UInt8 execute(UInt16, const DateLUTImpl &) { throw Exception("Illegal type Date of argument for function toSecond", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } @@ -404,11 +404,11 @@ struct ToRelativeYearNumImpl { static constexpr auto name = "toRelativeYearNum"; - static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toYear(t); } - static inline UInt32 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toYear(DayNum_t(d)); } @@ -420,11 +420,11 @@ struct ToRelativeQuarterNumImpl { static constexpr auto name = "toRelativeQuarterNum"; - static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeQuarterNum(t); } - static inline UInt32 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toRelativeQuarterNum(DayNum_t(d)); } @@ -436,11 +436,11 @@ struct ToRelativeMonthNumImpl { static constexpr auto name = "toRelativeMonthNum"; - static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeMonthNum(t); } - static inline UInt32 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toRelativeMonthNum(DayNum_t(d)); } @@ -452,11 +452,11 @@ struct ToRelativeWeekNumImpl { static constexpr auto name = "toRelativeWeekNum"; - static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeWeekNum(t); } - static inline UInt32 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toRelativeWeekNum(DayNum_t(d)); } @@ -468,11 +468,11 @@ struct ToRelativeDayNumImpl { static constexpr auto name = "toRelativeDayNum"; - static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toDayNum(t); } - static inline UInt32 execute(UInt32 d, const DateLUTImpl &) + static inline UInt16 execute(UInt16 d, const DateLUTImpl &) { return static_cast(d); } @@ -485,11 +485,11 @@ struct ToRelativeHourNumImpl { static constexpr auto name = "toRelativeHourNum"; - static inline Int64 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeHourNum(t); } - static inline Int64 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toRelativeHourNum(DayNum_t(d)); } @@ -501,11 +501,11 @@ struct ToRelativeMinuteNumImpl { static constexpr auto name = "toRelativeMinuteNum"; - static inline Int64 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeMinuteNum(t); } - static inline Int64 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toRelativeMinuteNum(DayNum_t(d)); } @@ -517,11 +517,11 @@ struct ToRelativeSecondNumImpl { static constexpr auto name = "toRelativeSecondNum"; - static inline Int64 execute(Int64 t, const DateLUTImpl &) + static inline UInt32 execute(UInt32 t, const DateLUTImpl &) { return t; } - static inline Int64 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.fromDayNum(DayNum_t(d)); } @@ -533,11 +533,11 @@ struct ToYYYYMMImpl { static constexpr auto name = "toYYYYMM"; - static inline Int64 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toNumYYYYMM(t); } - static inline Int64 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toNumYYYYMM(static_cast(d)); } @@ -549,11 +549,11 @@ struct ToYYYYMMDDImpl { static constexpr auto name = "toYYYYMMDD"; - static inline Int64 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toNumYYYYMMDD(t); } - static inline Int64 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toNumYYYYMMDD(static_cast(d)); } @@ -565,11 +565,11 @@ struct ToYYYYMMDDhhmmssImpl { static constexpr auto name = "toYYYYMMDDhhmmss"; - static inline Int64 execute(Int64 t, const DateLUTImpl & time_zone) + static inline UInt64 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toNumYYYYMMDDhhmmss(t); } - static inline Int64 execute(UInt32 d, const DateLUTImpl & time_zone) + static inline UInt64 execute(UInt16 d, const DateLUTImpl & time_zone) { return time_zone.toNumYYYYMMDDhhmmss(time_zone.toDate(static_cast(d))); } @@ -708,14 +708,14 @@ class FunctionDateOrDateTimeToSomething : public IFunction if (checkAndGetDataType(&type)) { - return Transform::FactorTransform::execute(UInt32(left.get()), date_lut) - == Transform::FactorTransform::execute(UInt32(right.get()), date_lut) + return Transform::FactorTransform::execute(UInt16(left.get()), date_lut) + == Transform::FactorTransform::execute(UInt16(right.get()), date_lut) ? is_monotonic : is_not_monotonic; } else { - return Transform::FactorTransform::execute(Int64(left.get()), date_lut) - == Transform::FactorTransform::execute(Int64(right.get()), date_lut) + return Transform::FactorTransform::execute(UInt32(left.get()), date_lut) + == Transform::FactorTransform::execute(UInt32(right.get()), date_lut) ? is_monotonic : is_not_monotonic; } } @@ -726,12 +726,12 @@ struct AddSecondsImpl { static constexpr auto name = "addSeconds"; - static inline Int64 execute(Int64 t, Int64 delta, const DateLUTImpl &) + static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &) { return t + delta; } - static inline Int64 execute(UInt32 d, Int64 delta, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.fromDayNum(DayNum_t(d)) + delta; } @@ -741,12 +741,12 @@ struct AddMinutesImpl { static constexpr auto name = "addMinutes"; - static inline Int64 execute(Int64 t, Int64 delta, const DateLUTImpl &) + static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &) { return t + delta * 60; } - static inline Int64 execute(UInt32 d, Int64 delta, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.fromDayNum(DayNum_t(d)) + delta * 60; } @@ -756,12 +756,12 @@ struct AddHoursImpl { static constexpr auto name = "addHours"; - static inline Int64 execute(Int64 t, Int64 delta, const DateLUTImpl &) + static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &) { return t + delta * 3600; } - static inline Int64 execute(UInt32 d, Int64 delta, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.fromDayNum(DayNum_t(d)) + delta * 3600; } @@ -771,12 +771,12 @@ struct AddDaysImpl { static constexpr auto name = "addDays"; - static inline Int64 execute(Int64 t, Int64 delta, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addDays(t, delta); } - static inline UInt32 execute(UInt32 d, Int64 delta, const DateLUTImpl &) + static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl &) { return d + delta; } @@ -786,12 +786,12 @@ struct AddWeeksImpl { static constexpr auto name = "addWeeks"; - static inline Int64 execute(Int64 t, Int64 delta, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addWeeks(t, delta); } - static inline UInt32 execute(UInt32 d, Int64 delta, const DateLUTImpl &) + static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl &) { return d + delta * 7; } @@ -801,12 +801,12 @@ struct AddMonthsImpl { static constexpr auto name = "addMonths"; - static inline Int64 execute(Int64 t, Int64 delta, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addMonths(t, delta); } - static inline UInt32 execute(UInt32 d, Int64 delta, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addMonths(DayNum_t(d), delta); } @@ -816,12 +816,12 @@ struct AddYearsImpl { static constexpr auto name = "addYears"; - static inline Int64 execute(Int64 t, Int64 delta, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addYears(t, delta); } - static inline UInt32 execute(UInt32 d, Int64 delta, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addYears(DayNum_t(d), delta); } @@ -831,12 +831,12 @@ struct AddYearsImpl template struct SubtractIntervalImpl { - static inline Int64 execute(Int64 t, Int64 delta, const DateLUTImpl & time_zone) + static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return Transform::execute(t, -delta, time_zone); } - static inline UInt32 execute(UInt32 d, Int64 delta, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { return Transform::execute(d, -delta, time_zone); } @@ -972,14 +972,14 @@ class FunctionDateOrDateTimeAddInterval : public IFunction if (checkDataType(arguments[0].type.get())) { - if (std::is_same_v())), UInt32>) + if (std::is_same_v())), UInt16>) return std::make_shared(); else return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 2, 0)); } else { - if (std::is_same_v())), UInt32>) + if (std::is_same_v())), UInt16>) return std::make_shared(); else return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 2, 0)); @@ -1102,14 +1102,14 @@ class FunctionDateDiff : public IFunction const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y, ColumnInt64::Container & result) { - if (auto * x_vec = checkAndGetColumn(&x)) + if (auto * x_vec = checkAndGetColumn(&x)) dispatchForSecondColumn(*x_vec, y, timezone_x, timezone_y, result); - else if (auto * x_vec = checkAndGetColumn(&x)) + else if (auto * x_vec = checkAndGetColumn(&x)) dispatchForSecondColumn(*x_vec, y, timezone_x, timezone_y, result); + else if (auto * x_const = checkAndGetColumnConst(&x)) + dispatchConstForSecondColumn(x_const->getValue(), y, timezone_x, timezone_y, result); else if (auto * x_const = checkAndGetColumnConst(&x)) dispatchConstForSecondColumn(x_const->getValue(), y, timezone_x, timezone_y, result); - else if (auto * x_const = checkAndGetColumnConst(&x)) - dispatchConstForSecondColumn(x_const->getValue(), y, timezone_x, timezone_y, result); else throw Exception("Illegal column for first argument of function " + getName() + ", must be Date or DateTime", ErrorCodes::ILLEGAL_COLUMN); } @@ -1120,14 +1120,14 @@ class FunctionDateDiff : public IFunction const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y, ColumnInt64::Container & result) { - if (auto * y_vec = checkAndGetColumn(&y)) + if (auto * y_vec = checkAndGetColumn(&y)) vector_vector(x, *y_vec, timezone_x, timezone_y, result); - else if (auto * y_vec = checkAndGetColumn(&y)) + else if (auto * y_vec = checkAndGetColumn(&y)) vector_vector(x, *y_vec, timezone_x, timezone_y, result); + else if (auto * y_const = checkAndGetColumnConst(&y)) + vector_constant(x, y_const->getValue(), timezone_x, timezone_y, result); else if (auto * y_const = checkAndGetColumnConst(&y)) vector_constant(x, y_const->getValue(), timezone_x, timezone_y, result); - else if (auto * y_const = checkAndGetColumnConst(&y)) - vector_constant(x, y_const->getValue(), timezone_x, timezone_y, result); else throw Exception("Illegal column for second argument of function " + getName() + ", must be Date or DateTime", ErrorCodes::ILLEGAL_COLUMN); } @@ -1138,9 +1138,9 @@ class FunctionDateDiff : public IFunction const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y, ColumnInt64::Container & result) { - if (auto * y_vec = checkAndGetColumn(&y)) + if (auto * y_vec = checkAndGetColumn(&y)) constant_vector(x, *y_vec, timezone_x, timezone_y, result); - else if (auto * y_vec = checkAndGetColumn(&y)) + else if (auto * y_vec = checkAndGetColumn(&y)) constant_vector(x, *y_vec, timezone_x, timezone_y, result); else throw Exception("Illegal column for second argument of function " + getName() + ", must be Date or DateTime", ErrorCodes::ILLEGAL_COLUMN); @@ -1212,9 +1212,9 @@ class FunctionNow : public IFunction void executeImpl(Block & block, const ColumnNumbers & /*arguments*/, size_t result) override { - block.getByPosition(result).column = DataTypeInt64().createColumnConst( + block.getByPosition(result).column = DataTypeUInt32().createColumnConst( block.rows(), - static_cast(time(nullptr))); + static_cast(time(nullptr))); } }; @@ -1241,9 +1241,9 @@ class FunctionToday : public IFunction void executeImpl(Block & block, const ColumnNumbers & /*arguments*/, size_t result) override { - block.getByPosition(result).column = DataTypeUInt32().createColumnConst( + block.getByPosition(result).column = DataTypeUInt16().createColumnConst( block.rows(), - Int64(DateLUT::instance().toDayNum(time(nullptr)))); + UInt64(DateLUT::instance().toDayNum(time(nullptr)))); } }; @@ -1270,9 +1270,9 @@ class FunctionYesterday : public IFunction void executeImpl(Block & block, const ColumnNumbers & /*arguments*/, size_t result) override { - block.getByPosition(result).column = DataTypeUInt32().createColumnConst( + block.getByPosition(result).column = DataTypeUInt16().createColumnConst( block.rows(), - Int64(DateLUT::instance().toDayNum(time(nullptr)) - 1)); + UInt64(DateLUT::instance().toDayNum(time(nullptr)) - 1)); } }; @@ -1340,11 +1340,11 @@ class FunctionTimeSlot : public IFunction void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) override { - if (const ColumnInt64 * times = typeid_cast(block.getByPosition(arguments[0]).column.get())) + if (const ColumnUInt32 * times = typeid_cast(block.getByPosition(arguments[0]).column.get())) { - auto res = ColumnInt64::create(); - ColumnInt64::Container & res_vec = res->getData(); - const ColumnInt64::Container & vec = times->getData(); + auto res = ColumnUInt32::create(); + ColumnUInt32::Container & res_vec = res->getData(); + const ColumnUInt32::Container & vec = times->getData(); size_t size = vec.size(); res_vec.resize(size); @@ -1366,8 +1366,8 @@ template struct TimeSlotsImpl { static void vector_vector( - const PaddedPODArray & starts, const PaddedPODArray & durations, - PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) + const PaddedPODArray & starts, const PaddedPODArray & durations, + PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) { size_t size = starts.size(); @@ -1377,7 +1377,7 @@ struct TimeSlotsImpl ColumnArray::Offset current_offset = 0; for (size_t i = 0; i < size; ++i) { - for (Int64 value = starts[i] / TIME_SLOT_SIZE; value <= (starts[i] + durations[i]) / TIME_SLOT_SIZE; ++value) + for (UInt32 value = starts[i] / TIME_SLOT_SIZE; value <= (starts[i] + durations[i]) / TIME_SLOT_SIZE; ++value) { result_values.push_back(value * TIME_SLOT_SIZE); ++current_offset; @@ -1388,8 +1388,8 @@ struct TimeSlotsImpl } static void vector_constant( - const PaddedPODArray & starts, DurationType duration, - PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) + const PaddedPODArray & starts, DurationType duration, + PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) { size_t size = starts.size(); @@ -1399,7 +1399,7 @@ struct TimeSlotsImpl ColumnArray::Offset current_offset = 0; for (size_t i = 0; i < size; ++i) { - for (Int64 value = starts[i] / TIME_SLOT_SIZE; value <= (starts[i] + duration) / TIME_SLOT_SIZE; ++value) + for (UInt32 value = starts[i] / TIME_SLOT_SIZE; value <= (starts[i] + duration) / TIME_SLOT_SIZE; ++value) { result_values.push_back(value * TIME_SLOT_SIZE); ++current_offset; @@ -1410,8 +1410,8 @@ struct TimeSlotsImpl } static void constant_vector( - Int64 start, const PaddedPODArray & durations, - PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) + UInt32 start, const PaddedPODArray & durations, + PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) { size_t size = durations.size(); @@ -1421,7 +1421,7 @@ struct TimeSlotsImpl ColumnArray::Offset current_offset = 0; for (size_t i = 0; i < size; ++i) { - for (Int64 value = start / TIME_SLOT_SIZE; value <= (start + durations[i]) / TIME_SLOT_SIZE; ++value) + for (UInt32 value = start / TIME_SLOT_SIZE; value <= (start + durations[i]) / TIME_SLOT_SIZE; ++value) { result_values.push_back(value * TIME_SLOT_SIZE); ++current_offset; @@ -1432,11 +1432,11 @@ struct TimeSlotsImpl } static void constant_constant( - Int64 start, DurationType duration, + UInt32 start, DurationType duration, Array & result) { - for (Int64 value = start / TIME_SLOT_SIZE; value <= (start + duration) / TIME_SLOT_SIZE; ++value) - result.push_back(static_cast(value * TIME_SLOT_SIZE)); + for (UInt32 value = start / TIME_SLOT_SIZE; value <= (start + duration) / TIME_SLOT_SIZE; ++value) + result.push_back(static_cast(value * TIME_SLOT_SIZE)); } }; @@ -1460,8 +1460,8 @@ class FunctionTimeSlots : public IFunction throw Exception("Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() + ". Must be DateTime.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (!checkDataType(arguments[1].get())) - throw Exception("Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() + ". Must be Int64.", + if (!checkDataType(arguments[1].get())) + throw Exception("Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() + ". Must be UInt32.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return std::make_shared(std::make_shared()); @@ -1469,34 +1469,34 @@ class FunctionTimeSlots : public IFunction void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) override { - auto starts = checkAndGetColumn(block.getByPosition(arguments[0]).column.get()); - auto const_starts = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); + auto starts = checkAndGetColumn(block.getByPosition(arguments[0]).column.get()); + auto const_starts = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - auto durations = checkAndGetColumn(block.getByPosition(arguments[1]).column.get()); - auto const_durations = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); + auto durations = checkAndGetColumn(block.getByPosition(arguments[1]).column.get()); + auto const_durations = checkAndGetColumnConst(block.getByPosition(arguments[1]).column.get()); - auto res = ColumnArray::create(ColumnInt64::create()); - ColumnInt64::Container & res_values = typeid_cast(res->getData()).getData(); + auto res = ColumnArray::create(ColumnUInt32::create()); + ColumnUInt32::Container & res_values = typeid_cast(res->getData()).getData(); if (starts && durations) { - TimeSlotsImpl::vector_vector(starts->getData(), durations->getData(), res_values, res->getOffsets()); + TimeSlotsImpl::vector_vector(starts->getData(), durations->getData(), res_values, res->getOffsets()); block.getByPosition(result).column = std::move(res); } else if (starts && const_durations) { - TimeSlotsImpl::vector_constant(starts->getData(), const_durations->getValue(), res_values, res->getOffsets()); + TimeSlotsImpl::vector_constant(starts->getData(), const_durations->getValue(), res_values, res->getOffsets()); block.getByPosition(result).column = std::move(res); } else if (const_starts && durations) { - TimeSlotsImpl::constant_vector(const_starts->getValue(), durations->getData(), res_values, res->getOffsets()); + TimeSlotsImpl::constant_vector(const_starts->getValue(), durations->getData(), res_values, res->getOffsets()); block.getByPosition(result).column = std::move(res); } else if (const_starts && const_durations) { Array const_res; - TimeSlotsImpl::constant_constant(const_starts->getValue(), const_durations->getValue(), const_res); + TimeSlotsImpl::constant_constant(const_starts->getValue(), const_durations->getValue(), const_res); block.getByPosition(result).column = block.getByPosition(result).type->createColumnConst(block.rows(), const_res); } else @@ -1508,7 +1508,7 @@ class FunctionTimeSlots : public IFunction }; -using FunctionToYear = FunctionDateOrDateTimeToSomething; +using FunctionToYear = FunctionDateOrDateTimeToSomething; using FunctionToQuarter = FunctionDateOrDateTimeToSomething; using FunctionToMonth = FunctionDateOrDateTimeToSomething; using FunctionToDayOfMonth = FunctionDateOrDateTimeToSomething; @@ -1527,18 +1527,18 @@ using FunctionToStartOfFifteenMinutes = FunctionDateOrDateTimeToSomething; using FunctionToTime = FunctionDateOrDateTimeToSomething; -using FunctionToRelativeYearNum = FunctionDateOrDateTimeToSomething; -using FunctionToRelativeQuarterNum = FunctionDateOrDateTimeToSomething; -using FunctionToRelativeMonthNum = FunctionDateOrDateTimeToSomething; -using FunctionToRelativeWeekNum = FunctionDateOrDateTimeToSomething; -using FunctionToRelativeDayNum = FunctionDateOrDateTimeToSomething; -using FunctionToRelativeHourNum = FunctionDateOrDateTimeToSomething; -using FunctionToRelativeMinuteNum = FunctionDateOrDateTimeToSomething; -using FunctionToRelativeSecondNum = FunctionDateOrDateTimeToSomething; - -using FunctionToYYYYMM = FunctionDateOrDateTimeToSomething; -using FunctionToYYYYMMDD = FunctionDateOrDateTimeToSomething; -using FunctionToYYYYMMDDhhmmss = FunctionDateOrDateTimeToSomething; +using FunctionToRelativeYearNum = FunctionDateOrDateTimeToSomething; +using FunctionToRelativeQuarterNum = FunctionDateOrDateTimeToSomething; +using FunctionToRelativeMonthNum = FunctionDateOrDateTimeToSomething; +using FunctionToRelativeWeekNum = FunctionDateOrDateTimeToSomething; +using FunctionToRelativeDayNum = FunctionDateOrDateTimeToSomething; +using FunctionToRelativeHourNum = FunctionDateOrDateTimeToSomething; +using FunctionToRelativeMinuteNum = FunctionDateOrDateTimeToSomething; +using FunctionToRelativeSecondNum = FunctionDateOrDateTimeToSomething; + +using FunctionToYYYYMM = FunctionDateOrDateTimeToSomething; +using FunctionToYYYYMMDD = FunctionDateOrDateTimeToSomething; +using FunctionToYYYYMMDDhhmmss = FunctionDateOrDateTimeToSomething; using FunctionAddSeconds = FunctionDateOrDateTimeAddInterval; using FunctionAddMinutes = FunctionDateOrDateTimeAddInterval; diff --git a/dbms/src/Functions/FunctionsExternalDictionaries.h b/dbms/src/Functions/FunctionsExternalDictionaries.h index ee59a71d2e6..bbf9b1b5c60 100644 --- a/dbms/src/Functions/FunctionsExternalDictionaries.h +++ b/dbms/src/Functions/FunctionsExternalDictionaries.h @@ -694,8 +694,8 @@ DECLARE_DICT_GET_TRAITS(Int32, DataTypeInt32) DECLARE_DICT_GET_TRAITS(Int64, DataTypeInt64) DECLARE_DICT_GET_TRAITS(Float32, DataTypeFloat32) DECLARE_DICT_GET_TRAITS(Float64, DataTypeFloat64) -DECLARE_DICT_GET_TRAITS(UInt32, DataTypeDate) -DECLARE_DICT_GET_TRAITS(Int64, DataTypeDateTime) +DECLARE_DICT_GET_TRAITS(UInt16, DataTypeDate) +DECLARE_DICT_GET_TRAITS(UInt32, DataTypeDateTime) DECLARE_DICT_GET_TRAITS(UInt128, DataTypeUUID) #undef DECLARE_DICT_GET_TRAITS diff --git a/dbms/src/IO/ReadHelpers.h b/dbms/src/IO/ReadHelpers.h index 7160d64fce1..02a830201f5 100644 --- a/dbms/src/IO/ReadHelpers.h +++ b/dbms/src/IO/ReadHelpers.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -539,6 +540,38 @@ void parseUUID(const UInt8 * src36, std::reverse_iterator dst16); template void formatHex(IteratorSrc src, IteratorDst dst, const size_t num_bytes); +inline void readMyDateText(UInt64 & date, ReadBuffer & buf) +{ + /// Optimistic path, when whole value is in buffer. + if (buf.position() + 10 <= buf.buffer().end()) + { + UInt16 year = (buf.position()[0] - '0') * 1000 + (buf.position()[1] - '0') * 100 + (buf.position()[2] - '0') * 10 + (buf.position()[3] - '0'); + buf.position() += 5; + + UInt8 month = buf.position()[0] - '0'; + if (isNumericASCII(buf.position()[1])) + { + month = month * 10 + buf.position()[1] - '0'; + buf.position() += 3; + } + else + buf.position() += 2; + + UInt8 day = buf.position()[0] - '0'; + if (isNumericASCII(buf.position()[1])) + { + day = day * 10 + buf.position()[1] - '0'; + buf.position() += 2; + } + else + buf.position() += 1; + + date = MyDate(year, month, day).toPackedUInt(); + } + + throw Exception("wrong date format.", ErrorCodes::CANNOT_PARSE_DATE); +} + void readDateTextFallback(LocalDate & date, ReadBuffer & buf); @@ -606,6 +639,59 @@ inline T parse(const char * data, size_t size); void readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut); +inline void readMyDateTimeText(UInt64 & packed, int fsp, ReadBuffer & buf) +{ + + const char * s = buf.position(); + if (s + 19 <= buf.buffer().end()) + { + if (s[4] < '0' || s[4] > '9') + { + UInt16 year = (s[0] - '0') * 1000 + (s[1] - '0') * 100 + (s[2] - '0') * 10 + (s[3] - '0'); + UInt8 month = (s[5] - '0') * 10 + (s[6] - '0'); + UInt8 day = (s[8] - '0') * 10 + (s[9] - '0'); + + UInt8 hour = (s[11] - '0') * 10 + (s[12] - '0'); + UInt8 minute = (s[14] - '0') * 10 + (s[15] - '0'); + UInt8 second = (s[17] - '0') * 10 + (s[18] - '0'); + + UInt32 micro_second = 0; + bool fractional = false; + int digit = 0; + buf.position() += 19; + while(buf.position() <= buf.buffer().end()) + { + char x = *buf.position(); + if (x == '.') + { + fractional = true; + } + else if(!fractional) + { + break; + } + else if (x <= '9' && x >= '0') + { + if (digit < fsp) + { + micro_second = micro_second * 10 + x; + digit ++; + } + } + else + { + break; + } + buf.position() ++; + } + + packed = MyDateTime(year, month, day, hour, minute, second, micro_second, fsp).toPackedUInt(); + } + } + + throw Exception("wrong datetime format.", ErrorCodes::CANNOT_PARSE_DATETIME); +} + /** In YYYY-MM-DD hh:mm:ss format, according to specified time zone. * As an exception, also supported parsing of unix timestamp in form of decimal number. */ @@ -787,6 +873,21 @@ inline void readCSVDecimal(Decimal & x, ReadBuffer & buf, PrecType precision, assertChar(maybe_quote, buf); } +inline void readMyDateTimeCSV(UInt64 & datetime, int fsp, ReadBuffer & buf) +{ + if (buf.eof()) + throwReadAfterEOF(); + + char maybe_quote = *buf.position(); + + if (maybe_quote == '\'' || maybe_quote == '\"') + ++buf.position(); + + readMyDateTimeText(datetime, fsp, buf); + + if (maybe_quote == '\'' || maybe_quote == '\"') + assertChar(maybe_quote, buf); +} inline void readDateTimeCSV(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut) { diff --git a/dbms/src/IO/WriteHelpers.h b/dbms/src/IO/WriteHelpers.h index f95ea9dedeb..29351fc7215 100644 --- a/dbms/src/IO/WriteHelpers.h +++ b/dbms/src/IO/WriteHelpers.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -495,6 +496,12 @@ inline void writeUUIDText(const UUID & uuid, WriteBuffer & buf) buf.write(s, sizeof(s)); } +inline void writeMyDateText(UInt64 date, WriteBuffer & buf) +{ + const String & tmp = MyDate(date).toString(); + buf.write(tmp.c_str(), tmp.size()); +} + /// in YYYY-MM-DD format template inline void writeDateText(const LocalDate & date, WriteBuffer & buf) @@ -617,6 +624,12 @@ inline void writeDateTimeText(const LocalDateTime & datetime, WriteBuffer & buf) } } +inline void writeMyDateTimeText(UInt64 packed, int fsp, WriteBuffer & buf) +{ + const String & str = MyDateTime(packed, fsp).toString(); + buf.write(str.c_str(), str.size()); +} + /// In the format YYYY-MM-DD HH:MM:SS, according to the specified time zone. template inline void writeDateTimeText(time_t datetime, WriteBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) diff --git a/dbms/src/Interpreters/convertFieldToType.cpp b/dbms/src/Interpreters/convertFieldToType.cpp index 0b9d9012460..08ca468eb71 100644 --- a/dbms/src/Interpreters/convertFieldToType.cpp +++ b/dbms/src/Interpreters/convertFieldToType.cpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include #include @@ -15,6 +17,7 @@ #include #include +#include #include #include #include @@ -224,12 +227,15 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type) return convertDecimalType(src, *ptype); const bool is_date = typeid_cast(&type); + const bool is_my_date = typeid_cast(&type); bool is_datetime = false; + bool is_my_datetime = false; bool is_enum = false; bool is_uuid = false; - if (!is_date) + if (!is_date && !is_my_date) if (!(is_datetime = typeid_cast(&type))) + if (!(is_my_datetime = typeid_cast(&type))) if (!(is_uuid = typeid_cast(&type))) if (!(is_enum = dynamic_cast(&type))) throw Exception{"Logical error: unknown numeric type " + type.getName(), ErrorCodes::LOGICAL_ERROR}; @@ -247,6 +253,11 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type) /// Convert 'YYYY-MM-DD' Strings to Date return UInt64(stringToDate(src.get())); } + else if (is_my_date || is_my_datetime) + { + /// Convert 'YYYY-MM-DD' Strings to Date + return (parseMyDateTime(src.get())); + } else if (is_datetime) { /// Convert 'YYYY-MM-DD hh:mm:ss' Strings to DateTime diff --git a/dbms/src/Storages/Transaction/Datum.cpp b/dbms/src/Storages/Transaction/Datum.cpp index 3e62648dc6a..08acb1fd63a 100644 --- a/dbms/src/Storages/Transaction/Datum.cpp +++ b/dbms/src/Storages/Transaction/Datum.cpp @@ -22,84 +22,6 @@ struct DatumOp static bool overflow(const Field &, const ColumnInfo &) { return false; } }; -/// Specialized for Date/Datetime/Timestamp, actual does unflatten/flatten. -template -struct DatumOp::type> -{ - static void unflatten(const Field & orig, std::optional & copy) - { - if (orig.isNull()) - return; - - UInt64 packed = orig.get(); - UInt64 ymdhms = packed >> 24; - UInt64 ymd = ymdhms >> 17; - UInt8 day = UInt8(ymd & ((1 << 5) - 1)); - UInt64 ym = ymd >> 5; - UInt8 month = UInt8(ym % 13); - UInt16 year = UInt16(ym / 13); - - const auto & date_lut = DateLUT::instance(); - - if constexpr (tp == TypeDate) - { - auto date = date_lut.makeDayNum(year, month, day); - copy = static_cast(date); - } - else - { - time_t date_time; - // TODO: Temporary hack invalid date time to zero. - if (unlikely(year == 0)) - date_time = 0; - else - { - if (unlikely(month == 0 || day == 0)) - { - throw Exception( - "wrong datetime format: " + std::to_string(year) + " " + std::to_string(month) + " " + std::to_string(day) + ".", - DB::ErrorCodes::LOGICAL_ERROR); - } - UInt64 hms = ymdhms & ((1 << 17) - 1); - UInt8 second = UInt8(hms & ((1 << 6) - 1)); - UInt8 minute = UInt8((hms >> 6) & ((1 << 6) - 1)); - UInt8 hour = UInt8(hms >> 12); - - date_time = date_lut.makeDateTime(year, month, day, hour, minute, second); - } - copy = static_cast(date_time); - } - } - - static void flatten(const Field & orig, std::optional & copy) - { - if (orig.isNull()) - return; - - DateLUTImpl::Values values; - UInt8 hour = 0, minute = 0, second = 0; - const auto & date_lut = DateLUT::instance(); - if constexpr (tp == TypeDate) - { - DayNum_t day_num(static_cast(orig.get())); - values = date_lut.getValues(day_num); - } - else - { - time_t date_time(static_cast(orig.get())); - values = date_lut.getValues(date_time); - hour = date_lut.toHour(date_time); - minute = date_lut.toMinute(date_time); - second = date_lut.toSecond(date_time); - } - UInt64 ymd = ((UInt64)values.year * 13 + values.month) << 5 | values.day_of_month; - UInt64 hms = (UInt64)hour << 12 | minute << 6 | second; - copy = (ymd << 17 | hms) << 24; - } - - static bool overflow(const Field &, const ColumnInfo &) { return false; } -}; - /// Specialized for integer types less than 64b, checks overflow. template struct DatumOp::type> diff --git a/dbms/src/Storages/Transaction/MyTimeParser.h b/dbms/src/Storages/Transaction/MyTimeParser.h deleted file mode 100644 index 009bba29390..00000000000 --- a/dbms/src/Storages/Transaction/MyTimeParser.h +++ /dev/null @@ -1,174 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace DB -{ - -int adjustYear(int year) -{ - if (year >= 0 && year <= 69) - return 2000 + year; - if (year >= 70 && year <= 99) - return 1900 + year; - return year; -} - -void scanTimeArgs(const std::vector & seps, std::initializer_list && list) -{ - int i = 0; - for (auto * ptr : list) - { - *ptr = std::stoi(seps[i]); - i++; - } -} - -int getFracIndex(const String & format) -{ - int idx = -1; - for (int i = int(format.size()) - 1; i >= 0; i--) - { - if (std::ispunct(format[i])) - { - if (format[i] == '.') - { - idx = i; - } - break; - } - } - return idx; -} - -std::vector parseDateFormat(String format) -{ - format = Poco::trimInPlace(format); - - std::vector seps; - size_t start = 0; - for (size_t i = 0; i < format.size(); i++) - { - if (i == 0 || i + 1 == format.size()) - { - if (!std::isdigit(format[i])) - return {}; - continue; - } - - if (!std::isdigit(format[i])) - { - if (!std::isdigit(format[i - 1])) - return {}; - seps.push_back(format.substr(start, i - start)); - start = i + 1; - } - } - seps.push_back(format.substr(start)); - return seps; -} - -std::vector splitDatetime(String format) -{ - int idx = getFracIndex(format); - if (idx > 0) - { - format = format.substr(0, idx); - } - return parseDateFormat(format); -} - -Field parseMyDatetime(const String & str) -{ - Int32 year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0; - - const auto & seps = splitDatetime(str); - - switch (seps.size()) - { - // No delimiter - case 1: - { - size_t l = seps[0].size(); - switch (l) - { - case 14: - // YYYYMMDDHHMMSS - { - std::sscanf(seps[0].c_str(), "%4d%2d%2d%2d%2d%2d", &year, &month, &day, &hour, &minute, &second); - break; - } - case 12: - { - std::sscanf(seps[0].c_str(), "%2d%2d%2d%2d%2d%2d", &year, &month, &day, &hour, &minute, &second); - year = adjustYear(year); - break; - } - case 11: - { - std::sscanf(seps[0].c_str(), "%2d%2d%2d%2d%2d%1d", &year, &month, &day, &hour, &minute, &second); - year = adjustYear(year); - break; - } - case 10: - { - std::sscanf(seps[0].c_str(), "%2d%2d%2d%2d%2d", &year, &month, &day, &hour, &minute); - year = adjustYear(year); - break; - } - case 9: - { - std::sscanf(seps[0].c_str(), "%2d%2d%2d%2d%1d", &year, &month, &day, &hour, &minute); - year = adjustYear(year); - break; - } - case 8: - { - std::sscanf(seps[0].c_str(), "%4d%2d%2d", &year, &month, &day); - break; - } - case 6: - case 5: - { - std::sscanf(seps[0].c_str(), "%2d%2d%2d", &year, &month, &day); - year = adjustYear(year); - break; - } - default: - { - throw Exception("Wrong datetime format"); - } - // TODO Process frac! - } - break; - } - case 3: - { - scanTimeArgs(seps, {&year, &month, &day}); - break; - } - case 6: - { - scanTimeArgs(seps, {&year, &month, &day, &hour, &minute, &second}); - break; - } - default: - { - throw Exception("Wrong datetime format"); - } - } - - UInt64 ymd = ((year * 13 + month) << 5) | day; - UInt64 hms = (hour << 12) | (minute << 6) | second; - return Field((ymd << 17 | hms) << 24); -} - -} // namespace DB diff --git a/dbms/src/Storages/Transaction/TiDB.cpp b/dbms/src/Storages/Transaction/TiDB.cpp index 67f58f16fc1..eda69f5b915 100644 --- a/dbms/src/Storages/Transaction/TiDB.cpp +++ b/dbms/src/Storages/Transaction/TiDB.cpp @@ -1,7 +1,7 @@ #include +#include #include #include -#include #include namespace TiDB @@ -40,7 +40,7 @@ Field ColumnInfo::defaultValueToField() const case TypeDate: case TypeDatetime: case TypeTimestamp: - return DB::parseMyDatetime(value.convert()); + return DB::parseMyDateTime(value.convert()); case TypeVarchar: case TypeTinyBlob: case TypeMediumBlob: diff --git a/dbms/src/Storages/Transaction/TiDB.h b/dbms/src/Storages/Transaction/TiDB.h index 3a9650fa0b0..4e74b9ff0f4 100644 --- a/dbms/src/Storages/Transaction/TiDB.h +++ b/dbms/src/Storages/Transaction/TiDB.h @@ -43,14 +43,14 @@ using DB::Timestamp; M(Float, 4, Float, Float32, false) \ M(Double, 5, Float, Float64, false) \ M(Null, 6, Nil, Nothing, false) \ - M(Timestamp, 7, Int, DateTime, false) \ + M(Timestamp, 7, Int, MyDateTime, false) \ M(LongLong, 8, Int, Int64, false) \ M(Int24, 9, VarInt, Int32, true) \ - M(Date, 10, Int, Date, false) \ + M(Date, 10, Int, MyDate, false) \ M(Time, 11, Duration, Int64, false) \ - M(Datetime, 12, Int, DateTime, false) \ + M(Datetime, 12, Int, MyDateTime, false) \ M(Year, 13, Int, Int16, false) \ - M(NewDate, 14, Int, Date, false) \ + M(NewDate, 14, Int, MyDate, false) \ M(Varchar, 15, CompactBytes, String, false) \ M(Bit, 16, CompactBytes, UInt64, false) \ M(JSON, 0xf5, Json, String, false) \ diff --git a/dbms/src/Storages/Transaction/TypeMapping.cpp b/dbms/src/Storages/Transaction/TypeMapping.cpp index be8540dc92b..ba7f3ff3aab 100644 --- a/dbms/src/Storages/Transaction/TypeMapping.cpp +++ b/dbms/src/Storages/Transaction/TypeMapping.cpp @@ -1,9 +1,9 @@ #include -#include -#include #include #include +#include +#include #include #include #include @@ -80,8 +80,8 @@ template inline constexpr bool IsEnumType = EnumType::value; template -std::enable_if_t && !IsDecimalType && !IsEnumType, DataTypePtr> getDataTypeByColumnInfoBase( - const ColumnInfo &, const T *) +std::enable_if_t && !IsDecimalType && !IsEnumType && !std::is_same_v, DataTypePtr> +getDataTypeByColumnInfoBase(const ColumnInfo &, const T *) { DataTypePtr t = std::make_shared(); @@ -127,6 +127,21 @@ std::enable_if_t, DataTypePtr> getDataTypeByColumnInfoBase(cons return t; } + +template +std::enable_if_t, DataTypePtr> getDataTypeByColumnInfoBase(const ColumnInfo & column_info, const T *) +{ + DataTypePtr t = std::make_shared(column_info.decimal); + + if (should_widen) + { + auto widen = t->widen(); + t.swap(widen); + } + + return t; +} + template std::enable_if_t, DataTypePtr> getDataTypeByColumnInfoBase(const ColumnInfo & column_info, const T *) { diff --git a/libs/libcommon/CMakeLists.txt b/libs/libcommon/CMakeLists.txt index 05810c44aa1..4968393bef1 100644 --- a/libs/libcommon/CMakeLists.txt +++ b/libs/libcommon/CMakeLists.txt @@ -19,6 +19,7 @@ endif () add_library (common ${SPLIT_SHARED} src/DateLUT.cpp + src/DateLUTImpl.cpp src/preciseExp10.c src/shift10.cpp src/mremap.cpp diff --git a/libs/libcommon/include/common/DateLUT.h b/libs/libcommon/include/common/DateLUT.h index 75a7457d101..451bfa4d991 100644 --- a/libs/libcommon/include/common/DateLUT.h +++ b/libs/libcommon/include/common/DateLUT.h @@ -1,6 +1,6 @@ #pragma once -#include +#include "DateLUTImpl.h" #include #include #include diff --git a/libs/libcommon/include/common/DateLUTImpl.h b/libs/libcommon/include/common/DateLUTImpl.h index f3a641e8746..456d3ae80f7 100644 --- a/libs/libcommon/include/common/DateLUTImpl.h +++ b/libs/libcommon/include/common/DateLUTImpl.h @@ -1,65 +1,51 @@ #pragma once -#include -#include -#include -#include -#include - -#include -#include -#include - +#include "Types.h" +#include "DayNum.h" +#include "likely.h" #include +#include -#define DATE_LUT_MAX (0xFFFFFFFFFFFFFFFLL - 86400) -#define DATE_LUT_MIN -30610224000LL -#define DATE_LUT_MAX_DAY_NUM 36500000 +#define DATE_LUT_MAX (0xFFFFFFFFU - 86400) +#define DATE_LUT_MAX_DAY_NUM (0xFFFFFFFFU / 86400) /// Table size is bigger than DATE_LUT_MAX_DAY_NUM to fill all indices within UInt16 range: this allows to remove extra check. -#define DATE_LUT_SIZE 36652424 -#define DATE_LUT_MIN_YEAR 1400 -#define DATE_LUT_MAX_YEAR 9999 /// Last supported year +#define DATE_LUT_SIZE 0x10000 +#define DATE_LUT_MIN_YEAR 1970 +#define DATE_LUT_MAX_YEAR 2105 /// Last supported year #define DATE_LUT_YEARS (1 + DATE_LUT_MAX_YEAR - DATE_LUT_MIN_YEAR) /// Number of years in lookup table +#if defined(__PPC__) +#if !__clang__ +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif +#endif -STRONG_TYPEDEF(UInt32, DayNum_t); - +/// Flags for toYearWeek() function. +enum class WeekModeFlag : UInt8 +{ + MONDAY_FIRST = 1, + YEAR = 2, + FIRST_WEEKDAY = 4, + NEWYEAR_DAY = 8 +}; +typedef std::pair YearWeek; /** Lookup table to conversion of time to date, and to month / year / day of week / day of month and so on. * First time was implemented for OLAPServer, that needed to do billions of such transformations. */ class DateLUTImpl { - using Exception = Poco::Exception; - using DateType = boost::gregorian::date; - using DurationType = boost::posix_time::time_duration; - using ptime = boost::posix_time::ptime; - using TzPtr = boost::local_time::time_zone_ptr; - using ltime = boost::local_time::local_date_time; - - TzPtr zone; - UInt32 start_day_num; - public: - DateLUTImpl(const std::string & time_zone_) : time_zone(time_zone_) { - DurationType utc_offset(boost::date_time::second_clock::local_time() - boost::date_time::second_clock::universal_time()); - std::ostringstream ss; - if (!utc_offset.is_negative()) { - ss << "UTC+"<< utc_offset; - } else { - ss << "UTC" << utc_offset; - } - zone = TzPtr(new boost::local_time::posix_time_zone(ss.str())); - start_day_num = DateType(1400,1,1).day_count().as_number(); - }; + DateLUTImpl(const std::string & time_zone); public: + /// The order of fields matters for alignment and sizeof. struct Values { /// Least significat 32 bits from time_t at beginning of the day. /// If the unix timestamp of beginning of the day is negative (example: 1970-01-01 MSK, where time_t == -10800), then value is zero. /// Change to time_t; change constants above; and recompile the sources if you need to support time after 2105 year. - time_t date; + UInt32 date; /// Properties of the day. UInt16 year; @@ -72,267 +58,214 @@ class DateLUTImpl UInt8 days_in_month; /// For days, when offset from UTC was changed due to daylight saving time or permanent change, following values could be non zero. - UInt16 time_at_offset_change; /// In seconds from beginning of the day. Assuming offset never changed close to the end of day (so, value < 65536). Int16 amount_of_offset_change; /// Usually -3600 or 3600, but look at Lord Howe Island. + UInt32 time_at_offset_change; /// In seconds from beginning of the day. }; + static_assert(sizeof(Values) == 16); + private: - /// Lookup table is indexed by DayNum. + /// Lookup table is indexed by DayNum_t. /// Day nums are the same in all time zones. 1970-01-01 is 0 and so on. /// Table is relatively large, so better not to place the object on stack. /// In comparison to std::vector, plain array is cheaper by one indirection. - //Values lut[DATE_LUT_SIZE]; + Values lut[DATE_LUT_SIZE]; /// Year number after DATE_LUT_MIN_YEAR -> day num for start of year. - //DayNum_t years_lut[DATE_LUT_YEARS]; + DayNum_t years_lut[DATE_LUT_YEARS]; /// Year number after DATE_LUT_MIN_YEAR * month number starting at zero -> day num for first day of month - //DayNum_t years_months_lut[DATE_LUT_YEARS * 12]; + DayNum_t years_months_lut[DATE_LUT_YEARS * 12]; /// UTC offset at beginning of the Unix epoch. The same as unix timestamp of 1970-01-01 00:00:00 local time. - //time_t offset_at_start_of_epoch; - //bool offset_is_whole_number_of_hours_everytime; + time_t offset_at_start_of_epoch; + bool offset_is_whole_number_of_hours_everytime; /// Time zone name. std::string time_zone; /// We can correctly process only timestamps that less DATE_LUT_MAX (i.e. up to 2105 year inclusively) - //inline size_t findIndex(time_t /*t*/) const - //{ - // /// First guess. - // size_t guess = (t - DATE_LUT_MIN) / 86400; - // if (guess >= DATE_LUT_MAX_DAY_NUM) - // return 0; - // if (t >= lut[guess].date && t < lut[guess + 1].date) - // return guess; - - // for (size_t i = 1;; ++i) - // { - // if (guess + i >= DATE_LUT_MAX_DAY_NUM) - // return 0; - // if (t >= lut[guess + i].date && t < lut[guess + i + 1].date) - // return guess + i; - // if (guess < i) - // return 0; - // if (t >= lut[guess - i].date && t < lut[guess - i + 1].date) - // return guess - i; - // } - //} - - //inline const Values & find(time_t /*t*/) const - //{ - // return lut[findIndex(/*t*/)]; - //} + /// We don't care about overflow. + inline DayNum_t findIndex(time_t t) const + { + /// First guess. + DayNum_t guess(t / 86400); + + /// UTC offset is from -12 to +14 in all known time zones. This requires checking only three indices. + + if ((guess == 0 || t >= lut[guess].date) && t < lut[DayNum_t(guess + 1)].date) + return guess; + + /// Time zones that have offset 0 from UTC do daylight saving time change (if any) towards increasing UTC offset (example: British Standard Time). + if (offset_at_start_of_epoch >= 0) + return DayNum_t(guess + 1); + + return DayNum_t(guess - 1); + } + + inline const Values & find(time_t t) const + { + return lut[findIndex(t)]; + } public: const std::string & getTimeZone() const { return time_zone; } /// All functions below are thread-safe; arguments are not checked. - inline time_t toDate(time_t /*t*/) const { - std::cout<<__LINE__<= lut[index].time_at_offset_change) - // res += lut[index].amount_of_offset_change; + if (res >= lut[index].time_at_offset_change) + res += lut[index].amount_of_offset_change; - //return res - offset_at_start_of_epoch; /// Starting at 1000-01-01 00:00:00 local time. + return res - offset_at_start_of_epoch; /// Starting at 1970-01-01 00:00:00 local time. } inline unsigned toHour(time_t t) const { - ptime tm = boost::posix_time::from_time_t(t); - ltime lm (tm, zone); - return lm.local_time().time_of_day().hours(); + DayNum_t index = findIndex(t); + + /// If it is not 1970 year (findIndex found nothing appropriate), + /// than limit number of hours to avoid insane results like 1970-01-01 89:28:15 + if (unlikely(index == 0)) + return static_cast((t + offset_at_start_of_epoch) / 3600) % 24; + + time_t res = t - lut[index].date; + + /// Data is cleaned to avoid possibility of underflow. + if (res >= lut[index].time_at_offset_change) + res += lut[index].amount_of_offset_change; + + return res / 3600; } /** Only for time zones with/when offset from UTC is multiple of five minutes. @@ -346,57 +279,30 @@ class DateLUTImpl * each minute, with added or subtracted leap second, spans exactly 60 unix timestamps. */ - inline unsigned toSecond(time_t t) const - { - //size_t index = findIndex(/*t*/); - - ///// If it is not 1970 year (findIndex found nothing appropriate), - ///// than limit number of hours to avoid insane results like 1970-01-01 89:28:15 - //if (unlikely(index == 0)) - // return static_cast((t + offset_at_start_of_epoch) / 3600) % 24; - - //time_t res = t - lut[index].date; - - //if (res >= lut[index].time_at_offset_change) - // res += lut[index].amount_of_offset_change; - ptime tm = boost::posix_time::from_time_t(t); - ltime lm (tm, zone); - return lm.local_time().time_of_day().seconds(); - } + inline unsigned toSecond(time_t t) const { return t % 60; } inline unsigned toMinute(time_t t) const { - ptime tm = boost::posix_time::from_time_t(t); - ltime lm (tm, zone); - return lm.local_time().time_of_day().minutes(); - } + if (offset_is_whole_number_of_hours_everytime) + return (t / 60) % 60; - inline time_t toStartOfMinute(time_t /*t*/) const { - std::cout<<__LINE__<(WeekModeFlag::NEWYEAR_DAY); + week_mode = check_week_mode(week_mode); + bool monday_first_mode = week_mode & static_cast(WeekModeFlag::MONDAY_FIRST); + bool week_year_mode = week_mode & static_cast(WeekModeFlag::YEAR); + bool first_weekday_mode = week_mode & static_cast(WeekModeFlag::FIRST_WEEKDAY); + + // Calculate week number of WeekModeFlag::NEWYEAR_DAY mode + if (newyear_day_mode) + { + return toYearWeekOfNewyearMode(d, monday_first_mode); + } + + YearWeek yw(toYear(d), 0); + UInt16 days = 0; + UInt16 daynr = makeDayNum(yw.first, toMonth(d), toDayOfMonth(d)); + UInt16 first_daynr = makeDayNum(yw.first, 1, 1); + + // 0 for monday, 1 for tuesday ... + // get weekday from first day in year. + UInt16 weekday = calc_weekday(DayNum_t(first_daynr), !monday_first_mode); + + if (toMonth(d) == 1 && toDayOfMonth(d) <= static_cast(7 - weekday)) + { + if (!week_year_mode && ((first_weekday_mode && weekday != 0) || (!first_weekday_mode && weekday >= 4))) + return yw; + week_year_mode = 1; + (yw.first)--; + first_daynr -= (days = calc_days_in_year(yw.first)); + weekday = (weekday + 53 * 7 - days) % 7; + } + + if ((first_weekday_mode && weekday != 0) || (!first_weekday_mode && weekday >= 4)) + days = daynr - (first_daynr + (7 - weekday)); + else + days = daynr - (first_daynr - weekday); + + if (week_year_mode && days >= 52 * 7) + { + weekday = (weekday + calc_days_in_year(yw.first)) % 7; + if ((!first_weekday_mode && weekday < 4) || (first_weekday_mode && weekday == 0)) + { + (yw.first)++; + yw.second = 1; + return yw; + } + } + yw.second = days / 7 + 1; + return yw; + } + + /// Calculate week number of WeekModeFlag::NEWYEAR_DAY mode + /// The week number 1 is the first week in year that contains January 1, + inline YearWeek toYearWeekOfNewyearMode(DayNum_t d, bool monday_first_mode) const + { + YearWeek yw(0, 0); + UInt16 offset_day = monday_first_mode ? 0U : 1U; + + // Checking the week across the year + yw.first = toYear(DayNum_t(d + 7 - toDayOfWeek(DayNum_t(d + offset_day)))); + + DayNum_t first_day = makeDayNum(yw.first, 1, 1); + DayNum_t this_day = d; + + if (monday_first_mode) + { + // Rounds down a date to the nearest Monday. + first_day = toFirstDayNumOfWeek(first_day); + this_day = toFirstDayNumOfWeek(d); + } + else + { + // Rounds down a date to the nearest Sunday. + if (toDayOfWeek(first_day) != 7) + first_day = DayNum_t(first_day - toDayOfWeek(first_day)); + if (toDayOfWeek(d) != 7) + this_day = DayNum_t(d - toDayOfWeek(d)); + } + yw.second = (this_day - first_day) / 7 + 1; + return yw; + } + + /** + * get first day of week with week_mode, return Sunday or Monday + */ + inline DayNum_t toFirstDayNumOfWeek(DayNum_t d, UInt8 week_mode) const + { + bool monday_first_mode = week_mode & static_cast(WeekModeFlag::MONDAY_FIRST); + if (monday_first_mode) + { + return toFirstDayNumOfWeek(d); + } + else + { + return (toDayOfWeek(d) != 7) ? DayNum_t(d - toDayOfWeek(d)) : d; + } + } + + /* + * check and change mode to effective + */ + inline UInt8 check_week_mode(UInt8 mode) const + { + UInt8 week_format = (mode & 7); + if (!(week_format & static_cast(WeekModeFlag::MONDAY_FIRST))) + week_format ^= static_cast(WeekModeFlag::FIRST_WEEKDAY); + return week_format; + } + + /* + * Calc weekday from d + * Returns 0 for monday, 1 for tuesday ... + */ + inline unsigned calc_weekday(DayNum_t d, bool sunday_first_day_of_week) const + { + if (!sunday_first_day_of_week) + return toDayOfWeek(d) - 1; + else + return toDayOfWeek(DayNum_t(d + 1)) - 1; + } + + /* Calc days in one year. */ + inline unsigned calc_days_in_year(UInt16 year) const + { + return ((year & 3) == 0 && (year % 100 || (year % 400 == 0 && year)) ? 366 : 365); } /// Number of month from some fixed moment in the past (year * 12 + month) - inline unsigned toRelativeMonthNum(DayNum_t /*d*/) const + inline unsigned toRelativeMonthNum(DayNum_t d) const { - std::cout<<__LINE__< DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31)) return DayNum_t(0); - auto day_num = DateType(year, month, day_of_month).day_count().as_number(); - return DayNum_t(day_num- start_day_num); + return DayNum_t(years_months_lut[(year - DATE_LUT_MIN_YEAR) * 12 + month - 1] + day_of_month - 1); } - inline time_t makeDate(UInt16 , UInt8 , UInt8) const + inline time_t makeDate(UInt16 year, UInt8 month, UInt8 day_of_month) const { - std::cout<<__LINE__<= lut[index].time_at_offset_change) + time_offset -= lut[index].amount_of_offset_change; + + return lut[index].date + time_offset; } - inline UInt32 toNumYYYYMM(time_t /*t*/) const + inline const Values & getValues(DayNum_t d) const { return lut[d]; } + inline const Values & getValues(time_t t) const { return lut[findIndex(t)]; } + + inline UInt32 toNumYYYYMM(time_t t) const { - std::cout<<__LINE__<= lut[index].time_at_offset_change) - // time_offset -= lut[index].amount_of_offset_change; + if (time_offset >= lut[index].time_at_offset_change) + time_offset -= lut[index].amount_of_offset_change; - //return lut[index].date + time_offset; + return lut[index].date + time_offset; } - inline time_t addWeeks(time_t, Int64 ) const + inline time_t addWeeks(time_t t, Int64 delta) const { - std::cout<<__LINE__< days_in_month) - // day_of_month = days_in_month; + if (day_of_month > days_in_month) + day_of_month = days_in_month; - //return day_of_month; + return day_of_month; } /// If resulting month has less deys than source month, then saturation can happen. /// Example: 31 Aug + 1 month = 30 Sep. - inline time_t addMonths(time_t, Int64 ) const + inline time_t addMonths(time_t t, Int64 delta) const { - std::cout<<__LINE__<= lut[result_day].time_at_offset_change) - // time_offset -= lut[result_day].amount_of_offset_change; + if (time_offset >= lut[result_day].time_at_offset_change) + time_offset -= lut[result_day].amount_of_offset_change; - //return lut[result_day].date + time_offset; + return lut[result_day].date + time_offset; } - inline DayNum_t addMonths(DayNum_t, Int64) const + inline DayNum_t addMonths(DayNum_t d, Int64 delta) const { - std::cout<<__LINE__<(values.month) + delta; - //Int64 month = static_cast(values.month) + delta; + if (month > 0) + { + auto year = values.year + (month - 1) / 12; + month = ((month - 1) % 12) + 1; + auto day_of_month = saturateDayOfMonth(year, month, values.day_of_month); + + return makeDayNum(year, month, day_of_month); + } + else + { + auto year = values.year - (12 - month) / 12; + month = 12 - (-month % 12); + auto day_of_month = saturateDayOfMonth(year, month, values.day_of_month); - //if (month > 0) - //{ - // auto year = values.year + (month - 1) / 12; - // month = ((month - 1) % 12) + 1; - // auto day_of_month = saturateDayOfMonth(year, month, values.day_of_month); + return makeDayNum(year, month, day_of_month); + } + } - // return makeDayNum(year, month, day_of_month); - //} - //else - //{ - // auto year = values.year - (12 - month) / 12; - // month = 12 - (-month % 12); - // auto day_of_month = saturateDayOfMonth(year, month, values.day_of_month); + inline time_t addQuarters(time_t t, Int64 delta) const + { + return addMonths(t, delta * 3); + } - // return makeDayNum(year, month, day_of_month); - //} + inline DayNum_t addQuarters(DayNum_t d, Int64 delta) const + { + return addMonths(d, delta * 3); } /// Saturation can occur if 29 Feb is mapped to non-leap year. - inline time_t addYears(time_t, Int64) const + inline time_t addYears(time_t t, Int64 delta) const { - std::cout<<__LINE__<= lut[result_day].time_at_offset_change) - // time_offset -= lut[result_day].amount_of_offset_change; + if (time_offset >= lut[result_day].time_at_offset_change) + time_offset -= lut[result_day].amount_of_offset_change; - //return lut[result_day].date + time_offset; + return lut[result_day].date + time_offset; } - inline DayNum_t addYears(DayNum_t, Int64) const + inline DayNum_t addYears(DayNum_t d, Int64 delta) const { - std::cout<<__LINE__< +#include + +/** Represents number of days since 1970-01-01. + * See DateLUTImpl for usage examples. + */ +STRONG_TYPEDEF(UInt16, DayNum_t) diff --git a/libs/libcommon/src/DateLUT.cpp b/libs/libcommon/src/DateLUT.cpp index 22c50a9f694..b6015b06b8e 100644 --- a/libs/libcommon/src/DateLUT.cpp +++ b/libs/libcommon/src/DateLUT.cpp @@ -6,6 +6,7 @@ #include #include + namespace { @@ -13,15 +14,16 @@ Poco::DigestEngine::Digest calcSHA1(const std::string & path) { std::ifstream stream(path); if (!stream) - throw Poco::Exception("Error while opening file: `" + path + "'."); + throw Poco::Exception("Error while opening file: '" + path + "'."); Poco::SHA1Engine digest_engine; Poco::DigestInputStream digest_stream(digest_engine, stream); digest_stream.ignore(std::numeric_limits::max()); if (!stream.eof()) - throw Poco::Exception("Error while reading file: `" + path + "'."); + throw Poco::Exception("Error while reading file: '" + path + "'."); return digest_engine.digest(); } + std::string determineDefaultTimeZone() { namespace fs = boost::filesystem; @@ -32,34 +34,71 @@ std::string determineDefaultTimeZone() fs::path tz_file_path; std::string error_prefix; const char * tz_env_var = std::getenv("TZ"); + + /// In recent tzdata packages some files now are symlinks and canonical path resolution + /// may give wrong timezone names - store the name as it is, if possible. + std::string tz_name; + if (tz_env_var) { - error_prefix = std::string("Could not determine time zone from TZ variable value: `") + tz_env_var + "': "; + error_prefix = std::string("Could not determine time zone from TZ variable value: '") + tz_env_var + "': "; if (*tz_env_var == ':') ++tz_env_var; tz_file_path = tz_env_var; + tz_name = tz_env_var; } else { error_prefix = "Could not determine local time zone: "; tz_file_path = "/etc/localtime"; + + /// No TZ variable and no tzdata installed (e.g. Docker) + if (!fs::exists(tz_file_path)) + return "UTC"; + + /// Read symlink but not transitive. + /// Example: + /// /etc/localtime -> /usr/share/zoneinfo//UTC + /// /usr/share/zoneinfo//UTC -> UCT + /// But the preferred time zone name is pointed by the first link (UTC), and the second link is just an internal detail. + if (fs::is_symlink(tz_file_path)) + { + tz_file_path = fs::read_symlink(tz_file_path); + /// If it's relative - make it absolute. + if (tz_file_path.is_relative()) + tz_file_path = (fs::path("/etc/") / tz_file_path).lexically_normal(); + } } try { tz_database_path = fs::canonical(tz_database_path); - tz_file_path = fs::canonical(tz_file_path, tz_database_path); /// The tzdata file exists. If it is inside the tz_database_dir, /// then the relative path is the time zone id. - fs::path relative_path = tz_file_path.lexically_relative(tz_database_path); - if (!relative_path.empty() && *relative_path.begin() != ".." && *relative_path.begin() != ".") - return relative_path.string(); + { + fs::path relative_path = tz_file_path.lexically_relative(tz_database_path); + + if (!relative_path.empty() && *relative_path.begin() != ".." && *relative_path.begin() != ".") + return tz_name.empty() ? relative_path.string() : tz_name; + } + + /// Try the same with full symlinks resolution + { + if (!tz_file_path.is_absolute()) + tz_file_path = tz_database_path / tz_file_path; + + tz_file_path = fs::canonical(tz_file_path); + + fs::path relative_path = tz_file_path.lexically_relative(tz_database_path); + if (!relative_path.empty() && *relative_path.begin() != ".." && *relative_path.begin() != ".") + return tz_name.empty() ? relative_path.string() : tz_name; + } - /// The file is not inside the tz_database_dir, so we hope that it was copied and - /// try to find the file with exact same contents in the database. + /// The file is not inside the tz_database_dir, so we hope that it was copied (not symlinked) + /// and try to find the file with exact same contents in the database. size_t tzfile_size = fs::file_size(tz_file_path); Poco::SHA1Engine::Digest tzfile_sha1 = calcSHA1(tz_file_path.string()); @@ -73,11 +112,11 @@ std::string determineDefaultTimeZone() { /// Some timezone databases contain copies of toplevel tzdata files in the posix/ directory /// and tzdata files with leap seconds in the right/ directory. Skip them. - candidate_it.no_push(); + candidate_it.disable_recursion_pending(); continue; } - if (candidate_it->status().type() != fs::regular_file || path.filename() == "localtime") + if (!fs::is_regular_file(*candidate_it) || path.filename() == "localtime") continue; if (fs::file_size(path) == tzfile_size && calcSHA1(path.string()) == tzfile_sha1) diff --git a/libs/libcommon/src/DateLUTImpl.cpp b/libs/libcommon/src/DateLUTImpl.cpp index 9f93b5e1fe4..3f812accb48 100644 --- a/libs/libcommon/src/DateLUTImpl.cpp +++ b/libs/libcommon/src/DateLUTImpl.cpp @@ -18,6 +18,9 @@ #include #include +#define DATE_LUT_MIN 0 + + namespace { @@ -51,17 +54,17 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) if (!cctz::load_time_zone(time_zone.data(), &cctz_time_zone)) throw Poco::Exception("Cannot load time zone " + time_zone_); - cctz::time_zone::absolute_lookup start_of_epoch_lookup = cctz_time_zone.lookup(std::chrono::time_point(std::chrono::seconds(DATE_LUT_MIN))); + cctz::time_zone::absolute_lookup start_of_epoch_lookup = cctz_time_zone.lookup(std::chrono::system_clock::from_time_t(start_of_day)); offset_at_start_of_epoch = start_of_epoch_lookup.offset; offset_is_whole_number_of_hours_everytime = true; - cctz::civil_day date{1000, 1, 1}; + cctz::civil_day date{1970, 1, 1}; do { cctz::time_zone::civil_lookup lookup = cctz_time_zone.lookup(date); - start_of_day = lookup.pre.time_since_epoch().count(); /*fuck chrono*/ /// Ambiguity is possible. + start_of_day = std::chrono::system_clock::to_time_t(lookup.pre); /// Ambiguity is possible. Values & values = lut[i]; values.year = date.year(); @@ -98,7 +101,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) /// when UTC offset was changed. Search is performed with 15-minute granularity, assuming it is enough. time_t time_at_offset_change = 900; - while (time_at_offset_change < 65536) + while (time_at_offset_change < 86400) { auto utc_offset_at_current_time = cctz_time_zone.lookup(std::chrono::system_clock::from_time_t( lut[i - 1].date + time_at_offset_change)).offset; @@ -109,10 +112,11 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) time_at_offset_change += 900; } - lut[i - 1].time_at_offset_change = time_at_offset_change >= 65536 ? 0 : time_at_offset_change; + lut[i - 1].time_at_offset_change = time_at_offset_change; -/* std::cerr << lut[i - 1].year << "-" << int(lut[i - 1].month) << "-" << int(lut[i - 1].day_of_month) - << " offset was changed at " << lut[i - 1].time_at_offset_change << " for " << lut[i - 1].amount_of_offset_change << " seconds.\n";*/ + /// We doesn't support cases when time change results in switching to previous day. + if (static_cast(lut[i - 1].time_at_offset_change) + static_cast(lut[i - 1].amount_of_offset_change) < 0) + lut[i - 1].time_at_offset_change = -lut[i - 1].amount_of_offset_change; } } @@ -125,7 +129,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) /// Fill excessive part of lookup table. This is needed only to simplify handling of overflow cases. while (i < DATE_LUT_SIZE) { - lut[i] = lut[0]; + lut[i] = lut[DATE_LUT_MAX_DAY_NUM]; ++i; } diff --git a/tests/mutable-test/datetime/insert.test b/tests/mutable-test/datetime/insert.test new file mode 100644 index 00000000000..4525f2d4765 --- /dev/null +++ b/tests/mutable-test/datetime/insert.test @@ -0,0 +1,35 @@ +>> DBGInvoke __enable_schema_sync_service('false') + +>> drop table if exists test + +>> create table test(a MyDateTime(5)) engine = Log + +>> insert into test values('1991-09-05 11:11:11') + +>> insert into test values('1991-09-05 11:11:11.123') + +>> insert into test values('1991-00-00 11:11:11') + +>> select a from test +┌─a─────────────────────────┐ +│ 1991-09-05 11:11:11.00000 │ +│ 1991-09-05 11:11:11.12300 │ +│ 1991-00-00 11:11:11.00000 │ +└───────────────────────────┘ + +>> drop table if exists test + +>> create table test(a MyDate) engine = Log + +>> insert into test values('1991-09-05') +>> insert into test values('1991-00-05') +>> insert into test values('1001-01-00') + +>> select a from test +┌─a───────────┐ +│ 1991-09-05 │ +│ 1991-00-05 │ +│ 1001-01-00 │ +└─────────────┘ + +>> drop table if exists test diff --git a/tests/mutable-test/txn_schema/mydate.test b/tests/mutable-test/txn_schema/mydate.test new file mode 100644 index 00000000000..aab1ace6fb4 --- /dev/null +++ b/tests/mutable-test/txn_schema/mydate.test @@ -0,0 +1,44 @@ + +=> DBGInvoke __enable_schema_sync_service('false') + +=> DBGInvoke __drop_tidb_table(default, test) +=> DBGInvoke __refresh_schemas() +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 MyDateTime(1)') + +=> DBGInvoke __put_region(4, 0, 100, default, test) + +=> DBGInvoke __raft_insert_row(default, test, 4, 51, '1991-11-12 11:12:13.234') +=> DBGInvoke __raft_insert_row(default, test, 4, 52, '1991-00-14 11:00:01') +=> DBGInvoke __raft_insert_row(default, test, 4, 53, '2001-12-13 11:11:11') +=> DBGInvoke __try_flush_region(4) +=> select * from default.test + +┌───────────────col_1───┬─_tidb_rowid─┐ +│ 1991-11-12 11:12:13.2 │ 51 │ +│ 1991-00-14 11:00:01.0 │ 52 │ +│ 2001-12-13 11:11:11.0 │ 53 │ +└───────────────────────┴─────────────┘ + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test + +=> DBGInvoke __mock_tidb_table(default, test, 'col_1 MyDate') + +=> DBGInvoke __put_region(4, 0, 100, default, test) + +=> DBGInvoke __raft_insert_row(default, test, 4, 51, '1991-11-12') +=> DBGInvoke __raft_insert_row(default, test, 4, 52, '1991-00-14') +=> DBGInvoke __raft_insert_row(default, test, 4, 53, '2001-12-13') +=> DBGInvoke __try_flush_region(4) +=> select * from default.test + +┌──────col_1─┬─_tidb_rowid─┐ +│ 1991-11-12 │ 51 │ +│ 1991-00-14 │ 52 │ +│ 2001-12-13 │ 53 │ +└────────────┴─────────────┘ + +=> DBGInvoke __drop_tidb_table(default, test) +=> drop table if exists default.test + +=> DBGInvoke __enable_schema_sync_service('true')