From 71be2102df58cb5ab8b518088e9c0cae062e46db Mon Sep 17 00:00:00 2001 From: HappenLee Date: Mon, 7 Jun 2021 05:30:32 -0500 Subject: [PATCH] Support Time Function (#62) --- be/src/exprs/timestamp_functions.cpp | 31 +- be/src/exprs/timestamp_functions.h | 2 + be/src/runtime/datetime_value.cpp | 12 +- be/src/runtime/datetime_value.h | 120 ++++--- be/src/vec/CMakeLists.txt | 2 + be/src/vec/functions/date_time_transforms.h | 253 +++++++++++--- .../function_date_or_datetime_computation.cpp | 12 +- .../function_date_or_datetime_computation.h | 308 +++++++++--------- .../function_date_or_datetime_to_string.cpp | 32 ++ .../function_date_or_datetime_to_string.h | 66 ++++ .../function_datetime_string_to_string.cpp | 32 ++ .../function_datetime_string_to_string.h | 104 ++++++ .../vec/functions/simple_function_factory.h | 10 +- be/src/vec/functions/to_time_function.cpp | 13 +- build.sh | 2 +- .../org/apache/doris/catalog/FunctionSet.java | 2 +- gensrc/script/doris_builtins_functions.py | 20 +- gensrc/script/gen_builtins_functions.py | 12 +- 18 files changed, 738 insertions(+), 295 deletions(-) create mode 100644 be/src/vec/functions/function_date_or_datetime_to_string.cpp create mode 100644 be/src/vec/functions/function_date_or_datetime_to_string.h create mode 100644 be/src/vec/functions/function_datetime_string_to_string.cpp create mode 100644 be/src/vec/functions/function_datetime_string_to_string.h diff --git a/be/src/exprs/timestamp_functions.cpp b/be/src/exprs/timestamp_functions.cpp index a8cf9a7e29a36f..610187a43b8083 100644 --- a/be/src/exprs/timestamp_functions.cpp +++ b/be/src/exprs/timestamp_functions.cpp @@ -62,6 +62,29 @@ bool TimestampFunctions::check_format(const StringVal& format, DateTimeValue& t) return false; } +std::string TimestampFunctions::convert_format(const std::string& format) { + switch (format.size()) { + case 8: + if (strncmp(format.c_str(), "yyyyMMdd", 8) == 0) { + return std::string("%Y%m%d"); + } + break; + case 10: + if (strncmp(format.c_str(), "yyyy-MM-dd", 10) == 0) { + return std::string("%Y-%m-%d"); + } + break; + case 19: + if (strncmp(format.c_str(), "yyyy-MM-dd HH:mm:ss", 19) == 0) { + return std::string("%Y-%m-%d %H:%i:%s"); + } + break; + default: + break; + } + return format; +} + StringVal TimestampFunctions::convert_format(FunctionContext* ctx, const StringVal& format) { switch (format.len) { case 8: @@ -521,10 +544,10 @@ DateTimeVal from_olap_datetime(uint64_t datetime) { static const DateTimeVal FIRST_DAY = from_olap_datetime(19700101000000); static const DateTimeVal FIRST_SUNDAY = from_olap_datetime(19700104000000); -#define TIME_ROUND(UNIT, unit, ORIGIN) \ - _TR_4(FLOOR, floor, UNIT, unit) \ - _TR_4(CEIL, ceil, UNIT, unit) _TR_5(FLOOR, floor, UNIT, unit, ORIGIN) \ - _TR_5(CEIL, ceil, UNIT, unit, ORIGIN) +#define TIME_ROUND(UNIT, unit, ORIGIN) \ + _TR_4(FLOOR, floor, UNIT, unit) \ + _TR_4(CEIL, ceil, UNIT, unit) \ + _TR_5(FLOOR, floor, UNIT, unit, ORIGIN) _TR_5(CEIL, ceil, UNIT, unit, ORIGIN) TIME_ROUND(YEAR, year, FIRST_DAY) TIME_ROUND(MONTH, month, FIRST_DAY) diff --git a/be/src/exprs/timestamp_functions.h b/be/src/exprs/timestamp_functions.h index 17f38f62eb42aa..155837c606e102 100644 --- a/be/src/exprs/timestamp_functions.h +++ b/be/src/exprs/timestamp_functions.h @@ -420,6 +420,8 @@ class TimestampFunctions { // Todo(kks): remove this method when 0.12 release static StringVal convert_format(doris_udf::FunctionContext* ctx, const StringVal& format); + static std::string convert_format(const std::string& format); + // Issue a warning for a bad format string. static void report_bad_format(const StringVal* format); diff --git a/be/src/runtime/datetime_value.cpp b/be/src/runtime/datetime_value.cpp index b417b8827cb701..eaafb5c66699a8 100644 --- a/be/src/runtime/datetime_value.cpp +++ b/be/src/runtime/datetime_value.cpp @@ -28,19 +28,15 @@ #include "util/timezone_utils.h" namespace doris { - const uint64_t log_10_int[] = {1, 10, 100, 1000, 10000UL, 100000UL, 1000000UL, 10000000UL, 100000000UL, 1000000000UL, 10000000000UL, 100000000000UL}; static int s_days_in_month[13] = {0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; -static const char* s_month_name[] = {"", "January", "February", "March", "April", - "May", "June", "July", "August", "September", - "October", "November", "December", NULL}; + static const char* s_ab_month_name[] = {"", "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", NULL}; -static const char* s_day_name[] = {"Monday", "Tuesday", "Wednesday", "Thursday", - "Friday", "Saturday", "Sunday", NULL}; + static const char* s_ab_day_name[] = {"Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun", NULL}; uint8_t mysql_week_mode(uint32_t mode) { @@ -1154,7 +1150,7 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con date_part_used = true; break; case 'M': - int_value = check_word(s_month_name, val, val_end, &val); + int_value = check_word(const_cast(s_month_name), val, val_end, &val); if (int_value < 0) { return false; } @@ -1249,7 +1245,7 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con break; // Weekday case 'W': - int_value = check_word(s_day_name, val, val_end, &val); + int_value = check_word(const_cast(s_day_name), val, val_end, &val); if (int_value < 0) { return false; } diff --git a/be/src/runtime/datetime_value.h b/be/src/runtime/datetime_value.h index 23258e61c7d1c6..52e7b1b6189e9b 100644 --- a/be/src/runtime/datetime_value.h +++ b/be/src/runtime/datetime_value.h @@ -137,6 +137,28 @@ const int TIME_MAX_SECOND = 59; const int TIME_MAX_VALUE = 10000 * TIME_MAX_HOUR + 100 * TIME_MAX_MINUTE + TIME_MAX_SECOND; const int TIME_MAX_VALUE_SECONDS = 3600 * TIME_MAX_HOUR + 60 * TIME_MAX_MINUTE + TIME_MAX_SECOND; +constexpr size_t const_length(const char* str) { + return (str == nullptr || *str == 0) ? 0 : const_length(str + 1) + 1; +} + +constexpr size_t max_char_length(const char* const* name, size_t end) { + size_t res = 0; + for (int i = 0; i < end; ++i) { + res = std::max(const_length(name[i]), res); + } + return res; +} + +static constexpr const char* s_month_name[] = { + "", "January", "February", "March", "April", "May", "June", + "July", "August", "September", "October", "November", "December", NULL}; + +static constexpr const char* s_day_name[] = {"Monday", "Tuesday", "Wednesday", "Thursday", + "Friday", "Saturday", "Sunday", NULL}; + +static constexpr size_t MAX_DAY_NAME_LEN = max_char_length(s_day_name, std::size(s_day_name)); +static constexpr size_t MAX_MONTH_NAME_LEN = max_char_length(s_month_name, std::size(s_month_name)); + uint8_t mysql_week_mode(uint32_t mode); class DateTimeValue { @@ -258,57 +280,59 @@ class DateTimeValue { template static int64_t datetime_diff(const DateTimeValue& ts_value1, const DateTimeValue& ts_value2) { switch (unit) { - case YEAR: { - int year = (ts_value2.year() - ts_value1.year()); - if (year > 0) { - year -= (ts_value2.to_int64() % 10000000000 - ts_value1.to_int64() % 10000000000) < 0; - } else if (year < 0) { - year += (ts_value2.to_int64() % 10000000000 - ts_value1.to_int64() % 10000000000) > 0; - } - return year; - } - case MONTH: { - int month = (ts_value2.year() - ts_value1.year()) * 12 + - (ts_value2.month() - ts_value1.month()); - if (month > 0) { - month -= (ts_value2.to_int64() % 100000000 - ts_value1.to_int64() % 100000000) < 0; - } else if (month < 0) { - month += (ts_value2.to_int64() % 100000000 - ts_value1.to_int64() % 100000000) > 0; - } - return month; - } - case WEEK: { - int day = ts_value2.daynr() - ts_value1.daynr(); - if (day > 0) { - day -= ts_value2.time_part_diff(ts_value1) < 0; - } else if (day < 0) { - day += ts_value2.time_part_diff(ts_value1) > 0; - } - return day / 7; - } - case DAY: { - int day = ts_value2.daynr() - ts_value1.daynr(); - if (day > 0) { - day -= ts_value2.time_part_diff(ts_value1) < 0; - } else if (day < 0) { - day += ts_value2.time_part_diff(ts_value1) > 0; - } - return day; + case YEAR: { + int year = (ts_value2.year() - ts_value1.year()); + if (year > 0) { + year -= (ts_value2.to_int64() % 10000000000 - ts_value1.to_int64() % 10000000000) < + 0; + } else if (year < 0) { + year += (ts_value2.to_int64() % 10000000000 - ts_value1.to_int64() % 10000000000) > + 0; } - case HOUR: { - int64_t second = ts_value2.second_diff(ts_value1); - int64_t hour = second / 60 / 60; - return hour; + return year; + } + case MONTH: { + int month = (ts_value2.year() - ts_value1.year()) * 12 + + (ts_value2.month() - ts_value1.month()); + if (month > 0) { + month -= (ts_value2.to_int64() % 100000000 - ts_value1.to_int64() % 100000000) < 0; + } else if (month < 0) { + month += (ts_value2.to_int64() % 100000000 - ts_value1.to_int64() % 100000000) > 0; } - case MINUTE: { - int64_t second = ts_value2.second_diff(ts_value1); - int64_t minute = second / 60; - return minute; + return month; + } + case WEEK: { + int day = ts_value2.daynr() - ts_value1.daynr(); + if (day > 0) { + day -= ts_value2.time_part_diff(ts_value1) < 0; + } else if (day < 0) { + day += ts_value2.time_part_diff(ts_value1) > 0; } - case SECOND: { - int64_t second = ts_value2.second_diff(ts_value1); - return second; + return day / 7; + } + case DAY: { + int day = ts_value2.daynr() - ts_value1.daynr(); + if (day > 0) { + day -= ts_value2.time_part_diff(ts_value1) < 0; + } else if (day < 0) { + day += ts_value2.time_part_diff(ts_value1) > 0; } + return day; + } + case HOUR: { + int64_t second = ts_value2.second_diff(ts_value1); + int64_t hour = second / 60 / 60; + return hour; + } + case MINUTE: { + int64_t second = ts_value2.second_diff(ts_value1); + int64_t minute = second / 60; + return minute; + } + case SECOND: { + int64_t second = ts_value2.second_diff(ts_value1); + return second; + } } // Rethink the default return value return 0; @@ -328,7 +352,7 @@ class DateTimeValue { int year() const { return _year; } int month() const { return _month; } - int quarter() const { return (_month - 1) / 3 + 1; } + int quarter() const { return (_month - 1) / 3 + 1; } int day() const { return _day; } int hour() const { return _hour; } int minute() const { return _minute; } diff --git a/be/src/vec/CMakeLists.txt b/be/src/vec/CMakeLists.txt index 7daf1dacf1864e..5ecc3c6a36ce94 100644 --- a/be/src/vec/CMakeLists.txt +++ b/be/src/vec/CMakeLists.txt @@ -104,6 +104,8 @@ set(VEC_FILES functions/time_of_function.cpp functions/if.cpp functions/function_date_or_datetime_computation.cpp + functions/function_date_or_datetime_to_string.cpp + functions/function_datetime_string_to_string.cpp sink/mysql_result_writer.cpp sink/result_sink.cpp sink/vdata_stream_sender.cpp diff --git a/be/src/vec/functions/date_time_transforms.h b/be/src/vec/functions/date_time_transforms.h index 6f9f036856ae52..cf98f8c35b6d65 100644 --- a/be/src/vec/functions/date_time_transforms.h +++ b/be/src/vec/functions/date_time_transforms.h @@ -18,22 +18,19 @@ #pragma once #include "runtime/datetime_value.h" - -#include "vec/core/types.h" -#include "vec/common/exception.h" +#include "vec/columns/column_string.h" #include "vec/columns/column_vector.h" +#include "vec/common/exception.h" +#include "vec/core/types.h" #include "vec/functions/function_helpers.h" //#include "vec/functions/extract_time_zone_from_function_arguments.h> +namespace doris::vectorized { -namespace doris::vectorized -{ - -namespace ErrorCodes -{ - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int ILLEGAL_COLUMN; -} +namespace ErrorCodes { +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int ILLEGAL_COLUMN; +} // namespace ErrorCodes /** Transformations. * Represents two functions - from datetime (UInt32) and from date (UInt16). @@ -48,9 +45,9 @@ namespace ErrorCodes * factor-transformation F is "round to the nearest month" (2015-02-03 -> 2015-02-01). */ -static inline UInt32 dateIsNotSupported(const char * name) -{ - throw Exception("Illegal type Date of argument for function " + std::string(name), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); +static inline UInt32 dateIsNotSupported(const char* name) { + throw Exception("Illegal type Date of argument for function " + std::string(name), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } ///// This factor transformation will say that the function is monotone everywhere. @@ -274,15 +271,14 @@ static inline UInt32 dateIsNotSupported(const char * name) // using FactorTransform = ZeroTransform; //}; -#define TIME_FUNCTION_IMPL(CLASS, UNIT, FUNCTION) \ -struct CLASS \ -{ \ - static constexpr auto name = #UNIT; \ - static inline auto execute(const Int128& t) { \ - const auto& date_time_value = (doris::DateTimeValue&)(t); \ - return date_time_value.FUNCTION; \ - } \ -} +#define TIME_FUNCTION_IMPL(CLASS, UNIT, FUNCTION) \ + struct CLASS { \ + static constexpr auto name = #UNIT; \ + static inline auto execute(const Int128& t) { \ + const auto& date_time_value = (doris::DateTimeValue&)(t); \ + return date_time_value.FUNCTION; \ + } \ + } #define TO_TIME_FUNCTION(CLASS, UNIT) TIME_FUNCTION_IMPL(CLASS, UNIT, UNIT()) @@ -300,6 +296,138 @@ TIME_FUNCTION_IMPL(DayOfMonthImpl, dayofmonth, day()); TIME_FUNCTION_IMPL(DayOfWeekImpl, dayofweek, day_of_week()); TIME_FUNCTION_IMPL(ToDaysImpl, to_days, daynr()); +struct ToDateImpl { + static constexpr auto name = "to_date"; + + static inline auto execute(const Int128& t) { + auto res = t; + auto& dt = (doris::DateTimeValue&)(res); + dt.cast_to_date(); + return res; + } +}; +struct DateImpl : public ToDateImpl { + static constexpr auto name = "date"; +}; + +// TODO: This function look like no need do indeed copy here, we should optimize +// this function +struct TimeStampImpl { + static constexpr auto name = "timestamp"; + static inline auto execute(const Int128& t) { return t; } +}; + +struct UnixTimeStampImpl { + static constexpr auto name = "unix_timestamp"; + static inline int execute(const Int128& t) { + // TODO: use default time zone, slowly and incorrect, just for test use + static cctz::time_zone time_zone = cctz::fixed_time_zone(cctz::seconds(8 * 60 * 60)); + + const auto& dt = (doris::DateTimeValue&)(t); + int64_t timestamp = 0; + dt.unix_timestamp(×tamp, time_zone); + + return (timestamp < 0 || timestamp > INT_MAX) ? 0 : timestamp; + } +}; + +struct DayNameImpl { + static constexpr auto name = "dayname"; + static constexpr auto max_size = MAX_DAY_NAME_LEN; + + static inline auto execute(const DateTimeValue& dt, ColumnString::Chars& res_data, + size_t& offset) { + const auto* day_name = dt.day_name(); + if (day_name == nullptr) { + offset += 1; + res_data[offset - 1] = 0; + } else { + auto len = strlen(day_name); + memcpySmallAllowReadWriteOverflow15(&res_data[offset], day_name, len); + offset += len + 1; + res_data[offset - 1] = 0; + } + return offset; + } +}; + +struct MonthNameImpl { + static constexpr auto name = "monthname"; + static constexpr auto max_size = MAX_MONTH_NAME_LEN; + + static inline auto execute(const DateTimeValue& dt, ColumnString::Chars& res_data, + size_t& offset) { + const auto* month_name = dt.month_name(); + if (month_name == nullptr) { + offset += 1; + res_data[offset - 1] = 0; + } else { + auto len = strlen(month_name); + memcpySmallAllowReadWriteOverflow15(&res_data[offset], month_name, len); + offset += len + 1; + res_data[offset - 1] = 0; + } + return offset; + } +}; + +struct DateFormatImpl { + using FromType = Int128; + + static constexpr auto name = "date_format"; + + static inline auto execute(const Int128& t, StringRef format, ColumnString::Chars& res_data, + size_t& offset) { + const auto& dt = (DateTimeValue&)t; + if (format.size > 128) { + offset += 1; + res_data.emplace_back(0); + return std::pair{offset, true}; + } + char buf[128]; + if (!dt.to_format_string(format.data, format.size, buf)) { + offset += 1; + res_data.emplace_back(0); + return std::pair{offset, true}; + } + + auto len = strlen(buf) + 1; + res_data.insert(buf, buf + len); + offset += len; + return std::pair{offset, false}; + } +}; + +struct FromUnixTimeImpl { + using FromType = Int32; + + static constexpr auto name = "from_unixtime"; + + static inline auto execute(FromType val, StringRef format, ColumnString::Chars& res_data, + size_t& offset) { + // TODO: use default time zone, slowly and incorrect, just for test use + static cctz::time_zone time_zone = cctz::fixed_time_zone(cctz::seconds(8 * 60 * 60)); + + DateTimeValue dt; + if (format.size > 128 || val < 0 || val > INT_MAX || !dt.from_unixtime(val, time_zone)) { + offset += 1; + res_data.emplace_back(0); + return std::pair{offset, true}; + } + + char buf[128]; + if (!dt.to_format_string(format.data, format.size, buf)) { + offset += 1; + res_data.emplace_back(0); + return std::pair{offset, true}; + } + + auto len = strlen(buf) + 1; + res_data.insert(buf, buf + len); + offset += len; + return std::pair{offset, false}; + } +}; //struct ToRelativeYearNumImpl //{ @@ -479,46 +607,77 @@ TIME_FUNCTION_IMPL(ToDaysImpl, to_days, daynr()); //}; // // + +template +struct TransformerToStringOneArgument { + static void vector(const PaddedPODArray& ts, ColumnString::Chars& res_data, + ColumnString::Offsets& res_offsets) { + auto len = ts.size(); + res_data.resize(len * Transform::max_size); + res_offsets.resize(len); + + size_t offset = 0; + for (int i = 0; i < len; ++i) { + const auto& t = ts[i]; + const auto& date_time_value = reinterpret_cast(t); + res_offsets[i] = Transform::execute(date_time_value, res_data, offset); + } + } +}; + +template +struct TransformerToStringTwoArgument { + static void vector_constant(const PaddedPODArray& ts, + const std::string& format, ColumnString::Chars& res_data, + ColumnString::Offsets& res_offsets, + PaddedPODArray& null_map) { + auto len = ts.size(); + res_offsets.resize(len); + + size_t offset = 0; + for (int i = 0; i < len; ++i) { + const auto& t = ts[i]; + const auto [new_offset, is_null] = Transform::execute( + t, StringRef(format.c_str(), format.size()), res_data, offset); + + res_offsets[i] = new_offset; + null_map[i] = is_null; + } + } +}; + template -struct Transformer -{ -// static void vector(const PaddedPODArray & vec_from, PaddedPODArray & vec_to, const DateLUTImpl & time_zone) - static void vector(const PaddedPODArray & vec_from, PaddedPODArray & vec_to) - { +struct Transformer { + // static void vector(const PaddedPODArray & vec_from, PaddedPODArray & vec_to, const DateLUTImpl & time_zone) + static void vector(const PaddedPODArray& vec_from, PaddedPODArray& vec_to) { size_t size = vec_from.size(); vec_to.resize(size); - for (size_t i = 0; i < size; ++i) -// vec_to[i] = Transform::execute(vec_from[i], time_zone); - vec_to[i] = Transform::execute(vec_from[i]); + for (size_t i = 0; i < size; ++i) vec_to[i] = Transform::execute(vec_from[i]); } }; - template -struct DateTimeTransformImpl -{ - static void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) - { +struct DateTimeTransformImpl { + static void execute(Block& block, const ColumnNumbers& arguments, size_t result, + size_t /*input_rows_count*/) { using Op = Transformer; -// const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(block, arguments, 1, 0); + // const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(block, arguments, 1, 0); const ColumnPtr source_col = block.getByPosition(arguments[0]).column; - if (const auto * sources = checkAndGetColumn>(source_col.get())) - { + if (const auto* sources = checkAndGetColumn>(source_col.get())) { auto col_to = ColumnVector::create(); -// Op::vector(sources->getData(), col_to->getData(), time_zone); + // Op::vector(sources->getData(), col_to->getData(), time_zone); Op::vector(sources->getData(), col_to->getData()); block.getByPosition(result).column = std::move(col_to); - } - else - { - throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName() - + " of first argument of function " + Transform::name, - ErrorCodes::ILLEGAL_COLUMN); + } else { + throw Exception("Illegal column " + + block.getByPosition(arguments[0]).column->getName() + + " of first argument of function " + Transform::name, + ErrorCodes::ILLEGAL_COLUMN); } } }; -} \ No newline at end of file +} // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/functions/function_date_or_datetime_computation.cpp b/be/src/vec/functions/function_date_or_datetime_computation.cpp index 3e54d8089274ce..ce4d18b340364f 100644 --- a/be/src/vec/functions/function_date_or_datetime_computation.cpp +++ b/be/src/vec/functions/function_date_or_datetime_computation.cpp @@ -15,12 +15,11 @@ // specific language governing permissions and limitations // under the License. -#include "vec/functions/function.h" -#include "vec/functions/simple_function_factory.h" #include "vec/functions/function_date_or_datetime_computation.h" -namespace doris::vectorized -{ +#include "vec/functions/simple_function_factory.h" + +namespace doris::vectorized { using FunctionAddSeconds = FunctionDateOrDateTimeComputation; using FunctionAddMinutes = FunctionDateOrDateTimeComputation; @@ -50,8 +49,7 @@ using FunctionHoursDiff = FunctionDateOrDateTimeComputation; using FunctionMinutesDiff = FunctionDateOrDateTimeComputation; using FunctionSecondsDiff = FunctionDateOrDateTimeComputation; -void registerFunctionDateTimeComputation(SimpleFunctionFactory & factory) -{ +void registerFunctionDateTimeComputation(SimpleFunctionFactory& factory) { factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); @@ -81,4 +79,4 @@ void registerFunctionDateTimeComputation(SimpleFunctionFactory & factory) factory.registerFunction(); } -} \ No newline at end of file +} // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/functions/function_date_or_datetime_computation.h b/be/src/vec/functions/function_date_or_datetime_computation.h index 9e7d679ec2b5f5..43f4c570b87198 100644 --- a/be/src/vec/functions/function_date_or_datetime_computation.h +++ b/be/src/vec/functions/function_date_or_datetime_computation.h @@ -16,23 +16,20 @@ // under the License. #include "runtime/datetime_value.h" - +#include "vec/columns/column_vector.h" #include "vec/data_types/data_type_date.h" #include "vec/data_types/data_type_date_time.h" #include "vec/data_types/data_type_number.h" -#include "vec/columns/column_vector.h" #include "vec/functions/function.h" #include "vec/functions/function_helpers.h" -namespace doris::vectorized -{ +namespace doris::vectorized { -namespace ErrorCodes -{ - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int ILLEGAL_COLUMN; -} +namespace ErrorCodes { +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int ILLEGAL_COLUMN; +} // namespace ErrorCodes template inline Int128 date_time_add(const Int128& t, Int64 delta) { @@ -44,15 +41,14 @@ inline Int128 date_time_add(const Int128& t, Int64 delta) { return res; } -#define ADD_TIME_FUNCTION_IMPL(CLASS, NAME, UNIT) \ -struct CLASS \ -{ \ - using ReturnType = DataTypeDateTime; \ - static constexpr auto name = #NAME; \ - static inline Int128 execute(const Int128& t, Int64 delta) { \ - return date_time_add(t, delta); \ - } \ -} +#define ADD_TIME_FUNCTION_IMPL(CLASS, NAME, UNIT) \ + struct CLASS { \ + using ReturnType = DataTypeDateTime; \ + static constexpr auto name = #NAME; \ + static inline Int128 execute(const Int128& t, Int64 delta) { \ + return date_time_add(t, delta); \ + } \ + } ADD_TIME_FUNCTION_IMPL(AddSecondsImpl, seconds_add, SECOND); ADD_TIME_FUNCTION_IMPL(AddMinutesImpl, minutes_add, MINUTE); @@ -62,74 +58,80 @@ ADD_TIME_FUNCTION_IMPL(AddWeeksImpl, weeks_add, WEEK); ADD_TIME_FUNCTION_IMPL(AddMonthsImpl, months_add, MONTH); ADD_TIME_FUNCTION_IMPL(AddYearsImpl, years_add, YEAR); -struct AddQuartersImpl -{ +struct AddQuartersImpl { using ReturnType = DataTypeDateTime; static constexpr auto name = "quarters_add"; - static inline Int128 execute(const Int128& t, Int64 delta) - { + static inline Int128 execute(const Int128& t, Int64 delta) { return date_time_add(t, delta * 3); } }; template -struct SubtractIntervalImpl -{ +struct SubtractIntervalImpl { using ReturnType = DataTypeDateTime; - static inline Int128 execute(const Int128& t, Int64 delta) - { + static inline Int128 execute(const Int128& t, Int64 delta) { return Transform::execute(t, -delta); } }; -struct SubtractSecondsImpl : SubtractIntervalImpl { static constexpr auto name = "seconds_sub"; }; -struct SubtractMinutesImpl : SubtractIntervalImpl { static constexpr auto name = "minutes_sub"; }; -struct SubtractHoursImpl : SubtractIntervalImpl { static constexpr auto name = "hours_sub"; }; -struct SubtractDaysImpl : SubtractIntervalImpl { static constexpr auto name = "days_sub"; }; -struct SubtractWeeksImpl : SubtractIntervalImpl { static constexpr auto name = "weeks_sub"; }; -struct SubtractMonthsImpl : SubtractIntervalImpl { static constexpr auto name = "months_sub"; }; -struct SubtractQuartersImpl : SubtractIntervalImpl { static constexpr auto name = "quarters_sub"; }; -struct SubtractYearsImpl : SubtractIntervalImpl { static constexpr auto name = "years_sub"; }; - -struct DateDiffImpl -{ +struct SubtractSecondsImpl : SubtractIntervalImpl { + static constexpr auto name = "seconds_sub"; +}; +struct SubtractMinutesImpl : SubtractIntervalImpl { + static constexpr auto name = "minutes_sub"; +}; +struct SubtractHoursImpl : SubtractIntervalImpl { + static constexpr auto name = "hours_sub"; +}; +struct SubtractDaysImpl : SubtractIntervalImpl { + static constexpr auto name = "days_sub"; +}; +struct SubtractWeeksImpl : SubtractIntervalImpl { + static constexpr auto name = "weeks_sub"; +}; +struct SubtractMonthsImpl : SubtractIntervalImpl { + static constexpr auto name = "months_sub"; +}; +struct SubtractQuartersImpl : SubtractIntervalImpl { + static constexpr auto name = "quarters_sub"; +}; +struct SubtractYearsImpl : SubtractIntervalImpl { + static constexpr auto name = "years_sub"; +}; + +struct DateDiffImpl { using ReturnType = DataTypeInt32; static constexpr auto name = "datediff"; - static inline Int32 execute(const Int128& t0, const Int128& t1) - { + static inline Int32 execute(const Int128& t0, const Int128& t1) { const auto& ts0 = reinterpret_cast(t0); const auto& ts1 = reinterpret_cast(t1); return ts0.daynr() - ts1.daynr(); } }; -struct TimeDiffImpl -{ +struct TimeDiffImpl { using ReturnType = DataTypeFloat64; static constexpr auto name = "timediff"; - static inline double execute(const Int128& t0, const Int128& t1) - { + static inline double execute(const Int128& t0, const Int128& t1) { const auto& ts0 = reinterpret_cast(t0); const auto& ts1 = reinterpret_cast(t1); return ts0.second_diff(ts1); } }; -#define TIME_DIFF_FUNCTION_IMPL(CLASS, NAME, UNIT) \ -struct CLASS \ -{ \ - using ReturnType = DataTypeInt64; \ - static constexpr auto name = #NAME; \ - static inline int64_t execute(const Int128& t0, const Int128& t1) \ - { \ - const auto& ts0 = reinterpret_cast(t0);\ - const auto& ts1 = reinterpret_cast(t1);\ - return DateTimeValue::datetime_diff(ts1, ts0); \ - } \ -} +#define TIME_DIFF_FUNCTION_IMPL(CLASS, NAME, UNIT) \ + struct CLASS { \ + using ReturnType = DataTypeInt64; \ + static constexpr auto name = #NAME; \ + static inline int64_t execute(const Int128& t0, const Int128& t1) { \ + const auto& ts0 = reinterpret_cast(t0); \ + const auto& ts1 = reinterpret_cast(t1); \ + return DateTimeValue::datetime_diff(ts1, ts0); \ + } \ + } TIME_DIFF_FUNCTION_IMPL(YearsDiffImpl, years_diff, YEAR); TIME_DIFF_FUNCTION_IMPL(MonthsDiffImpl, months_diff, MONTH); @@ -139,14 +141,12 @@ TIME_DIFF_FUNCTION_IMPL(HoursDiffImpl, hours_diff, HOUR); TIME_DIFF_FUNCTION_IMPL(MintueSDiffImpl, minutes_diff, MINUTE); TIME_DIFF_FUNCTION_IMPL(SecondsDiffImpl, seconds_diff, SECOND); - template -struct DateTimeOp -{ +struct DateTimeOp { // use for (DateTime, DateTime) -> other_type - static void vector_vector(const PaddedPODArray & vec_from0, const PaddedPODArray & vec_from1, - PaddedPODArray & vec_to) - { + static void vector_vector(const PaddedPODArray& vec_from0, + const PaddedPODArray& vec_from1, + PaddedPODArray& vec_to) { size_t size = vec_from0.size(); vec_to.resize(size); @@ -155,66 +155,58 @@ struct DateTimeOp } // use for (DateTime, const DateTime) -> other_type - static void vector_constant(const PaddedPODArray & vec_from, PaddedPODArray & vec_to, Int128& delta) - { + static void vector_constant(const PaddedPODArray& vec_from, + PaddedPODArray& vec_to, Int128& delta) { size_t size = vec_from.size(); vec_to.resize(size); - for (size_t i = 0; i < size; ++i) - vec_to[i] = Transform::execute(vec_from[i], delta); + for (size_t i = 0; i < size; ++i) vec_to[i] = Transform::execute(vec_from[i], delta); } // use for (DateTime, const ColumnNumber) -> other_type - static void vector_constant(const PaddedPODArray & vec_from, PaddedPODArray & vec_to, Int64 delta) - { + static void vector_constant(const PaddedPODArray& vec_from, + PaddedPODArray& vec_to, Int64 delta) { size_t size = vec_from.size(); vec_to.resize(size); - for (size_t i = 0; i < size; ++i) - vec_to[i] = Transform::execute(vec_from[i], delta); + for (size_t i = 0; i < size; ++i) vec_to[i] = Transform::execute(vec_from[i], delta); } // use for (const DateTime, ColumnNumber) -> other_type - static void constant_vector(const FromType & from, PaddedPODArray & vec_to, const IColumn & delta) - { + static void constant_vector(const FromType& from, PaddedPODArray& vec_to, + const IColumn& delta) { size_t size = delta.size(); vec_to.resize(size); - for (size_t i = 0; i < size; ++i) - vec_to[i] = Transform::execute(from, delta.getInt(i)); + for (size_t i = 0; i < size; ++i) vec_to[i] = Transform::execute(from, delta.getInt(i)); } // use for (const DateTime, DateTime) -> other_type - static void constant_vector(const FromType& from, PaddedPODArray& vec_to, const PaddedPODArray& delta) - { + static void constant_vector(const FromType& from, PaddedPODArray& vec_to, + const PaddedPODArray& delta) { size_t size = delta.size(); vec_to.resize(size); - for (size_t i = 0; i < size; ++i) - vec_to[i] = Transform::execute(from, delta[i]); + for (size_t i = 0; i < size; ++i) vec_to[i] = Transform::execute(from, delta[i]); } }; - template -struct DateTimeAddIntervalImpl -{ - static void execute(Block & block, const ColumnNumbers & arguments, size_t result) - { +struct DateTimeAddIntervalImpl { + static void execute(Block& block, const ColumnNumbers& arguments, size_t result) { using ToType = typename Transform::ReturnType::FieldType; using Op = DateTimeOp; -// const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(block, arguments, 2, 0); + // const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(block, arguments, 2, 0); const ColumnPtr source_col = block.getByPosition(arguments[0]).column; - if (const auto * sources = checkAndGetColumn>(source_col.get())) - { + if (const auto* sources = checkAndGetColumn>(source_col.get())) { auto col_to = ColumnVector::create(); - const IColumn & delta_column = *block.getByPosition(arguments[1]).column; + const IColumn& delta_column = *block.getByPosition(arguments[1]).column; - if (const auto *delta_const_column = typeid_cast(&delta_column)) { + if (const auto* delta_const_column = typeid_cast(&delta_column)) { if (delta_const_column->getField().getType() == Field::Types::Int128) { Op::vector_constant(sources->getData(), col_to->getData(), delta_const_column->getField().get()); @@ -223,113 +215,113 @@ struct DateTimeAddIntervalImpl delta_const_column->getField().get()); } } else { - const auto *delta_vec_column = checkAndGetColumn>(delta_column); - Op::vector_vector(sources->getData(), delta_vec_column->getData(), col_to->getData()); + const auto* delta_vec_column = + checkAndGetColumn>(delta_column); + Op::vector_vector(sources->getData(), delta_vec_column->getData(), + col_to->getData()); } block.getByPosition(result).column = std::move(col_to); - } - else if (const auto * sources_const = checkAndGetColumnConst>(source_col.get())) - { + } else if (const auto* sources_const = + checkAndGetColumnConst>(source_col.get())) { auto col_to = ColumnVector::create(); - if (const auto *delta_vec_column = checkAndGetColumn>(*block.getByPosition(arguments[1]).column)) { - Op::constant_vector(sources_const->template getValue(), col_to->getData(), delta_vec_column->getData()); + if (const auto* delta_vec_column = checkAndGetColumn>( + *block.getByPosition(arguments[1]).column)) { + Op::constant_vector(sources_const->template getValue(), col_to->getData(), + delta_vec_column->getData()); } else { Op::constant_vector(sources_const->template getValue(), col_to->getData(), - *block.getByPosition(arguments[1]).column); + *block.getByPosition(arguments[1]).column); } block.getByPosition(result).column = std::move(col_to); - } - else - { - throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName() - + " of first argument of function " + Transform::name, - ErrorCodes::ILLEGAL_COLUMN); + } else { + throw Exception("Illegal column " + + block.getByPosition(arguments[0]).column->getName() + + " of first argument of function " + Transform::name, + ErrorCodes::ILLEGAL_COLUMN); } } }; - template -class FunctionDateOrDateTimeComputation : public IFunction -{ +class FunctionDateOrDateTimeComputation : public IFunction { public: static constexpr auto name = Transform::name; -// static FunctionPtr create(const Context &) { return std::make_shared(); } + // static FunctionPtr create(const Context &) { return std::make_shared(); } static FunctionPtr create() { return std::make_shared(); } - String getName() const override - { - return name; - } + String getName() const override { return name; } bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } - DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override - { + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName& arguments) const override { if (arguments.size() != 2 && arguments.size() != 3) - throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " - + std::to_string(arguments.size()) + ", should be 2 or 3", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception("Number of arguments for function " + getName() + + " doesn't match: passed " + std::to_string(arguments.size()) + + ", should be 2 or 3", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); -// if (!isNativeNumber(arguments[1].type)) -// throw Exception("Second argument for function " + getName() + " (delta) must be number", -// ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + // if (!isNativeNumber(arguments[1].type)) + // throw Exception("Second argument for function " + getName() + " (delta) must be number", + // ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (arguments.size() == 2) - { + if (arguments.size() == 2) { if (!isDateOrDateTime(arguments[0].type)) - throw Exception{"Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() + - ". Should be a date or a date with time", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - } - else - { - if (!WhichDataType(arguments[0].type).isDateTime() - || !WhichDataType(arguments[2].type).isString()) - throw Exception( - "Function " + getName() + " supports 2 or 3 arguments. The 1st argument " - "must be of type Date or DateTime. The 2nd argument must be number. " - "The 3rd argument (optional) must be " - "a constant string with timezone name. The timezone argument is allowed " - "only when the 1st argument has the type DateTime", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception{"Illegal type " + arguments[0].type->getName() + + " of argument of function " + getName() + + ". Should be a date or a date with time", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + } else { + if (!WhichDataType(arguments[0].type).isDateTime() || + !WhichDataType(arguments[2].type).isString()) + throw Exception("Function " + getName() + + " supports 2 or 3 arguments. The 1st argument " + "must be of type Date or DateTime. The 2nd argument must " + "be number. " + "The 3rd argument (optional) must be " + "a constant string with timezone name. The timezone " + "argument is allowed " + "only when the 1st argument has the type DateTime", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } -// if (WhichDataType(arguments[0].type).isDate()) -// { -// if (std::is_same_v())), UInt16>) -// return std::make_shared(); -// else -// return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 2, 0)); -// } -// else -// { -// if (std::is_same_v())), UInt16>) - return std::make_shared(); -// else -// return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 2, 0)); -// return std::make_shared(); -// } + // if (WhichDataType(arguments[0].type).isDate()) + // { + // if (std::is_same_v())), UInt16>) + // return std::make_shared(); + // else + // return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 2, 0)); + // } + // else + // { + // if (std::is_same_v())), UInt16>) + return std::make_shared(); + // else + // return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 2, 0)); + // return std::make_shared(); + // } } bool useDefaultImplementationForConstants() const override { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {2}; } - Status executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override - { - const IDataType * from_type = block.getByPosition(arguments[0]).type.get(); + Status executeImpl(Block& block, const ColumnNumbers& arguments, size_t result, + size_t /*input_rows_count*/) override { + const IDataType* from_type = block.getByPosition(arguments[0]).type.get(); WhichDataType which(from_type); if (which.isDate()) - DateTimeAddIntervalImpl::execute(block, arguments, result); + DateTimeAddIntervalImpl::execute(block, arguments, + result); else if (which.isDateTime()) - DateTimeAddIntervalImpl::execute(block, arguments, result); + DateTimeAddIntervalImpl::execute( + block, arguments, result); else - throw Exception("Illegal type " + block.getByPosition(arguments[0]).type->getName() + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception("Illegal type " + block.getByPosition(arguments[0]).type->getName() + + " of argument of function " + getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return Status::OK(); } }; -} \ No newline at end of file +} // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/functions/function_date_or_datetime_to_string.cpp b/be/src/vec/functions/function_date_or_datetime_to_string.cpp new file mode 100644 index 00000000000000..293be4985f92e4 --- /dev/null +++ b/be/src/vec/functions/function_date_or_datetime_to_string.cpp @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/functions/function_date_or_datetime_to_string.h" + +#include "vec/functions/simple_function_factory.h" + +namespace doris::vectorized { + +using FunctionDayName = FunctionDateOrDateTimeToString; +using FunctionMonthName = FunctionDateOrDateTimeToString; + +void registerFunctionDateTimeToString(SimpleFunctionFactory& factory) { + factory.registerFunction(); + factory.registerFunction(); +} + +} // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/functions/function_date_or_datetime_to_string.h b/be/src/vec/functions/function_date_or_datetime_to_string.h new file mode 100644 index 00000000000000..6623f2ccbf92a2 --- /dev/null +++ b/be/src/vec/functions/function_date_or_datetime_to_string.h @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/data_types/data_type_date.h" +#include "vec/data_types/data_type_date_time.h" +#include "vec/data_types/data_type_string.h" +#include "vec/functions/date_time_transforms.h" +#include "vec/functions/function.h" + +namespace doris::vectorized { + +namespace ErrorCodes { +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} // namespace ErrorCodes + +template +class FunctionDateOrDateTimeToString : public IFunction { +public: + static constexpr auto name = Transform::name; + static FunctionPtr create() { return std::make_shared(); } + + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 1; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName& arguments) const override { + return std::make_shared(); + } + + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + + Status executeImpl(Block& block, const ColumnNumbers& arguments, size_t result, + size_t input_rows_count) override { + const ColumnPtr source_col = block.getByPosition(arguments[0]).column; + const auto* sources = checkAndGetColumn>(source_col.get()); + auto col_res = ColumnString::create(); + if (sources) { + TransformerToStringOneArgument::vector( + sources->getData(), col_res->getChars(), col_res->getOffsets()); + block.getByPosition(result).column = std::move(col_res); + } else { + return Status::InternalError("Illegal column " + + block.getByPosition(arguments[0]).column->getName() + + " of first argument of function " + name); + } + return Status::OK(); + } +}; + +} // namespace doris::vectorized diff --git a/be/src/vec/functions/function_datetime_string_to_string.cpp b/be/src/vec/functions/function_datetime_string_to_string.cpp new file mode 100644 index 00000000000000..1a66c7b775f103 --- /dev/null +++ b/be/src/vec/functions/function_datetime_string_to_string.cpp @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/functions/function_datetime_string_to_string.h" + +#include "vec/functions/simple_function_factory.h" + +namespace doris::vectorized { + +using FunctionDateFormat = FunctionDateTimeStringToString; +using FunctionFromUnixTime = FunctionDateTimeStringToString; + +void registerFunctionDateTimeStringToString(SimpleFunctionFactory& factory) { + factory.registerFunction(); + factory.registerFunction(); +} + +} // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/functions/function_datetime_string_to_string.h b/be/src/vec/functions/function_datetime_string_to_string.h new file mode 100644 index 00000000000000..d9b1ff06224670 --- /dev/null +++ b/be/src/vec/functions/function_datetime_string_to_string.h @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "vec/columns/columns_number.h" +#include "vec/data_types/data_type_date.h" +#include "vec/data_types/data_type_date_time.h" +#include "vec/data_types/data_type_nullable.h" +#include "vec/data_types/data_type_string.h" +#include "vec/functions/date_time_transforms.h" +#include "vec/functions/function.h" + +namespace doris::vectorized { + +namespace ErrorCodes { +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} // namespace ErrorCodes + +template +class FunctionDateTimeStringToString : public IFunction { +public: + static constexpr auto name = Transform::name; + static FunctionPtr create() { return std::make_shared(); } + + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 0; } + bool isVariadic() const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName& arguments) const override { + return makeNullable(std::make_shared()); + } + + bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + + Status executeImpl(Block& block, const ColumnNumbers& arguments, size_t result, + size_t input_rows_count) override { + const ColumnPtr source_col = block.getByPosition(arguments[0]).column; + + const auto* nullable_column = checkAndGetColumn(source_col.get()); + const auto* sources = checkAndGetColumn>( + nullable_column ? nullable_column->getNestedColumnPtr().get() : source_col.get()); + + if (sources) { + auto col_res = ColumnString::create(); + ColumnUInt8::MutablePtr col_null_map_to; + col_null_map_to = ColumnUInt8::create(sources->size()); + auto& vec_null_map_to = col_null_map_to->getData(); + + if (arguments.size() == 2) { + const IColumn& source_col1 = *block.getByPosition(arguments[1]).column; + if (const auto* delta_const_column = + typeid_cast(&source_col1)) { + TransformerToStringTwoArgument::vector_constant( + sources->getData(), delta_const_column->getField().get(), + col_res->getChars(), col_res->getOffsets(), vec_null_map_to); + } else { + return Status::InternalError( + "Illegal column " + + block.getByPosition(arguments[1]).column->getName() + " is not const" + + name); + } + } else { + TransformerToStringTwoArgument::vector_constant( + sources->getData(), "yyyy-MM-dd HH:mm:ss", col_res->getChars(), + col_res->getOffsets(), vec_null_map_to); + } + + if (nullable_column) { + const auto& origin_null_map = nullable_column->getNullMapColumn().getData(); + for (int i = 0; i < origin_null_map.size(); ++i) { + vec_null_map_to[i] |= origin_null_map[i]; + } + } + block.getByPosition(result).column = + ColumnNullable::create(std::move(col_res), std::move(col_null_map_to)); + } else { + return Status::InternalError("Illegal column " + + block.getByPosition(arguments[0]).column->getName() + + " of first argument of function " + name); + } + return Status::OK(); + } +}; + +} // namespace doris::vectorized diff --git a/be/src/vec/functions/simple_function_factory.h b/be/src/vec/functions/simple_function_factory.h index 5c52c626c74e69..28e508fe401c1d 100644 --- a/be/src/vec/functions/simple_function_factory.h +++ b/be/src/vec/functions/simple_function_factory.h @@ -41,9 +41,11 @@ void registerFunctionModulo(SimpleFunctionFactory& factory); void registerFunctionBitmap(SimpleFunctionFactory& factory); void registerFunctionIsNull(SimpleFunctionFactory& factory); void registerFunctionIsNotNull(SimpleFunctionFactory& factory); -void registerFunctionToTimeFuction(SimpleFunctionFactory & factory); -void registerFunctionTimeOfFuction(SimpleFunctionFactory & factory); +void registerFunctionToTimeFuction(SimpleFunctionFactory& factory); +void registerFunctionTimeOfFuction(SimpleFunctionFactory& factory); void registerFunctionString(SimpleFunctionFactory& factory); +void registerFunctionDateTimeToString(SimpleFunctionFactory& factory); +void registerFunctionDateTimeStringToString(SimpleFunctionFactory& factory); void registerFunctionIn(SimpleFunctionFactory& factory); void registerFunctionIf(SimpleFunctionFactory& factory); void registerFunctionDateTimeComputation(SimpleFunctionFactory& factory); @@ -63,7 +65,7 @@ class SimpleFunctionFactory { registerFunction(Function::name, &Function::create); } - void registerAlias(const std::string &name, const std::string &alias) { + void registerAlias(const std::string& name, const std::string& alias) { function_creators[alias] = function_creators[name]; } @@ -110,6 +112,8 @@ class SimpleFunctionFactory { registerFunctionIn(instance); registerFunctionIf(instance); registerFunctionDateTimeComputation(instance); + registerFunctionDateTimeToString(instance); + registerFunctionDateTimeStringToString(instance); }); return instance; } diff --git a/be/src/vec/functions/to_time_function.cpp b/be/src/vec/functions/to_time_function.cpp index ea045827105b89..e879c0c7da5938 100644 --- a/be/src/vec/functions/to_time_function.cpp +++ b/be/src/vec/functions/to_time_function.cpp @@ -31,9 +31,12 @@ using FunctionHour = FunctionDateOrDateTimeToSomething; using FunctionSecond = FunctionDateOrDateTimeToSomething; using FunctionToDays = FunctionDateOrDateTimeToSomething; +using FunctionToDate = FunctionDateOrDateTimeToSomething; +using FunctionDate = FunctionDateOrDateTimeToSomething; +using FunctionTimeStamp = FunctionDateOrDateTimeToSomething; +using FunctionUnixTimeStamp = FunctionDateOrDateTimeToSomething; -void registerFunctionToTimeFuction(SimpleFunctionFactory & factory) -{ +void registerFunctionToTimeFuction(SimpleFunctionFactory& factory) { factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); @@ -42,6 +45,10 @@ void registerFunctionToTimeFuction(SimpleFunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); } -} +} // namespace doris::vectorized diff --git a/build.sh b/build.sh index e7f8c53c87e501..5cec4b763cc6ab 100755 --- a/build.sh +++ b/build.sh @@ -85,7 +85,7 @@ fi eval set -- "$OPTS" -PARALLEL=$[$(nproc)/4+1] +PARALLEL=$[$(nproc)+1] BUILD_BE= BUILD_FE= BUILD_UI= diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index e04f7aed12a5dc..79c5d51ee813fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -1017,7 +1017,7 @@ public void addScalarBuiltin(String fnName, String symbol, boolean userVisible, symbol, prepareFnSymbol, closeFnSymbol, userVisible)); } - private void addScalarAndVectorizedBuiltin(String fnName, String symbol, boolean userVisible, + public void addScalarAndVectorizedBuiltin(String fnName, String symbol, boolean userVisible, String prepareFnSymbol, String closeFnSymbol, boolean varArgs, PrimitiveType retType, PrimitiveType ... args) { ArrayList argsType = new ArrayList(); diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 834e419c68858b..82c57210a5beb8 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -103,20 +103,20 @@ [['unix_timestamp'], 'INT', [], '_ZN5doris18TimestampFunctions7to_unixEPN9doris_udf15FunctionContextE'], [['unix_timestamp'], 'INT', ['DATETIME'], - '_ZN5doris18TimestampFunctions7to_unixEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions7to_unixEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', 'vec'], [['unix_timestamp'], 'INT', ['DATE'], - '_ZN5doris18TimestampFunctions7to_unixEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions7to_unixEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', 'vec'], [['unix_timestamp'], 'INT', ['VARCHAR', 'VARCHAR'], '_ZN5doris18TimestampFunctions7to_unixEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], [['from_unixtime'], 'VARCHAR', ['INT'], - '_ZN5doris18TimestampFunctions9from_unixEPN9doris_udf15FunctionContextERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions9from_unixEPN9doris_udf15FunctionContextERKNS1_6IntValE', 'vec'], [['from_unixtime'], 'VARCHAR', ['INT', 'VARCHAR'], '_ZN5doris18TimestampFunctions9from_unixEPN9doris_udf' '15FunctionContextERKNS1_6IntValERKNS1_9StringValE', '_ZN5doris18TimestampFunctions14format_prepareEPN9doris_udf' '15FunctionContextENS2_18FunctionStateScopeE', '_ZN5doris18TimestampFunctions12format_closeEPN9doris_udf' - '15FunctionContextENS2_18FunctionStateScopeE'], + '15FunctionContextENS2_18FunctionStateScopeE', 'vec'], [['now', 'current_timestamp', 'localtime', 'localtimestamp'], 'DATETIME', [], '_ZN5doris18TimestampFunctions3nowEPN9doris_udf15FunctionContextE'], [['curtime', 'current_time'], 'TIME', [], @@ -126,7 +126,7 @@ [['utc_timestamp'], 'DATETIME', [], '_ZN5doris18TimestampFunctions13utc_timestampEPN9doris_udf15FunctionContextE'], [['timestamp'], 'DATETIME', ['DATETIME'], - '_ZN5doris18TimestampFunctions9timestampEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions9timestampEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', 'vec'], [['from_days'], 'DATE', ['INT'], '_ZN5doris18TimestampFunctions9from_daysEPN9doris_udf15FunctionContextERKNS1_6IntValE'], @@ -232,23 +232,23 @@ '_ZN5doris18TimestampFunctions14format_prepareEPN9doris_udf' '15FunctionContextENS2_18FunctionStateScopeE', '_ZN5doris18TimestampFunctions12format_closeEPN9doris_udf' - '15FunctionContextENS2_18FunctionStateScopeE'], + '15FunctionContextENS2_18FunctionStateScopeE', 'vec'], [['date_format'], 'VARCHAR', ['DATE', 'VARCHAR'], '_ZN5doris18TimestampFunctions11date_formatEPN9doris_udf' '15FunctionContextERKNS1_11DateTimeValERKNS1_9StringValE', '_ZN5doris18TimestampFunctions14format_prepareEPN9doris_udf' '15FunctionContextENS2_18FunctionStateScopeE', '_ZN5doris18TimestampFunctions12format_closeEPN9doris_udf' - '15FunctionContextENS2_18FunctionStateScopeE'], + '15FunctionContextENS2_18FunctionStateScopeE', 'vec'], [['date', 'to_date'], 'DATE', ['DATETIME'], - '_ZN5doris18TimestampFunctions7to_dateEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions7to_dateEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', 'vec'], [['dayname'], 'VARCHAR', ['DATETIME'], '_ZN5doris18TimestampFunctions8day_nameEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValE'], + '15FunctionContextERKNS1_11DateTimeValE', 'vec'], [['monthname'], 'VARCHAR', ['DATETIME'], '_ZN5doris18TimestampFunctions10month_nameEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValE'], + '15FunctionContextERKNS1_11DateTimeValE', 'vec'], [['convert_tz'], 'DATETIME', ['DATETIME', 'VARCHAR', 'VARCHAR'], '_ZN5doris18TimestampFunctions10convert_tzEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_9StringValES9_', diff --git a/gensrc/script/gen_builtins_functions.py b/gensrc/script/gen_builtins_functions.py index d154ab6cfa9810..221fb206aa0851 100755 --- a/gensrc/script/gen_builtins_functions.py +++ b/gensrc/script/gen_builtins_functions.py @@ -59,7 +59,7 @@ def add_function(fn_meta_data, user_visible): """add function """ - assert 4 <= len(fn_meta_data) <= 6, \ + assert 4 <= len(fn_meta_data) <= 7, \ "Invalid function entry in doris_builtins_functions.py:\n\t" + repr(fn_meta_data) entry = {} entry["sql_names"] = fn_meta_data[0] @@ -67,12 +67,14 @@ def add_function(fn_meta_data, user_visible): entry["args"] = fn_meta_data[2] entry["symbol"] = fn_meta_data[3] if len(fn_meta_data) >= 5: - if fn_meta_data[4] != "vec": - entry["prepare"] = fn_meta_data[4] - else: - entry["vec"] = True + if fn_meta_data[4] != "vec": + entry["prepare"] = fn_meta_data[4] if len(fn_meta_data) >= 6: entry["close"] = fn_meta_data[5] + + if fn_meta_data[-1] == 'vec': + entry['vec'] = True + entry["user_visible"] = user_visible meta_data_entries.append(entry)