diff --git a/be/src/vec/functions/function_date_or_datetime_computation.cpp b/be/src/vec/functions/function_date_or_datetime_computation.cpp index ff081c270f5a12..cb58eef458fdbf 100644 --- a/be/src/vec/functions/function_date_or_datetime_computation.cpp +++ b/be/src/vec/functions/function_date_or_datetime_computation.cpp @@ -115,10 +115,10 @@ using FunctionDatetimeSubQuarters = using FunctionDatetimeSubYears = FunctionDateOrDateTimeComputation>; -using FunctionAddTimeDatetime = FunctionAddTime; -using FunctionAddTimeTime = FunctionAddTime; -using FunctionSubTimeDatetime = FunctionAddTime; -using FunctionSubTimeTime = FunctionAddTime; +using FunctionAddTimeDatetime = FunctionNeedsToHandleNull; +using FunctionAddTimeTime = FunctionNeedsToHandleNull; +using FunctionSubTimeDatetime = FunctionNeedsToHandleNull; +using FunctionSubTimeTime = FunctionNeedsToHandleNull; #define FUNCTION_TIME_DIFF(NAME, IMPL, TYPE) using NAME##_##TYPE = FunctionTimeDiff>; diff --git a/be/src/vec/functions/function_date_or_datetime_computation.h b/be/src/vec/functions/function_date_or_datetime_computation.h index 6efa38a24b5566..c6b3b5a989980b 100644 --- a/be/src/vec/functions/function_date_or_datetime_computation.h +++ b/be/src/vec/functions/function_date_or_datetime_computation.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -1679,118 +1680,71 @@ class PeriodDiffImpl { } }; -struct AddTimeImpl { - static constexpr auto name = "add_time"; - static bool is_negative() { return false; } -}; - -struct SubTimeImpl { - static constexpr auto name = "sub_time"; - static bool is_negative() { return true; } -}; - -template -class FunctionAddTime : public IFunction { +template +class AddTimeImplBase { public: - static constexpr auto name = Impl::name; - static constexpr PrimitiveType ReturnType = PType; - static constexpr PrimitiveType ArgType1 = PType; - static constexpr PrimitiveType ArgType2 = TYPE_TIMEV2; - using ColumnType1 = typename PrimitiveTypeTraits::ColumnType; - using ColumnType2 = typename PrimitiveTypeTraits::ColumnType; + static constexpr auto name = IsNegative ? "sub_time" : "add_time"; using InputType1 = typename PrimitiveTypeTraits::DataType::FieldType; using InputType2 = typename PrimitiveTypeTraits::DataType::FieldType; - using ReturnNativeType = InputType1; - using ReturnDataType = typename PrimitiveTypeTraits::DataType; + using ResultColumnType = typename PrimitiveTypeTraits::ColumnType; - String get_name() const override { return name; } - size_t get_number_of_arguments() const override { return 2; } - DataTypes get_variadic_argument_types_impl() const override { + static size_t get_number_of_arguments() { return 2; } + static bool is_variadic() { return true; } + static DataTypes get_variadic_argument_types_impl() { return {std::make_shared::DataType>(), std::make_shared::DataType>()}; } - DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override { - return std::make_shared(); - } - - ReturnNativeType compute(const InputType1& arg1, const InputType2& arg2) const { - if constexpr (PType == TYPE_DATETIMEV2) { - DateV2Value dtv1 = - binary_cast>(arg1); - auto tv2 = static_cast(arg2); - TimeInterval interval(TimeUnit::MICROSECOND, tv2, Impl::is_negative()); - bool out_range = dtv1.template date_add_interval(interval); - if (UNLIKELY(!out_range)) { - throw Exception(ErrorCode::INVALID_ARGUMENT, - "datetime value is out of range in function {}", name); - } - return binary_cast, ReturnNativeType>(dtv1); - } else if constexpr (PType == TYPE_TIMEV2) { - auto tv1 = static_cast(arg1); - auto tv2 = static_cast(arg2); - double res = TimeValue::limit_with_bound(Impl::is_negative() ? tv1 - tv2 : tv1 + tv2); - return res; - } else { - throw Exception(ErrorCode::FATAL_ERROR, "not support type for function {}", name); + static DataTypePtr get_return_type_impl(const DataTypes& arguments) { + if (arguments[0]->is_nullable() || arguments[1]->is_nullable()) { + return make_nullable(std::make_shared::DataType>()); } + return std::make_shared::DataType>(); } - static FunctionPtr create() { return std::make_shared(); } - - Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, - uint32_t result, size_t input_rows_count) const override { - DCHECK_EQ(arguments.size(), 2); - const auto& [left_col, left_const] = - unpack_if_const(block.get_by_position(arguments[0]).column); - const auto& [right_col, right_const] = - unpack_if_const(block.get_by_position(arguments[1]).column); - ColumnPtr nest_col1 = remove_nullable(left_col); - ColumnPtr nest_col2 = remove_nullable(right_col); - auto res = ColumnVector::create(input_rows_count, 0); - - if (left_const) { - execute_constant_vector(assert_cast(*nest_col1).get_element(0), - assert_cast(*nest_col2).get_data(), - res->get_data(), input_rows_count); - } else if (right_const) { - execute_vector_constant(assert_cast(*nest_col1).get_data(), - assert_cast(*nest_col2).get_element(0), - res->get_data(), input_rows_count); - } else { - execute_vector_vector(assert_cast(*nest_col1).get_data(), - assert_cast(*nest_col2).get_data(), - res->get_data(), input_rows_count); - } - - block.replace_by_position(result, std::move(res)); - return Status::OK(); - } - void execute_vector_vector(const PaddedPODArray& left_col, - const PaddedPODArray& right_col, - PaddedPODArray& res_data, - size_t input_rows_count) const { - for (size_t i = 0; i < input_rows_count; ++i) { - res_data[i] = compute(left_col[i], right_col[i]); - } - } + static void execute(const std::vector& cols_info, + typename ResultColumnType::MutablePtr& res_col, + PaddedPODArray& res_null_map_data, size_t input_rows_count) { + const auto& left_data = + assert_cast(cols_info[0].nested_col)->get_data(); + const auto& right_data = + assert_cast*>(cols_info[1].nested_col)->get_data(); - void execute_vector_constant(const PaddedPODArray& left_col, - const InputType2 right_value, - PaddedPODArray& res_data, - size_t input_rows_count) const { for (size_t i = 0; i < input_rows_count; ++i) { - res_data[i] = compute(left_col[i], right_value); - } - } + if (cols_info[0].is_null_at(i) || cols_info[1].is_null_at(i)) { + res_col->insert_default(); + res_null_map_data[i] = 1; + continue; + } - void execute_constant_vector(const InputType1 left_value, - const PaddedPODArray& right_col, - PaddedPODArray& res_data, - size_t input_rows_count) const { - for (size_t i = 0; i < input_rows_count; ++i) { - res_data[i] = compute(left_value, right_col[i]); + const auto& arg1 = left_data[index_check_const(i, cols_info[0].is_const)]; + const auto& arg2 = right_data[index_check_const(i, cols_info[1].is_const)]; + + if constexpr (PType == TYPE_DATETIMEV2) { + DateV2Value dtv1 = + binary_cast>(arg1); + auto tv2 = static_cast(arg2); + TimeInterval interval(TimeUnit::MICROSECOND, tv2, IsNegative); + bool out_range = dtv1.template date_add_interval(interval); + if (!out_range) [[unlikely]] { + throw_invalid_strings(name, dtv1.to_string(), std::to_string(arg2)); + } + res_col->insert_value( + binary_cast, InputType1>(dtv1)); + } else if constexpr (PType == TYPE_TIMEV2) { + auto tv1 = static_cast(arg1); + auto tv2 = static_cast(arg2); + double res = TimeValue::limit_with_bound(IsNegative ? tv1 - tv2 : tv1 + tv2); + res_col->insert_value(res); + } else { + throw Exception(ErrorCode::FATAL_ERROR, "not support type for function {}", name); + } } } }; + +using AddTimeDatetimeImpl = AddTimeImplBase; +using AddTimeTimeImpl = AddTimeImplBase; +using SubTimeDatetimeImpl = AddTimeImplBase; +using SubTimeTimeImpl = AddTimeImplBase; #include "common/compile_check_avoid_end.h" } // namespace doris::vectorized diff --git a/be/src/vec/functions/function_date_or_datetime_to_string.cpp b/be/src/vec/functions/function_date_or_datetime_to_string.cpp index 6b068ccf159fd2..8e38aaf0b7f5a2 100644 --- a/be/src/vec/functions/function_date_or_datetime_to_string.cpp +++ b/be/src/vec/functions/function_date_or_datetime_to_string.cpp @@ -77,8 +77,6 @@ class FunctionDateOrDateTimeToString : public IFunction { return {}; } - ColumnNumbers get_arguments_that_are_always_constant() const override { return {1}; } - // In ICU, Week_array: {"", "Sunday", "Monday", ..., "Saturday"}, size = 8 // Month_array: {"January", "February", ..., "December"}, size = 12 static constexpr size_t DAY_NUM_IN_ICU = 8; @@ -140,28 +138,48 @@ class FunctionDateOrDateTimeToString : public IFunction { return IFunction::open(context, scope); } + bool use_default_implementation_for_nulls() const override { return false; } + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, uint32_t result, size_t input_rows_count) const override { const ColumnPtr source_col = block.get_by_position(arguments[0]).column; + const NullMap* null_map = nullptr; + ColumnPtr actual_col = source_col; + + if (is_column_nullable(*source_col)) { + const auto* nullable_col = check_and_get_column(source_col.get()); + actual_col = nullable_col->get_nested_column_ptr(); + null_map = &nullable_col->get_null_map_data(); + } + const auto* sources = - check_and_get_column>(source_col.get()); - auto col_res = ColumnString::create(); + check_and_get_column>(actual_col.get()); + if (!sources) [[unlikely]] { + return Status::FatalError("Illegal column {} of first argument of function {}", + block.get_by_position(arguments[0]).column->get_name(), name); + } - // Support all input of datetime is valind to make sure not null return - if (sources) { - vector(context, sources->get_data(), col_res->get_chars(), col_res->get_offsets()); - block.replace_by_position(result, std::move(col_res)); + auto col_res = ColumnString::create(); + vector(context, sources->get_data(), col_res->get_chars(), col_res->get_offsets(), + null_map); + + if (null_map) { + const auto* nullable_col = check_and_get_column(source_col.get()); + block.replace_by_position( + result, + ColumnNullable::create(std::move(col_res), + nullable_col->get_null_map_column_ptr()->clone_resized( + input_rows_count))); } else { - return Status::InternalError("Illegal column {} of first argument of function {}", - block.get_by_position(arguments[0]).column->get_name(), - name); + block.replace_by_position(result, std::move(col_res)); } return Status::OK(); } private: static void vector(FunctionContext* context, const PaddedPODArray& ts, - ColumnString::Chars& res_data, ColumnString::Offsets& res_offsets) { + ColumnString::Chars& res_data, ColumnString::Offsets& res_offsets, + const NullMap* null_map = nullptr) { const auto len = ts.size(); res_data.resize(len * Transform::max_size); res_offsets.resize(len); @@ -176,7 +194,12 @@ class FunctionDateOrDateTimeToString : public IFunction { names_ptr = state->month_names; } - for (int i = 0; i < len; ++i) { + for (size_t i = 0; i < len; ++i) { + if (null_map && (*null_map)[i]) { + res_offsets[i] = cast_set(offset); + continue; + } + const auto& t = ts[i]; const auto date_time_value = binary_cast(t); res_offsets[i] = cast_set( diff --git a/be/src/vec/functions/function_needs_to_handle_null.h b/be/src/vec/functions/function_needs_to_handle_null.h index 0e60da92e68981..5ac6584924d6fb 100644 --- a/be/src/vec/functions/function_needs_to_handle_null.h +++ b/be/src/vec/functions/function_needs_to_handle_null.h @@ -53,6 +53,12 @@ class FunctionNeedsToHandleNull : public IFunction { } return false; } + DataTypes get_variadic_argument_types_impl() const override { + if constexpr (requires { Impl::get_variadic_argument_types_impl(); }) { + return Impl::get_variadic_argument_types_impl(); + } + return {}; + } bool use_default_implementation_for_nulls() const override { return false; } diff --git a/regression-test/data/nereids_function_p0/scalar_function/Map.out b/regression-test/data/nereids_function_p0/scalar_function/nereids_scalar_fn_map.out similarity index 99% rename from regression-test/data/nereids_function_p0/scalar_function/Map.out rename to regression-test/data/nereids_function_p0/scalar_function/nereids_scalar_fn_map.out index c3eb0f6c071d3f..b5e55783718a81 100644 --- a/regression-test/data/nereids_function_p0/scalar_function/Map.out +++ b/regression-test/data/nereids_function_p0/scalar_function/nereids_scalar_fn_map.out @@ -3722,3 +3722,11 @@ false -- !sql -- {"{"zip":10001}":"{"city":"NY"}", "{"code":10002}":"{"state":"NY"}"} +-- !element_at_tint_date_dayname -- +\N +Sunday + +-- !element_at_tint_date_dayname_notnull -- +\N +Saturday + diff --git a/regression-test/data/nereids_p0/join/test_outer_join.out b/regression-test/data/nereids_p0/join/test_outer_join.out index 9ae6731b34780e..60c7db8f72914e 100644 --- a/regression-test/data/nereids_p0/join/test_outer_join.out +++ b/regression-test/data/nereids_p0/join/test_outer_join.out @@ -10,3 +10,8 @@ -- !join -- +-- !join -- +\N \N +1 2023-12-18T23:30:01 +2 2023-12-15T23:30:01 + diff --git a/regression-test/suites/nereids_function_p0/scalar_function/Map.groovy b/regression-test/suites/nereids_function_p0/scalar_function/nereids_scalar_fn_map.groovy similarity index 97% rename from regression-test/suites/nereids_function_p0/scalar_function/Map.groovy rename to regression-test/suites/nereids_function_p0/scalar_function/nereids_scalar_fn_map.groovy index 7c686a08baa668..343b298a8728cd 100644 --- a/regression-test/suites/nereids_function_p0/scalar_function/Map.groovy +++ b/regression-test/suites/nereids_function_p0/scalar_function/nereids_scalar_fn_map.groovy @@ -35,6 +35,7 @@ suite("nereids_scalar_fn_map") { order_qt_element_at_str_tint """ select km_str_tint[kstr] from fn_test """ order_qt_element_at_date_tint """ select km_date_tint[kdt] from fn_test """ order_qt_element_at_dtm_tint """ select km_dtm_tint[kdtm] from fn_test """ + order_qt_element_at_bool_tint_notnull """ select km_bool_tint[kbool] from fn_test_not_nullable """ order_qt_element_at_tint_tint_notnull """ select km_tint_tint[ktint] from fn_test_not_nullable """ order_qt_element_at_sint_tint_notnull """ select km_sint_tint[ksint] from fn_test_not_nullable """ @@ -314,4 +315,24 @@ suite("nereids_scalar_fn_map") { qt_sql "select map('postal_code', 10001, 'area_code', 10002, 'zip_plus_4', 10003)" qt_sql "select map('{\"zip\":10001}', '{\"city\":\"NY\"}', '{\"code\":10002}', '{\"state\":\"NY\"}')" + sql """ drop table if exists fn_test_element_at_map_date """ + sql """ + create table fn_test_element_at_map_date ( + id int null, + m map null + ) engine=olap + distributed by hash(id) buckets 1 + properties('replication_num' = '1') + """ + sql """ + insert into fn_test_element_at_map_date values + (1, map(1,null, 2,'2023-12-16')), + (2, map(1,'2023-01-01', 2,null)); + """ + order_qt_element_at_tint_date_dayname """ + select dayname(element_at(m, 1)) from fn_test_element_at_map_date + """ + order_qt_element_at_tint_date_dayname_notnull """ + select dayname(element_at(m, 2)) from fn_test_element_at_map_date + """ } \ No newline at end of file diff --git a/regression-test/suites/nereids_p0/join/test_outer_join.groovy b/regression-test/suites/nereids_p0/join/test_outer_join.groovy index 9264f3fc57959d..f97c580d84c850 100644 --- a/regression-test/suites/nereids_p0/join/test_outer_join.groovy +++ b/regression-test/suites/nereids_p0/join/test_outer_join.groovy @@ -84,7 +84,30 @@ suite("test_outer_join", "nereids_p0") { contains "FULL OUTER JOIN(PARTITIONED)" } - sql "DROP TABLE IF EXISTS ${tbl1}" - sql "DROP TABLE IF EXISTS ${tbl2}" - sql "DROP TABLE IF EXISTS ${tbl3}" + def tbl4 = "test_outer_join4" + def tbl5 = "test_outer_join5" + + sql "DROP TABLE IF EXISTS test_outer_join4" + sql """ + CREATE TABLE test_outer_join4 ( + k INT, + d DATEV2 + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES ("replication_num" = "1"); + """ + sql "DROP TABLE IF EXISTS test_outer_join5" + sql """ + CREATE TABLE test_outer_join5 ( + k INT + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES ("replication_num" = "1"); + """ + sql """INSERT INTO test_outer_join4 VALUES (1, '2023-12-18'), (2, '2023-12-15');""" + sql """INSERT INTO test_outer_join5 VALUES (1), (2), (3);""" + + order_qt_join """ + SELECT k, add_time(d, '23:30:01') FROM test_outer_join4 RIGHT JOIN test_outer_join5 USING (k) ORDER BY k; + """ } diff --git a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy index d96a6f7c770dd0..977acfc36a23e6 100644 --- a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy +++ b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy @@ -175,7 +175,7 @@ suite("test_date_function_v2") { test{ sql("select add_time('9999-12-29 00:00:00', '122:35:22.123456');") - exception "datetime value is out of range in function add_time" + exception "Operation add_time of" } qt_sql_subtime1("select sub_time('2023-10-14 00:00:00', '22:35:22');") @@ -193,7 +193,7 @@ suite("test_date_function_v2") { test{ sql("select sub_time('0000-01-01 00:00:00', '122:35:22.123456');") - exception "datetime value is out of range in function sub_time" + exception "Operation sub_time of" } //test computetimearithmetic regular