branch-4.0: [Fix](function) fix skip null value for some date functions #59616

be/src/vec/functions/function_date_or_datetime_computation.cpp

@@ -115,10 +115,10 @@ using FunctionDatetimeSubQuarters =
 using FunctionDatetimeSubYears =
         FunctionDateOrDateTimeComputation<SubtractYearsImpl<TYPE_DATETIMEV2>>;
 
-using FunctionAddTimeDatetime = FunctionAddTime<TYPE_DATETIMEV2, AddTimeImpl>;
-using FunctionAddTimeTime = FunctionAddTime<TYPE_TIMEV2, AddTimeImpl>;
-using FunctionSubTimeDatetime = FunctionAddTime<TYPE_DATETIMEV2, SubTimeImpl>;
-using FunctionSubTimeTime = FunctionAddTime<TYPE_TIMEV2, SubTimeImpl>;
+using FunctionAddTimeDatetime = FunctionNeedsToHandleNull<AddTimeDatetimeImpl, TYPE_DATETIMEV2>;
+using FunctionAddTimeTime = FunctionNeedsToHandleNull<AddTimeTimeImpl, TYPE_TIMEV2>;
+using FunctionSubTimeDatetime = FunctionNeedsToHandleNull<SubTimeDatetimeImpl, TYPE_DATETIMEV2>;
+using FunctionSubTimeTime = FunctionNeedsToHandleNull<SubTimeTimeImpl, TYPE_TIMEV2>;
 
 #define FUNCTION_TIME_DIFF(NAME, IMPL, TYPE) using NAME##_##TYPE = FunctionTimeDiff<IMPL<TYPE>>;
 

be/src/vec/functions/function_date_or_datetime_computation.h

@@ -22,6 +22,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <memory>
+#include <string>
 #include <string_view>
 #include <type_traits>
 #include <utility>
@@ -1679,118 +1680,71 @@ class PeriodDiffImpl {
     }
 };
 
-struct AddTimeImpl {
-    static constexpr auto name = "add_time";
-    static bool is_negative() { return false; }
-};
-
-struct SubTimeImpl {
-    static constexpr auto name = "sub_time";
-    static bool is_negative() { return true; }
-};
-
-template <PrimitiveType PType, typename Impl>
-class FunctionAddTime : public IFunction {
+template <PrimitiveType PType, bool IsNegative>
+class AddTimeImplBase {
 public:
-    static constexpr auto name = Impl::name;
-    static constexpr PrimitiveType ReturnType = PType;
-    static constexpr PrimitiveType ArgType1 = PType;
-    static constexpr PrimitiveType ArgType2 = TYPE_TIMEV2;
-    using ColumnType1 = typename PrimitiveTypeTraits<PType>::ColumnType;
-    using ColumnType2 = typename PrimitiveTypeTraits<TYPE_TIMEV2>::ColumnType;
+    static constexpr auto name = IsNegative ? "sub_time" : "add_time";
     using InputType1 = typename PrimitiveTypeTraits<PType>::DataType::FieldType;
     using InputType2 = typename PrimitiveTypeTraits<TYPE_TIMEV2>::DataType::FieldType;
-    using ReturnNativeType = InputType1;
-    using ReturnDataType = typename PrimitiveTypeTraits<PType>::DataType;
+    using ResultColumnType = typename PrimitiveTypeTraits<PType>::ColumnType;
 
-    String get_name() const override { return name; }
-    size_t get_number_of_arguments() const override { return 2; }
-    DataTypes get_variadic_argument_types_impl() const override {
+    static size_t get_number_of_arguments() { return 2; }
+    static bool is_variadic() { return true; }
+    static DataTypes get_variadic_argument_types_impl() {
         return {std::make_shared<typename PrimitiveTypeTraits<PType>::DataType>(),
                 std::make_shared<typename PrimitiveTypeTraits<TYPE_TIMEV2>::DataType>()};
     }
-    DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override {
-        return std::make_shared<ReturnDataType>();
-    }
-
-    ReturnNativeType compute(const InputType1& arg1, const InputType2& arg2) const {
-        if constexpr (PType == TYPE_DATETIMEV2) {
-            DateV2Value<DateTimeV2ValueType> dtv1 =
-                    binary_cast<InputType1, DateV2Value<DateTimeV2ValueType>>(arg1);
-            auto tv2 = static_cast<TimeValue::TimeType>(arg2);
-            TimeInterval interval(TimeUnit::MICROSECOND, tv2, Impl::is_negative());
-            bool out_range = dtv1.template date_add_interval<TimeUnit::MICROSECOND>(interval);
-            if (UNLIKELY(!out_range)) {
-                throw Exception(ErrorCode::INVALID_ARGUMENT,
-                                "datetime value is out of range in function {}", name);
-            }
-            return binary_cast<DateV2Value<DateTimeV2ValueType>, ReturnNativeType>(dtv1);
-        } else if constexpr (PType == TYPE_TIMEV2) {
-            auto tv1 = static_cast<TimeValue::TimeType>(arg1);
-            auto tv2 = static_cast<TimeValue::TimeType>(arg2);
-            double res = TimeValue::limit_with_bound(Impl::is_negative() ? tv1 - tv2 : tv1 + tv2);
-            return res;
-        } else {
-            throw Exception(ErrorCode::FATAL_ERROR, "not support type for function {}", name);
+    static DataTypePtr get_return_type_impl(const DataTypes& arguments) {
+        if (arguments[0]->is_nullable() || arguments[1]->is_nullable()) {
+            return make_nullable(std::make_shared<typename PrimitiveTypeTraits<PType>::DataType>());
         }
+        return std::make_shared<typename PrimitiveTypeTraits<PType>::DataType>();
     }
 
-    static FunctionPtr create() { return std::make_shared<FunctionAddTime>(); }
-
-    Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
-                        uint32_t result, size_t input_rows_count) const override {
-        DCHECK_EQ(arguments.size(), 2);
-        const auto& [left_col, left_const] =
-                unpack_if_const(block.get_by_position(arguments[0]).column);
-        const auto& [right_col, right_const] =
-                unpack_if_const(block.get_by_position(arguments[1]).column);
-        ColumnPtr nest_col1 = remove_nullable(left_col);
-        ColumnPtr nest_col2 = remove_nullable(right_col);
-        auto res = ColumnVector<ReturnType>::create(input_rows_count, 0);
-
-        if (left_const) {
-            execute_constant_vector(assert_cast<const ColumnType1&>(*nest_col1).get_element(0),
-                                    assert_cast<const ColumnType2&>(*nest_col2).get_data(),
-                                    res->get_data(), input_rows_count);
-        } else if (right_const) {
-            execute_vector_constant(assert_cast<const ColumnType1&>(*nest_col1).get_data(),
-                                    assert_cast<const ColumnType2&>(*nest_col2).get_element(0),
-                                    res->get_data(), input_rows_count);
-        } else {
-            execute_vector_vector(assert_cast<const ColumnType1&>(*nest_col1).get_data(),
-                                  assert_cast<const ColumnType2&>(*nest_col2).get_data(),
-                                  res->get_data(), input_rows_count);
-        }
-
-        block.replace_by_position(result, std::move(res));
-        return Status::OK();
-    }
-    void execute_vector_vector(const PaddedPODArray<InputType1>& left_col,
-                               const PaddedPODArray<InputType2>& right_col,
-                               PaddedPODArray<ReturnNativeType>& res_data,
-                               size_t input_rows_count) const {
-        for (size_t i = 0; i < input_rows_count; ++i) {
-            res_data[i] = compute(left_col[i], right_col[i]);
-        }
-    }
+    static void execute(const std::vector<ColumnWithConstAndNullMap>& cols_info,
+                        typename ResultColumnType::MutablePtr& res_col,
+                        PaddedPODArray<UInt8>& res_null_map_data, size_t input_rows_count) {
+        const auto& left_data =
+                assert_cast<const ResultColumnType*>(cols_info[0].nested_col)->get_data();
+        const auto& right_data =
+                assert_cast<const ColumnVector<TYPE_TIMEV2>*>(cols_info[1].nested_col)->get_data();
 
-    void execute_vector_constant(const PaddedPODArray<InputType1>& left_col,
-                                 const InputType2 right_value,
-                                 PaddedPODArray<ReturnNativeType>& res_data,
-                                 size_t input_rows_count) const {
         for (size_t i = 0; i < input_rows_count; ++i) {
-            res_data[i] = compute(left_col[i], right_value);
-        }
-    }
+            if (cols_info[0].is_null_at(i) || cols_info[1].is_null_at(i)) {
+                res_col->insert_default();
+                res_null_map_data[i] = 1;
+                continue;
+            }
 
-    void execute_constant_vector(const InputType1 left_value,
-                                 const PaddedPODArray<InputType2>& right_col,
-                                 PaddedPODArray<ReturnNativeType>& res_data,
-                                 size_t input_rows_count) const {
-        for (size_t i = 0; i < input_rows_count; ++i) {
-            res_data[i] = compute(left_value, right_col[i]);
+            const auto& arg1 = left_data[index_check_const(i, cols_info[0].is_const)];
+            const auto& arg2 = right_data[index_check_const(i, cols_info[1].is_const)];
+
+            if constexpr (PType == TYPE_DATETIMEV2) {
+                DateV2Value<DateTimeV2ValueType> dtv1 =
+                        binary_cast<InputType1, DateV2Value<DateTimeV2ValueType>>(arg1);
+                auto tv2 = static_cast<TimeValue::TimeType>(arg2);
+                TimeInterval interval(TimeUnit::MICROSECOND, tv2, IsNegative);
+                bool out_range = dtv1.template date_add_interval<TimeUnit::MICROSECOND>(interval);
+                if (!out_range) [[unlikely]] {
+                    throw_invalid_strings(name, dtv1.to_string(), std::to_string(arg2));
+                }
+                res_col->insert_value(
+                        binary_cast<DateV2Value<DateTimeV2ValueType>, InputType1>(dtv1));
+            } else if constexpr (PType == TYPE_TIMEV2) {
+                auto tv1 = static_cast<TimeValue::TimeType>(arg1);
+                auto tv2 = static_cast<TimeValue::TimeType>(arg2);
+                double res = TimeValue::limit_with_bound(IsNegative ? tv1 - tv2 : tv1 + tv2);
+                res_col->insert_value(res);
+            } else {
+                throw Exception(ErrorCode::FATAL_ERROR, "not support type for function {}", name);
+            }
         }
     }
 };
+
+using AddTimeDatetimeImpl = AddTimeImplBase<TYPE_DATETIMEV2, false>;
+using AddTimeTimeImpl = AddTimeImplBase<TYPE_TIMEV2, false>;
+using SubTimeDatetimeImpl = AddTimeImplBase<TYPE_DATETIMEV2, true>;
+using SubTimeTimeImpl = AddTimeImplBase<TYPE_TIMEV2, true>;
 #include "common/compile_check_avoid_end.h"
 } // namespace doris::vectorized

be/src/vec/functions/function_date_or_datetime_to_string.cpp

@@ -77,8 +77,6 @@ class FunctionDateOrDateTimeToString : public IFunction {
         return {};
     }
 
-    ColumnNumbers get_arguments_that_are_always_constant() const override { return {1}; }
-
     // In ICU, Week_array: {"", "Sunday", "Monday", ..., "Saturday"}, size = 8
     // Month_array: {"January", "February", ..., "December"}, size = 12
     static constexpr size_t DAY_NUM_IN_ICU = 8;
@@ -140,28 +138,48 @@ class FunctionDateOrDateTimeToString : public IFunction {
         return IFunction::open(context, scope);
     }
 
+    bool use_default_implementation_for_nulls() const override { return false; }
+
     Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
                         uint32_t result, size_t input_rows_count) const override {
         const ColumnPtr source_col = block.get_by_position(arguments[0]).column;
+        const NullMap* null_map = nullptr;
+        ColumnPtr actual_col = source_col;
+
+        if (is_column_nullable(*source_col)) {
+            const auto* nullable_col = check_and_get_column<ColumnNullable>(source_col.get());
+            actual_col = nullable_col->get_nested_column_ptr();
+            null_map = &nullable_col->get_null_map_data();
+        }
+
         const auto* sources =
-                check_and_get_column<ColumnVector<Transform::OpArgType>>(source_col.get());
-        auto col_res = ColumnString::create();
+                check_and_get_column<ColumnVector<Transform::OpArgType>>(actual_col.get());
+        if (!sources) [[unlikely]] {
+            return Status::FatalError("Illegal column {} of first argument of function {}",
+                                      block.get_by_position(arguments[0]).column->get_name(), name);
+        }
 
-        // Support all input of datetime is valind to make sure not null return
-        if (sources) {
-            vector(context, sources->get_data(), col_res->get_chars(), col_res->get_offsets());
-            block.replace_by_position(result, std::move(col_res));
+        auto col_res = ColumnString::create();
+        vector(context, sources->get_data(), col_res->get_chars(), col_res->get_offsets(),
+               null_map);
+
+        if (null_map) {
+            const auto* nullable_col = check_and_get_column<ColumnNullable>(source_col.get());
+            block.replace_by_position(
+                    result,
+                    ColumnNullable::create(std::move(col_res),
+                                           nullable_col->get_null_map_column_ptr()->clone_resized(
+                                                   input_rows_count)));
         } else {
-            return Status::InternalError("Illegal column {} of first argument of function {}",
-                                         block.get_by_position(arguments[0]).column->get_name(),
-                                         name);
+            block.replace_by_position(result, std::move(col_res));
         }
         return Status::OK();
     }
 
 private:
     static void vector(FunctionContext* context, const PaddedPODArray<NativeType>& ts,
-                       ColumnString::Chars& res_data, ColumnString::Offsets& res_offsets) {
+                       ColumnString::Chars& res_data, ColumnString::Offsets& res_offsets,
+                       const NullMap* null_map = nullptr) {
         const auto len = ts.size();
         res_data.resize(len * Transform::max_size);
         res_offsets.resize(len);
@@ -176,7 +194,12 @@ class FunctionDateOrDateTimeToString : public IFunction {
             names_ptr = state->month_names;
         }
 
-        for (int i = 0; i < len; ++i) {
+        for (size_t i = 0; i < len; ++i) {
+            if (null_map && (*null_map)[i]) {
+                res_offsets[i] = cast_set<UInt32>(offset);
+                continue;
+            }
+
             const auto& t = ts[i];
             const auto date_time_value = binary_cast<NativeType, DateType>(t);
             res_offsets[i] = cast_set<UInt32>(

be/src/vec/functions/function_needs_to_handle_null.h

@@ -53,6 +53,12 @@ class FunctionNeedsToHandleNull : public IFunction {
         }
         return false;
     }
+    DataTypes get_variadic_argument_types_impl() const override {
+        if constexpr (requires { Impl::get_variadic_argument_types_impl(); }) {
+            return Impl::get_variadic_argument_types_impl();
+        }
+        return {};
+    }
 
     bool use_default_implementation_for_nulls() const override { return false; }
 

regression-test/data/nereids_function_p0/scalar_function/Map.out → regression-test/data/nereids_function_p0/scalar_function/nereids_scalar_fn_map.out

@@ -3722,3 +3722,11 @@ false
 -- !sql --
 {"{"zip":10001}":"{"city":"NY"}", "{"code":10002}":"{"state":"NY"}"}
 
+-- !element_at_tint_date_dayname --
+\N
+Sunday
+
+-- !element_at_tint_date_dayname_notnull --
+\N
+Saturday
+

regression-test/data/nereids_p0/join/test_outer_join.out

@@ -10,3 +10,8 @@
 
 -- !join --
 
+-- !join --
+\N	\N
+1	2023-12-18T23:30:01
+2	2023-12-15T23:30:01
+

regression-test/suites/nereids_function_p0/scalar_function/Map.groovy → regression-test/suites/nereids_function_p0/scalar_function/nereids_scalar_fn_map.groovy

@@ -35,6 +35,7 @@ suite("nereids_scalar_fn_map") {
     order_qt_element_at_str_tint    """ select km_str_tint[kstr] from fn_test """
     order_qt_element_at_date_tint   """ select km_date_tint[kdt] from fn_test """
     order_qt_element_at_dtm_tint    """ select km_dtm_tint[kdtm] from fn_test """
+
     order_qt_element_at_bool_tint_notnull   """ select km_bool_tint[kbool] from fn_test_not_nullable """
     order_qt_element_at_tint_tint_notnull   """ select km_tint_tint[ktint] from fn_test_not_nullable """
     order_qt_element_at_sint_tint_notnull   """ select km_sint_tint[ksint] from fn_test_not_nullable """
@@ -314,4 +315,24 @@ suite("nereids_scalar_fn_map") {
     qt_sql "select map('postal_code', 10001, 'area_code', 10002, 'zip_plus_4', 10003)"
     qt_sql "select map('{\"zip\":10001}', '{\"city\":\"NY\"}', '{\"code\":10002}', '{\"state\":\"NY\"}')"
 
+    sql """ drop table if exists fn_test_element_at_map_date """
+    sql """
+        create table fn_test_element_at_map_date (
+            id int null,
+            m map<tinyint, date> null
+        ) engine=olap
+        distributed by hash(id) buckets 1
+        properties('replication_num' = '1')
+    """
+    sql """
+        insert into fn_test_element_at_map_date values
+        (1, map(1,null, 2,'2023-12-16')),
+        (2, map(1,'2023-01-01', 2,null));
+    """
+    order_qt_element_at_tint_date_dayname         """
+    select dayname(element_at(m, 1)) from fn_test_element_at_map_date
+    """
+    order_qt_element_at_tint_date_dayname_notnull """
+    select dayname(element_at(m, 2)) from fn_test_element_at_map_date
+    """
 }

regression-test/suites/nereids_p0/join/test_outer_join.groovy

@@ -84,7 +84,30 @@ suite("test_outer_join", "nereids_p0") {
         contains "FULL OUTER JOIN(PARTITIONED)"
     }
 
-    sql "DROP TABLE IF EXISTS ${tbl1}"
-    sql "DROP TABLE IF EXISTS ${tbl2}"
-    sql "DROP TABLE IF EXISTS ${tbl3}"
+    def tbl4 = "test_outer_join4"
+    def tbl5 = "test_outer_join5"
+
+    sql "DROP TABLE IF EXISTS test_outer_join4"
+    sql """
+        CREATE TABLE test_outer_join4 (
+            k INT,
+            d DATEV2
+        )
+        DUPLICATE KEY(k)
+        DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES ("replication_num" = "1");
+    """
+    sql "DROP TABLE IF EXISTS test_outer_join5"
+    sql """
+        CREATE TABLE test_outer_join5 (
+            k INT
+        )
+        DUPLICATE KEY(k)
+        DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES ("replication_num" = "1");
+    """
+    sql """INSERT INTO test_outer_join4 VALUES (1, '2023-12-18'), (2, '2023-12-15');"""
+    sql """INSERT INTO test_outer_join5 VALUES (1), (2), (3);"""
+
+    order_qt_join """
+        SELECT k, add_time(d, '23:30:01') FROM test_outer_join4 RIGHT JOIN test_outer_join5 USING (k) ORDER BY k;
+    """
 }

regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy

@@ -175,7 +175,7 @@ suite("test_date_function_v2") {
 
     test{
         sql("select add_time('9999-12-29 00:00:00', '122:35:22.123456');")
-        exception "datetime value is out of range in function add_time"
+        exception "Operation add_time of"
     }
 
     qt_sql_subtime1("select sub_time('2023-10-14 00:00:00', '22:35:22');")
@@ -193,7 +193,7 @@ suite("test_date_function_v2") {
 
     test{
         sql("select sub_time('0000-01-01 00:00:00', '122:35:22.123456');")
-        exception "datetime value is out of range in function sub_time"
+        exception "Operation sub_time of"
     }
 
     //test computetimearithmetic regular