From 106afda9d6711ee191a176f64e7665e8bea24f79 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Thu, 27 Nov 2025 04:32:07 +0800 Subject: [PATCH 01/41] feat: map_concat --- be/src/vec/functions/function_map.cpp | 90 ++++++++++++++++++- .../doris/catalog/BuiltinScalarFunctions.java | 2 + .../functions/scalar/MapConcat.java | 76 ++++++++++++++++ .../visitor/ScalarFunctionVisitor.java | 4 + 4 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java diff --git a/be/src/vec/functions/function_map.cpp b/be/src/vec/functions/function_map.cpp index e0b20ec42aeb1e..b9006839fd5005 100644 --- a/be/src/vec/functions/function_map.cpp +++ b/be/src/vec/functions/function_map.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -36,6 +37,7 @@ #include "vec/columns/column_const.h" #include "vec/columns/column_map.h" #include "vec/columns/column_nullable.h" +#include "vec/columns/column_variant.h" #include "vec/columns/column_vector.h" #include "vec/common/assert_cast.h" #include "vec/common/typeid_cast.h" @@ -88,7 +90,11 @@ class FunctionMap : public IFunction { uint32_t result, size_t input_rows_count) const override { DCHECK(arguments.size() % 2 == 0) << "function: " << get_name() << ", arguments should not be even number"; - + LOG(INFO) << "[FunctionMap.execute_impl] input_rows_count: " << input_rows_count; + for (size_t i =0;i < arguments.size();++i) { + auto& col = block.get_by_position(arguments[i]).column; + LOG(INFO) << "[FunctionMap.execute_impl] argument " << i << " column type: " << col->get_name()<<" data:"<dump_structure(); + } size_t num_element = arguments.size(); auto result_col = block.get_by_position(result).type->create_column(); @@ -792,6 +798,87 @@ class FunctionDeduplicateMap : public IFunction { private: }; +class FunctionMapConcat : public IFunction{ +public: + static constexpr auto name = "map_concat"; + static FunctionPtr create() {return std::make_shared();} + String get_name() const override { return name; } + bool is_variadic() const override { return true; } + size_t get_number_of_arguments() const override { return 1; } + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + DCHECK(arguments.size()>0) + <<"function: "<get_primitive_type() == TYPE_MAP) + << "argument for function map_concat should be DataTypeMap" + << "and argument is "<get_name(); + } + LOG(INFO) << "[FunctionMapConcat.get_return_type_impl] return type:"<< arguments[0]->get_name(); + return arguments[0]; + } + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + const uint32_t result, size_t input_rows_count) const override { + auto result_col = block.get_by_position(result).type->create_column(); + LOG(INFO)<<"[FunctionMapConcat.execute_impl] result:"<(result_col.get()); + // map keys column + auto& result_col_map_keys_data = result_map_column->get_keys(); + result_col_map_keys_data.reserve(10); + // map values column + auto& result_col_map_vals_data = result_map_column->get_values(); + result_col_map_vals_data.reserve(10); + ColumnArray::Offsets64& column_offsets = result_map_column->get_offsets(); + column_offsets.resize(input_rows_count); + + size_t off = 0; + for(int row = 0; row < input_rows_count ; row++) { + for(size_t col:arguments){ + const ColumnMap*map_column = nullptr; + auto src_column = + block.get_by_position(col).column->convert_to_full_column_if_const(); + if (src_column->is_nullable()){ + auto nullable_column = reinterpret_cast(src_column.get()); + map_column = check_and_get_column(nullable_column->get_nested_column()); + }else{ + map_column = check_and_get_column(*src_column.get()); + } + if (!src_column){ + return Status::RuntimeError("unsupported types for function {}({})", get_name(), + block.get_by_position(col).type->get_name()); + } + const auto& src_column_offsets = map_column->get_offsets(); + const size_t length = src_column_offsets[row] - src_column_offsets[row - 1]; + off += length; + for(size_t i=src_column_offsets[row-1];iget_keys(),i); + result_col_map_vals_data.insert_from(map_column->get_values(),i); + LOG(INFO)<<"[FunctionMapConcat.execute_impl] last key:"<get_keys()[i].to_string(); + LOG(INFO)<<"[FunctionMapConcat.execute_impl] internel value:"<get_values()[i].to_string(); + } + LOG(INFO) << "[FunctionMapConcat.execute_impl] map_column:"<< block.get_by_position(col).dump_structure()<<" row "<deduplicate_keys()); + LOG(INFO) << "[DEBUG] After deduplicate - total keys: " << result_col_map_keys_data.size(); + for (size_t i = 0; i < result_col_map_keys_data.size(); ++i) { + Field key_field = result_col_map_keys_data[i]; + LOG(INFO) << "[DEBUG] Key " << i << ": " << key_field.to_string(); + } + block.replace_by_position(result, std::move(result_col)); + + return Status::OK(); + } +}; + void register_function_map(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function>(); @@ -801,6 +888,7 @@ void register_function_map(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); factory.register_function(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index b7970c448cf0ca..37324f4c49d7df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -550,6 +550,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsDiff; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsSub; +import org.apache.doris.nereids.trees.expressions.functions.scalar.MapConcat; import com.google.common.collect.ImmutableList; @@ -871,6 +872,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(MapKeys.class, "map_keys"), scalar(MapSize.class, "map_size"), scalar(MapValues.class, "map_values"), + scalar(MapConcat.class, "map_concat"), scalar(Mask.class, "mask"), scalar(MaskFirstN.class, "mask_first_n"), scalar(MaskLastN.class, "mask_last_n"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java new file mode 100644 index 00000000000000..b877b631741bc4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.MapType; +import org.apache.doris.nereids.types.coercion.AnyDataType; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.types.MapType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'map_concat' + */ +public class MapConcat extends ScalarFunction + implements ExplicitlyCastableSignature, PropagateNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature + .ret(MapType.of(new AnyDataType(0), new AnyDataType(1))) + .varArgs(MapType.of(new AnyDataType(0), new AnyDataType(1))) + ); + + /** + * constructor with more than 0 arguments. + */ + public MapConcat(Expression arg, Expression... varArgs) { + super("map_concat", ExpressionUtils.mergeArguments(arg, varArgs)); + } + + /** + * private constructor + */ + private MapConcat(ScalarFunctionParams functionParams) { + super(functionParams); + } + + @Override + public MapConcat withChildren(List children) { + Preconditions.checkArgument(children.size() >= 1); + return new MapConcat(getFunctionParams(children)); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitMapConcat(this, context); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index 5a556add8376b3..3810c435ed7a75 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -553,6 +553,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.YearWeek; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsDiff; +import org.apache.doris.nereids.trees.expressions.functions.scalar.MapConcat; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsSub; import org.apache.doris.nereids.trees.expressions.functions.udf.AliasUdf; import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdf; @@ -2721,6 +2722,9 @@ default R visitPeriodAdd(PeriodAdd periodAdd, C context) { default R visitPeriodDiff(PeriodDiff periodDiff, C context) { return visitScalarFunction(periodDiff, context); } + default R visitMapConcat(MapConcat mapConcat, C context) { + return visitScalarFunction(mapConcat, context); + } default R visitUnicodeNormalize(UnicodeNormalize func, C context) { return visitScalarFunction(func, context); From eaddffec9dcfb56e384bcaca1d4f71772e9b8c68 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Sat, 29 Nov 2025 17:00:12 +0800 Subject: [PATCH 02/41] fix: optimize code --- be/src/vec/functions/function_map.cpp | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/be/src/vec/functions/function_map.cpp b/be/src/vec/functions/function_map.cpp index b9006839fd5005..3cb6b63e194499 100644 --- a/be/src/vec/functions/function_map.cpp +++ b/be/src/vec/functions/function_map.cpp @@ -808,25 +808,16 @@ class FunctionMapConcat : public IFunction{ DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { DCHECK(arguments.size()>0) <<"function: "<get_primitive_type() == TYPE_MAP) - << "argument for function map_concat should be DataTypeMap" - << "and argument is "<get_name(); - } - LOG(INFO) << "[FunctionMapConcat.get_return_type_impl] return type:"<< arguments[0]->get_name(); return arguments[0]; } Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, const uint32_t result, size_t input_rows_count) const override { auto result_col = block.get_by_position(result).type->create_column(); - LOG(INFO)<<"[FunctionMapConcat.execute_impl] result:"<(result_col.get()); // map keys column auto& result_col_map_keys_data = result_map_column->get_keys(); - result_col_map_keys_data.reserve(10); // map values column auto& result_col_map_vals_data = result_map_column->get_values(); - result_col_map_vals_data.reserve(10); ColumnArray::Offsets64& column_offsets = result_map_column->get_offsets(); column_offsets.resize(input_rows_count); @@ -852,27 +843,11 @@ class FunctionMapConcat : public IFunction{ for(size_t i=src_column_offsets[row-1];iget_keys(),i); result_col_map_vals_data.insert_from(map_column->get_values(),i); - LOG(INFO)<<"[FunctionMapConcat.execute_impl] last key:"<get_keys()[i].to_string(); - LOG(INFO)<<"[FunctionMapConcat.execute_impl] internel value:"<get_values()[i].to_string(); } - LOG(INFO) << "[FunctionMapConcat.execute_impl] map_column:"<< block.get_by_position(col).dump_structure()<<" row "<deduplicate_keys()); - LOG(INFO) << "[DEBUG] After deduplicate - total keys: " << result_col_map_keys_data.size(); - for (size_t i = 0; i < result_col_map_keys_data.size(); ++i) { - Field key_field = result_col_map_keys_data[i]; - LOG(INFO) << "[DEBUG] Key " << i << ": " << key_field.to_string(); - } block.replace_by_position(result, std::move(result_col)); return Status::OK(); From 6473a0385c56e6bd59dd6649802b8fa9da45a279 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Fri, 5 Dec 2025 04:36:07 +0800 Subject: [PATCH 03/41] fix: nullable --- be/src/vec/functions/function_map.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/be/src/vec/functions/function_map.cpp b/be/src/vec/functions/function_map.cpp index 3cb6b63e194499..2c1a3a540d4518 100644 --- a/be/src/vec/functions/function_map.cpp +++ b/be/src/vec/functions/function_map.cpp @@ -813,7 +813,13 @@ class FunctionMapConcat : public IFunction{ Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, const uint32_t result, size_t input_rows_count) const override { auto result_col = block.get_by_position(result).type->create_column(); - auto* result_map_column = assert_cast(result_col.get()); + ColumnMap* result_map_column = nullptr; + if (result_col->is_nullable()){ + auto nullable_column = reinterpret_cast(result_col.get()); + result_map_column = check_and_get_column(nullable_column->get_nested_column()); + } else { + result_map_column = check_and_get_column(result_col.get()); + } // map keys column auto& result_col_map_keys_data = result_map_column->get_keys(); // map values column From 930ef85f66bcfd0d6481e6e124fd8ca64067f725 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Fri, 5 Dec 2025 04:36:47 +0800 Subject: [PATCH 04/41] feat: FunctionMapConcatTest --- .../data_type_date_or_datetime_v2.h | 6 +++ .../vec/function/function_map_concat_test.cpp | 28 ++++++++++++ be/test/vec/function/function_test_util.cpp | 44 ++++++++++++++++--- be/test/vec/function/function_test_util.h | 16 ++++++- 4 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 be/test/vec/function/function_map_concat_test.cpp diff --git a/be/src/vec/data_types/data_type_date_or_datetime_v2.h b/be/src/vec/data_types/data_type_date_or_datetime_v2.h index 82247777696428..3922f33de9472d 100644 --- a/be/src/vec/data_types/data_type_date_or_datetime_v2.h +++ b/be/src/vec/data_types/data_type_date_or_datetime_v2.h @@ -190,5 +190,11 @@ template constexpr bool IsDataTypeDateTimeV2 = false; template <> inline constexpr bool IsDataTypeDateTimeV2 = true; + +template +constexpr bool IsDataTypeMap = false; +template <> +inline constexpr bool IsDataTypeMap = true; + #include "common/compile_check_end.h" } // namespace doris::vectorized diff --git a/be/test/vec/function/function_map_concat_test.cpp b/be/test/vec/function/function_map_concat_test.cpp new file mode 100644 index 00000000000000..fa67e79f8c2e59 --- /dev/null +++ b/be/test/vec/function/function_map_concat_test.cpp @@ -0,0 +1,28 @@ +#include +#include +#include +#include "vec/core/types.h" +#include "function_test_util.h" + + +namespace doris::vectorized { +TEST(FunctionMapConcatTest, TestBase) { + const std::string func_name = "map_concat"; + { + InputTypeSet input_types = { + PrimitiveType::TYPE_MAP,PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_MAP,PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING + }; + DataSet data_set = { + { + TestArray({TestArray({std::int32_t(1),std::string("A")}),TestArray({std::int32_t(2),std::string("B")})}), + TestArray({std::int32_t(1),std::string("A"),std::int32_t(2),std::string("B")}) + } + }; + + // // 处理 check_function 的返回值 + Status status = check_function(func_name, input_types, data_set); + EXPECT_TRUE(status.ok()) << "Function test failed: " << status.to_string(); + } +} +} diff --git a/be/test/vec/function/function_test_util.cpp b/be/test/vec/function/function_test_util.cpp index d67f36a6e69502..155e9a4d53c880 100644 --- a/be/test/vec/function/function_test_util.cpp +++ b/be/test/vec/function/function_test_util.cpp @@ -213,14 +213,14 @@ static size_t type_index_to_data_type(const std::vector& input_types, s ut_type::UTDataTypeDesc value_desc; DataTypePtr value_type = nullptr; ++index; - size_t ret = type_index_to_data_type(input_types, index, key_desc, key_type); - if (ret <= 0) { - return ret; + size_t ret_key = type_index_to_data_type(input_types, index, key_desc, key_type); + if (ret_key <= 0) { + return ret_key; } ++index; - ret = type_index_to_data_type(input_types, index, value_desc, value_type); - if (ret <= 0) { - return ret; + size_t ret_value = type_index_to_data_type(input_types, index, value_desc, value_type); + if (ret_value <= 0) { + return ret_value; } if (key_desc.is_nullable) { key_type = make_nullable(key_type); @@ -230,7 +230,7 @@ static size_t type_index_to_data_type(const std::vector& input_types, s } type = std::make_shared(key_type, value_type); desc = type; - return ret + 1; + return ret_key + ret_value + 1; } case PrimitiveType::TYPE_STRUCT: { ++index; @@ -352,6 +352,32 @@ bool insert_array_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const Any return true; } +bool insert_map_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const AnyType& cell, + bool datetime_is_string_format) { + auto origin_input_array = any_cast(cell); + DataTypePtr key_type = assert_cast(type_ptr.get())->get_key_type(); + DataTypePtr value_type = assert_cast(type_ptr.get())->get_value_type(); + MutableColumnPtr key_column = key_type->create_column(); + MutableColumnPtr value_column = value_type->create_column(); + for(size_t i=0;i(key_array)); + map.push_back(Field::create_field(value_array)); + column->insert(Field::create_field(map)); + return true; +} + + // NOLINTBEGIN(readability-function-size) bool insert_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const AnyType& cell, bool datetime_is_string_format) { @@ -536,6 +562,10 @@ bool insert_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const AnyType& } break; } + case PrimitiveType::TYPE_MAP: { + RETURN_IF_FALSE((insert_map_cell(column, type_ptr, cell, datetime_is_string_format))); + break; + } default: { std::cerr << "dataset not supported for type:" << type_to_string(type); return false; diff --git a/be/test/vec/function/function_test_util.h b/be/test/vec/function/function_test_util.h index 7e2799c9dd5704..d9b2ce1b013bad 100644 --- a/be/test/vec/function/function_test_util.h +++ b/be/test/vec/function/function_test_util.h @@ -363,6 +363,12 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty } // 2. execute function + auto get_key_value_type=[&](){ + DataTypeFactory&factory = DataTypeFactory::instance(); + DataTypePtr key_type = factory.create_data_type(any_cast(input_types[1]), true); + DataTypePtr value_type = factory.create_data_type(any_cast(input_types[2]), true); + return std::make_pair(key_type,value_type); + }; auto return_type = [&]() { if constexpr (IsDataTypeDecimal) { // decimal return ResultNullable ? make_nullable(std::make_shared(result_precision, @@ -376,6 +382,10 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty } return ResultNullable ? make_nullable(std::make_shared(real_scale)) : std::make_shared(real_scale); + } else if constexpr (IsDataTypeMap) { + auto [key_type, value_type] = get_key_value_type(); + return ResultNullable ? make_nullable(std::make_shared(key_type, value_type)) + : std::make_shared(key_type, value_type); } else { return ResultNullable ? make_nullable(std::make_shared()) : std::make_shared(); @@ -428,7 +438,11 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty } result_type_ptr = ResultNullable ? make_nullable(std::make_shared(real_scale)) : std::make_shared(real_scale); - } else { + } else if constexpr (IsDataTypeMap){ + auto [key_type, value_type] = get_key_value_type(); + result_type_ptr = ResultNullable ? make_nullable(std::make_shared(key_type, value_type)) + : std::make_shared(key_type, value_type); + }else { result_type_ptr = ResultNullable ? make_nullable(std::make_shared()) : std::make_shared(); } From 7b0bf75d13ef88616a3ad5be505676362a2fe836 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Sun, 7 Dec 2025 17:34:13 +0800 Subject: [PATCH 05/41] fix: ColumnNullable --- be/src/vec/functions/function_map.cpp | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/be/src/vec/functions/function_map.cpp b/be/src/vec/functions/function_map.cpp index 2c1a3a540d4518..5f822309c32790 100644 --- a/be/src/vec/functions/function_map.cpp +++ b/be/src/vec/functions/function_map.cpp @@ -814,9 +814,10 @@ class FunctionMapConcat : public IFunction{ const uint32_t result, size_t input_rows_count) const override { auto result_col = block.get_by_position(result).type->create_column(); ColumnMap* result_map_column = nullptr; + ColumnNullable* result_nullable_column = nullptr; if (result_col->is_nullable()){ - auto nullable_column = reinterpret_cast(result_col.get()); - result_map_column = check_and_get_column(nullable_column->get_nested_column()); + result_nullable_column = reinterpret_cast(result_col.get()); + result_map_column = check_and_get_column(result_nullable_column->get_nested_column()); } else { result_map_column = check_and_get_column(result_col.get()); } @@ -827,19 +828,26 @@ class FunctionMapConcat : public IFunction{ ColumnArray::Offsets64& column_offsets = result_map_column->get_offsets(); column_offsets.resize(input_rows_count); + // Initialize null map if result is nullable + // reference to ColumnNullable::size() + if (result_nullable_column) { + auto& null_map_data = result_nullable_column->get_null_map_data(); + null_map_data.resize_fill(input_rows_count, 0); + } + size_t off = 0; - for(int row = 0; row < input_rows_count ; row++) { - for(size_t col:arguments){ - const ColumnMap*map_column = nullptr; + for(size_t row = 0; row < input_rows_count ; row++) { + for(size_t col: arguments){ + const ColumnMap* map_column = nullptr; auto src_column = block.get_by_position(col).column->convert_to_full_column_if_const(); if (src_column->is_nullable()){ auto nullable_column = reinterpret_cast(src_column.get()); map_column = check_and_get_column(nullable_column->get_nested_column()); - }else{ + } else { map_column = check_and_get_column(*src_column.get()); } - if (!src_column){ + if (!map_column){ return Status::RuntimeError("unsupported types for function {}({})", get_name(), block.get_by_position(col).type->get_name()); } @@ -855,7 +863,6 @@ class FunctionMapConcat : public IFunction{ } RETURN_IF_ERROR(result_map_column->deduplicate_keys()); block.replace_by_position(result, std::move(result_col)); - return Status::OK(); } }; From f0970be8f2ab37d491a1a26560dc1a5795f8f4e4 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Sun, 7 Dec 2025 17:34:51 +0800 Subject: [PATCH 06/41] feat: function_map_concat_test --- .../vec/function/function_map_concat_test.cpp | 314 +++++++++++++++++- 1 file changed, 309 insertions(+), 5 deletions(-) diff --git a/be/test/vec/function/function_map_concat_test.cpp b/be/test/vec/function/function_map_concat_test.cpp index fa67e79f8c2e59..b757158fa3965d 100644 --- a/be/test/vec/function/function_map_concat_test.cpp +++ b/be/test/vec/function/function_map_concat_test.cpp @@ -9,18 +9,322 @@ namespace doris::vectorized { TEST(FunctionMapConcatTest, TestBase) { const std::string func_name = "map_concat"; { + // simple case - two maps InputTypeSet input_types = { - PrimitiveType::TYPE_MAP,PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, - PrimitiveType::TYPE_MAP,PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING }; DataSet data_set = { { - TestArray({TestArray({std::int32_t(1),std::string("A")}),TestArray({std::int32_t(2),std::string("B")})}), - TestArray({std::int32_t(1),std::string("A"),std::int32_t(2),std::string("B")}) + TestArray({ + TestArray({std::int32_t(1), std::string("A")}), + TestArray({std::int32_t(2), std::string("B")}) + }), + TestArray({ + std::int32_t(1), std::string("A"), + std::int32_t(2), std::string("B") + }) + } + }; + + Status status = check_function(func_name, input_types, data_set); + EXPECT_TRUE(status.ok()) << "Function test failed: " << status.to_string(); + } + { + // two maps concatenation with string keys + InputTypeSet input_types = { + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_STRING + }; + DataSet data_set = { + { + TestArray({ + TestArray({std::string("A"), std::string("a")}), + TestArray({std::string("B"), std::string("b")}) + }), + TestArray({ + std::string("A"), std::string("a"), + std::string("B"), std::string("b") + }) + } + }; + Status status = check_function(func_name, input_types, data_set); + EXPECT_TRUE(status.ok()) << "Function test failed: " << status.to_string(); + } + { + // two maps concatenation with string keys + InputTypeSet input_types = { + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_STRING + }; + DataSet data_set = { + { + TestArray({ + TestArray({std::string("A"), std::string("a")}), + TestArray({std::string("B"), std::string("b")}) + }), + TestArray({ + std::string("A"), std::string("a"), + std::string("B"), std::string("b") + }) + }, + { + TestArray({ + TestArray({std::string("C"), std::string("c")}), + TestArray({std::string("D"), std::string("d")}) + }), + TestArray({ + std::string("C"), std::string("c"), + std::string("D"), std::string("d") + }) + } + }; + Status status = check_function(func_name, input_types, data_set); + EXPECT_TRUE(status.ok()) << "Function test failed: " << status.to_string(); + } +} + +TEST(FunctionMapConcatTest, TestEdgeCases) { + const std::string func_name = "map_concat"; + + // Test empty maps + { + InputTypeSet input_types = { + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING + }; + DataSet data_set = { + { + TestArray({TestArray({}), TestArray({})}), + TestArray({}) + }, + // { + // TestArray({TestArray({std::int32_t(1), std::string("A")}), TestArray({})}), + // TestArray({std::int32_t(1), std::string("A")}) + // }, + // { + // TestArray({TestArray({}), TestArray({std::int32_t(2), std::string("B")})}), + // TestArray({std::int32_t(2), std::string("B")}) + // } + }; + + Status status = check_function(func_name, input_types, data_set); + EXPECT_TRUE(status.ok()) << "Function test failed one: " << status.to_string(); + } + + // Test key conflicts (later map should override earlier) + { + InputTypeSet input_types = { + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING + }; + DataSet data_set = { + { + TestArray({ + TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), std::string("B")}), + TestArray({std::int32_t(1), std::string("C"), std::int32_t(3), std::string("D")}) + }), + TestArray({ + std::int32_t(2), std::string("B"), + std::int32_t(1), std::string("C"), + std::int32_t(3), std::string("D") + }) + } + }; + + Status status = check_function(func_name, input_types, data_set); + EXPECT_TRUE(status.ok()) << "Function test failed two: " << status.to_string(); + } + + // Test multiple maps (more than 2) + { + InputTypeSet input_types = { + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING + }; + DataSet data_set = { + { + TestArray({ + TestArray({std::int32_t(1), std::string("A")}), + TestArray({std::int32_t(2), std::string("B")}), + TestArray({std::int32_t(3), std::string("C")}) + }), + TestArray({ + std::int32_t(1), std::string("A"), + std::int32_t(2), std::string("B"), + std::int32_t(3), std::string("C") + }) + } + }; + + Status status = check_function(func_name, input_types, data_set); + EXPECT_TRUE(status.ok()) << "Function test failed three: " << status.to_string(); + } + + // Test different value types + { + InputTypeSet input_types = { + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT + }; + DataSet data_set = { + { + TestArray({ + TestArray({std::string("A"), std::int32_t(1), std::string("B"), std::int32_t(2)}), + TestArray({std::string("C"), std::int32_t(3), std::string("D"), std::int32_t(4)}) + }), + TestArray({ + std::string("A"), std::int32_t(1), + std::string("B"), std::int32_t(2), + std::string("C"), std::int32_t(3), + std::string("D"), std::int32_t(4) + }) + } + }; + + Status status = check_function(func_name, input_types, data_set); + EXPECT_TRUE(status.ok()) << "Function test failed four: " << status.to_string(); + } + + + // Test single map (should return the same map) + { + InputTypeSet input_types = { + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING + }; + TestArray mp_src_array = TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), std::string("B")}); + TestArray src; + src.push_back(mp_src_array); + TestArray mp_dest_array = TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), std::string("B")}); + DataSet data_set = { + { + src, + mp_dest_array + } + }; + any_cast(src[0]); + // Status status = check_function(func_name, input_types, data_set); + // EXPECT_TRUE(status.ok()) << "Function test failed six : " << status.to_string(); + } +} + +TEST(FunctionMapConcatTest, TestWithNULL){ + const std::string func_name = "map_concat"; + // Test with null map (one map is null) + { + // TODO cxr FIX Null + InputTypeSet input_types = { + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING + }; + DataSet data_set = { + { + TestArray({Null(), TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), std::string("B")})}), + Null() + }, + { + TestArray({TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), std::string("B")}), Null()}), + Null() + }, + { + TestArray({Null(), Null()}), + Null() + } + }; + + Status status = check_function(func_name, input_types, data_set); + EXPECT_TRUE(status.ok()) << "Function test failed seven: " << status.to_string(); + } + + // Test with null values + { + InputTypeSet input_types = { + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING + }; + DataSet data_set = { + { + TestArray({ + TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), Null()}), + TestArray({std::int32_t(2), std::string("B"), std::int32_t(3), std::string("C")}) + }), + TestArray({ + std::int32_t(1), std::string("A"), + std::int32_t(2), std::string("B"), + std::int32_t(3), std::string("C") + }) + } + }; + + Status status = check_function(func_name, input_types, data_set); + EXPECT_TRUE(status.ok()) << "Function test failed five: " << status.to_string(); + } +} + +TEST(FunctionMapConcatTest, DISABLED_TestComplexTypes) { + const std::string func_name = "map_concat"; + + // Test with nested types (array as value) + { + InputTypeSet input_types = { + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_ARRAY, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_ARRAY + }; + DataSet data_set = { + { + TestArray({ + TestArray({ + std::string("A"), + TestArray({std::int32_t(1), std::int32_t(2)}), + std::string("B"), + std::int32_t(3) + }), + TestArray({ + std::string("C"), + TestArray({std::int32_t(4), std::int32_t(5)}) + }) + }), + TestArray({ + std::string("A"), TestArray({std::int32_t(1), std::int32_t(2)}), + std::string("B"), std::int32_t(3), + std::string("C"), TestArray({std::int32_t(4), std::int32_t(5)}) + }) + } + }; + + Status status = check_function(func_name, input_types, data_set); + EXPECT_TRUE(status.ok()) << "Function test failed: " << status.to_string(); + } + + // Test with map as value + { + InputTypeSet input_types = { + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT + }; + DataSet data_set = { + { + TestArray({ + TestArray({ + std::string("outer1"), + TestArray({TestArray({std::string("inner1"), std::int32_t(1)})}), + std::string("outer2"), + TestArray({TestArray({std::string("inner2"), std::int32_t(2)})}) + }), + TestArray({ + std::string("outer3"), + TestArray({TestArray({std::string("inner3"), std::int32_t(3)})}) + }) + }), + TestArray({ + std::string("outer1"), TestArray({TestArray({std::string("inner1"), std::int32_t(1)})}), + std::string("outer2"), TestArray({TestArray({std::string("inner2"), std::int32_t(2)})}), + std::string("outer3"), TestArray({TestArray({std::string("inner3"), std::int32_t(3)})}) + }) } }; - // // 处理 check_function 的返回值 Status status = check_function(func_name, input_types, data_set); EXPECT_TRUE(status.ok()) << "Function test failed: " << status.to_string(); } From cef3520de925c2af967462b5716f59c345e85659 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Sun, 7 Dec 2025 17:35:41 +0800 Subject: [PATCH 07/41] fix: function_map_concat_test --- .../vec/function/function_map_concat_test.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/be/test/vec/function/function_map_concat_test.cpp b/be/test/vec/function/function_map_concat_test.cpp index b757158fa3965d..2355ebc78b8394 100644 --- a/be/test/vec/function/function_map_concat_test.cpp +++ b/be/test/vec/function/function_map_concat_test.cpp @@ -98,14 +98,14 @@ TEST(FunctionMapConcatTest, TestEdgeCases) { TestArray({TestArray({}), TestArray({})}), TestArray({}) }, - // { - // TestArray({TestArray({std::int32_t(1), std::string("A")}), TestArray({})}), - // TestArray({std::int32_t(1), std::string("A")}) - // }, - // { - // TestArray({TestArray({}), TestArray({std::int32_t(2), std::string("B")})}), - // TestArray({std::int32_t(2), std::string("B")}) - // } + { + TestArray({TestArray({std::int32_t(1), std::string("A")}), TestArray({})}), + TestArray({std::int32_t(1), std::string("A")}) + }, + { + TestArray({TestArray({}), TestArray({std::int32_t(2), std::string("B")})}), + TestArray({std::int32_t(2), std::string("B")}) + } }; Status status = check_function(func_name, input_types, data_set); From e5c3bd8405b991367cd3ca1b566adf08a3c8c89f Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Sun, 7 Dec 2025 19:10:50 +0800 Subject: [PATCH 08/41] remove: FunctionMapConcatTest.TestComplexTypes --- .../vec/function/function_map_concat_test.cpp | 126 +++++++++--------- be/test/vec/function/function_test_util.h | 1 + 2 files changed, 64 insertions(+), 63 deletions(-) diff --git a/be/test/vec/function/function_map_concat_test.cpp b/be/test/vec/function/function_map_concat_test.cpp index 2355ebc78b8394..7daf9f53db7c53 100644 --- a/be/test/vec/function/function_map_concat_test.cpp +++ b/be/test/vec/function/function_map_concat_test.cpp @@ -262,71 +262,71 @@ TEST(FunctionMapConcatTest, TestWithNULL){ } } -TEST(FunctionMapConcatTest, DISABLED_TestComplexTypes) { - const std::string func_name = "map_concat"; +// TEST(FunctionMapConcatTest, TestComplexTypes) { +// const std::string func_name = "map_concat"; - // Test with nested types (array as value) - { - InputTypeSet input_types = { - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_ARRAY, - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_ARRAY - }; - DataSet data_set = { - { - TestArray({ - TestArray({ - std::string("A"), - TestArray({std::int32_t(1), std::int32_t(2)}), - std::string("B"), - std::int32_t(3) - }), - TestArray({ - std::string("C"), - TestArray({std::int32_t(4), std::int32_t(5)}) - }) - }), - TestArray({ - std::string("A"), TestArray({std::int32_t(1), std::int32_t(2)}), - std::string("B"), std::int32_t(3), - std::string("C"), TestArray({std::int32_t(4), std::int32_t(5)}) - }) - } - }; +// // // Test with nested types (array as value) +// // { +// // InputTypeSet input_types = { +// // PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_ARRAY, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_INT, +// // PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_ARRAY, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_INT +// // }; +// // DataSet data_set = { +// // { +// // TestArray({ +// // TestArray({ +// // std::string("A"), +// // TestArray({std::int32_t(1), std::int32_t(2)}), +// // std::string("B"), +// // std::int32_t(3) +// // }), +// // TestArray({ +// // std::string("C"), +// // TestArray({std::int32_t(4), std::int32_t(5)}) +// // }) +// // }), +// // TestArray({ +// // std::string("A"), TestArray({std::int32_t(1), std::int32_t(2)}), +// // std::string("B"), std::int32_t(3), +// // std::string("C"), TestArray({std::int32_t(4), std::int32_t(5)}) +// // }) +// // } +// // }; - Status status = check_function(func_name, input_types, data_set); - EXPECT_TRUE(status.ok()) << "Function test failed: " << status.to_string(); - } +// // Status status = check_function(func_name, input_types, data_set); +// // EXPECT_TRUE(status.ok()) << "Function test failed 1: " << status.to_string(); +// // } - // Test with map as value - { - InputTypeSet input_types = { - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT, - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT - }; - DataSet data_set = { - { - TestArray({ - TestArray({ - std::string("outer1"), - TestArray({TestArray({std::string("inner1"), std::int32_t(1)})}), - std::string("outer2"), - TestArray({TestArray({std::string("inner2"), std::int32_t(2)})}) - }), - TestArray({ - std::string("outer3"), - TestArray({TestArray({std::string("inner3"), std::int32_t(3)})}) - }) - }), - TestArray({ - std::string("outer1"), TestArray({TestArray({std::string("inner1"), std::int32_t(1)})}), - std::string("outer2"), TestArray({TestArray({std::string("inner2"), std::int32_t(2)})}), - std::string("outer3"), TestArray({TestArray({std::string("inner3"), std::int32_t(3)})}) - }) - } - }; +// // Test with map as value +// { +// InputTypeSet input_types = { +// PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT, +// PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT +// }; +// DataSet data_set = { +// { +// TestArray({ +// TestArray({ +// std::string("outer1"), +// TestArray({TestArray({std::string("inner1"), std::int32_t(1)})}), +// std::string("outer2"), +// TestArray({TestArray({std::string("inner2"), std::int32_t(2)})}) +// }), +// TestArray({ +// std::string("outer3"), +// TestArray({TestArray({std::string("inner3"), std::int32_t(3)})}) +// }) +// }), +// TestArray({ +// std::string("outer1"), TestArray({TestArray({std::string("inner1"), std::int32_t(1)})}), +// std::string("outer2"), TestArray({TestArray({std::string("inner2"), std::int32_t(2)})}), +// std::string("outer3"), TestArray({TestArray({std::string("inner3"), std::int32_t(3)})}) +// }) +// } +// }; - Status status = check_function(func_name, input_types, data_set); - EXPECT_TRUE(status.ok()) << "Function test failed: " << status.to_string(); - } -} +// Status status = check_function(func_name, input_types, data_set); +// EXPECT_TRUE(status.ok()) << "Function test failed 2: " << status.to_string(); +// } +// } } diff --git a/be/test/vec/function/function_test_util.h b/be/test/vec/function/function_test_util.h index d9b2ce1b013bad..04327f48b24f6b 100644 --- a/be/test/vec/function/function_test_util.h +++ b/be/test/vec/function/function_test_util.h @@ -364,6 +364,7 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty // 2. execute function auto get_key_value_type=[&](){ + // cannot used to complex type DataTypeFactory&factory = DataTypeFactory::instance(); DataTypePtr key_type = factory.create_data_type(any_cast(input_types[1]), true); DataTypePtr value_type = factory.create_data_type(any_cast(input_types[2]), true); From 7869361313445a20e1448de9c8b4d9ca5f51fe5d Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Sun, 7 Dec 2025 22:38:23 +0800 Subject: [PATCH 09/41] fix: get_key_value_type --- .../vec/function/function_map_concat_test.cpp | 126 +++++++++--------- be/test/vec/function/function_test_util.h | 15 ++- 2 files changed, 73 insertions(+), 68 deletions(-) diff --git a/be/test/vec/function/function_map_concat_test.cpp b/be/test/vec/function/function_map_concat_test.cpp index 7daf9f53db7c53..a2b80560e13b62 100644 --- a/be/test/vec/function/function_map_concat_test.cpp +++ b/be/test/vec/function/function_map_concat_test.cpp @@ -262,71 +262,71 @@ TEST(FunctionMapConcatTest, TestWithNULL){ } } -// TEST(FunctionMapConcatTest, TestComplexTypes) { -// const std::string func_name = "map_concat"; +TEST(FunctionMapConcatTest, TestComplexTypes) { + const std::string func_name = "map_concat"; -// // // Test with nested types (array as value) -// // { -// // InputTypeSet input_types = { -// // PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_ARRAY, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_INT, -// // PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_ARRAY, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_INT -// // }; -// // DataSet data_set = { -// // { -// // TestArray({ -// // TestArray({ -// // std::string("A"), -// // TestArray({std::int32_t(1), std::int32_t(2)}), -// // std::string("B"), -// // std::int32_t(3) -// // }), -// // TestArray({ -// // std::string("C"), -// // TestArray({std::int32_t(4), std::int32_t(5)}) -// // }) -// // }), -// // TestArray({ -// // std::string("A"), TestArray({std::int32_t(1), std::int32_t(2)}), -// // std::string("B"), std::int32_t(3), -// // std::string("C"), TestArray({std::int32_t(4), std::int32_t(5)}) -// // }) -// // } -// // }; + // // Test with nested types (array as value) + // { + // InputTypeSet input_types = { + // PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_ARRAY, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_INT, + // PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_ARRAY, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_INT + // }; + // DataSet data_set = { + // { + // TestArray({ + // TestArray({ + // std::string("A"), + // TestArray({std::int32_t(1), std::int32_t(2)}), + // std::string("B"), + // std::int32_t(3) + // }), + // TestArray({ + // std::string("C"), + // TestArray({std::int32_t(4), std::int32_t(5)}) + // }) + // }), + // TestArray({ + // std::string("A"), TestArray({std::int32_t(1), std::int32_t(2)}), + // std::string("B"), std::int32_t(3), + // std::string("C"), TestArray({std::int32_t(4), std::int32_t(5)}) + // }) + // } + // }; -// // Status status = check_function(func_name, input_types, data_set); -// // EXPECT_TRUE(status.ok()) << "Function test failed 1: " << status.to_string(); -// // } + // Status status = check_function(func_name, input_types, data_set); + // EXPECT_TRUE(status.ok()) << "Function test failed 1: " << status.to_string(); + // } -// // Test with map as value -// { -// InputTypeSet input_types = { -// PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT, -// PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT -// }; -// DataSet data_set = { -// { -// TestArray({ -// TestArray({ -// std::string("outer1"), -// TestArray({TestArray({std::string("inner1"), std::int32_t(1)})}), -// std::string("outer2"), -// TestArray({TestArray({std::string("inner2"), std::int32_t(2)})}) -// }), -// TestArray({ -// std::string("outer3"), -// TestArray({TestArray({std::string("inner3"), std::int32_t(3)})}) -// }) -// }), -// TestArray({ -// std::string("outer1"), TestArray({TestArray({std::string("inner1"), std::int32_t(1)})}), -// std::string("outer2"), TestArray({TestArray({std::string("inner2"), std::int32_t(2)})}), -// std::string("outer3"), TestArray({TestArray({std::string("inner3"), std::int32_t(3)})}) -// }) -// } -// }; + // Test with map as value + { + InputTypeSet input_types = { + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT + }; + DataSet data_set = { + { + TestArray({ + TestArray({ + std::string("outer1"), + TestArray({TestArray({std::string("inner1"), std::int32_t(1)})}), + std::string("outer2"), + TestArray({TestArray({std::string("inner2"), std::int32_t(2)})}) + }), + TestArray({ + std::string("outer3"), + TestArray({TestArray({std::string("inner3"), std::int32_t(3)})}) + }) + }), + TestArray({ + std::string("outer1"), TestArray({TestArray({std::string("inner1"), std::int32_t(1)})}), + std::string("outer2"), TestArray({TestArray({std::string("inner2"), std::int32_t(2)})}), + std::string("outer3"), TestArray({TestArray({std::string("inner3"), std::int32_t(3)})}) + }) + } + }; -// Status status = check_function(func_name, input_types, data_set); -// EXPECT_TRUE(status.ok()) << "Function test failed 2: " << status.to_string(); -// } -// } + Status status = check_function(func_name, input_types, data_set); + EXPECT_TRUE(status.ok()) << "Function test failed 2: " << status.to_string(); + } +} } diff --git a/be/test/vec/function/function_test_util.h b/be/test/vec/function/function_test_util.h index 04327f48b24f6b..6b91533aa45a71 100644 --- a/be/test/vec/function/function_test_util.h +++ b/be/test/vec/function/function_test_util.h @@ -61,6 +61,7 @@ #include "vec/data_types/data_type_time.h" #include "vec/data_types/data_type_varbinary.h" #include "vec/functions/simple_function_factory.h" +#include "vec/functions/function_helpers.h" namespace doris::vectorized { @@ -364,11 +365,15 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty // 2. execute function auto get_key_value_type=[&](){ - // cannot used to complex type - DataTypeFactory&factory = DataTypeFactory::instance(); - DataTypePtr key_type = factory.create_data_type(any_cast(input_types[1]), true); - DataTypePtr value_type = factory.create_data_type(any_cast(input_types[2]), true); - return std::make_pair(key_type,value_type); + const DataTypeMap*map_type = nullptr; + if (descs[0].data_type->is_nullable()){ + auto*data_type_nullable = assert_cast(descs[0].data_type.get()); + map_type = check_and_get_data_type(data_type_nullable->get_nested_type().get()); + } else { + map_type = check_and_get_data_type(descs[0].data_type.get()); + } + assert(map_type); + return std::make_pair(map_type->get_key_type(),map_type->get_value_type()); }; auto return_type = [&]() { if constexpr (IsDataTypeDecimal) { // decimal From aea2e7f18386c6b5d3a84ab98aa41b38768bfe6a Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Mon, 8 Dec 2025 00:52:33 +0800 Subject: [PATCH 10/41] fix: TestComplexTypes of array --- .../vec/function/function_map_concat_test.cpp | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/be/test/vec/function/function_map_concat_test.cpp b/be/test/vec/function/function_map_concat_test.cpp index a2b80560e13b62..c5890582b55362 100644 --- a/be/test/vec/function/function_map_concat_test.cpp +++ b/be/test/vec/function/function_map_concat_test.cpp @@ -265,37 +265,37 @@ TEST(FunctionMapConcatTest, TestWithNULL){ TEST(FunctionMapConcatTest, TestComplexTypes) { const std::string func_name = "map_concat"; - // // Test with nested types (array as value) - // { - // InputTypeSet input_types = { - // PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_ARRAY, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_INT, - // PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_ARRAY, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_INT - // }; - // DataSet data_set = { - // { - // TestArray({ - // TestArray({ - // std::string("A"), - // TestArray({std::int32_t(1), std::int32_t(2)}), - // std::string("B"), - // std::int32_t(3) - // }), - // TestArray({ - // std::string("C"), - // TestArray({std::int32_t(4), std::int32_t(5)}) - // }) - // }), - // TestArray({ - // std::string("A"), TestArray({std::int32_t(1), std::int32_t(2)}), - // std::string("B"), std::int32_t(3), - // std::string("C"), TestArray({std::int32_t(4), std::int32_t(5)}) - // }) - // } - // }; + // Test with nested types (array as value) + { + InputTypeSet input_types = { + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_ARRAY, PrimitiveType::TYPE_INT, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_ARRAY, PrimitiveType::TYPE_INT + }; + DataSet data_set = { + { + TestArray({ + TestArray({ + std::string("A"), + TestArray({std::int32_t(1), std::int32_t(2)}), + std::string("B"), + TestArray({std::int32_t(3), std::int32_t(5)}) + }), + TestArray({ + std::string("C"), + TestArray({std::int32_t(4), std::int32_t(5)}) + }) + }), + TestArray({ + std::string("A"), TestArray({std::int32_t(1), std::int32_t(2)}), + std::string("B"), TestArray({std::int32_t(3), std::int32_t(5)}), + std::string("C"), TestArray({std::int32_t(4), std::int32_t(5)}) + }) + } + }; - // Status status = check_function(func_name, input_types, data_set); - // EXPECT_TRUE(status.ok()) << "Function test failed 1: " << status.to_string(); - // } + Status status = check_function(func_name, input_types, data_set); + EXPECT_TRUE(status.ok()) << "Function test failed 1: " << status.to_string(); + } // Test with map as value { From 71a6cb72c434b2c3d5263c190ae0895fe8974049 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Mon, 8 Dec 2025 22:00:14 +0800 Subject: [PATCH 11/41] fix: check_function_all_arg_comb --- .../vec/function/function_map_concat_test.cpp | 34 +++++++------------ 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/be/test/vec/function/function_map_concat_test.cpp b/be/test/vec/function/function_map_concat_test.cpp index c5890582b55362..c6e80bbd52d6c6 100644 --- a/be/test/vec/function/function_map_concat_test.cpp +++ b/be/test/vec/function/function_map_concat_test.cpp @@ -27,8 +27,8 @@ TEST(FunctionMapConcatTest, TestBase) { } }; - Status status = check_function(func_name, input_types, data_set); - EXPECT_TRUE(status.ok()) << "Function test failed: " << status.to_string(); + check_function_all_arg_comb(func_name, input_types, data_set); + // EXPECT_TRUE(status.ok()) << "Function test failed: " << status.to_string(); } { // two maps concatenation with string keys @@ -48,8 +48,7 @@ TEST(FunctionMapConcatTest, TestBase) { }) } }; - Status status = check_function(func_name, input_types, data_set); - EXPECT_TRUE(status.ok()) << "Function test failed: " << status.to_string(); + check_function_all_arg_comb(func_name, input_types, data_set); } { // two maps concatenation with string keys @@ -79,8 +78,7 @@ TEST(FunctionMapConcatTest, TestBase) { }) } }; - Status status = check_function(func_name, input_types, data_set); - EXPECT_TRUE(status.ok()) << "Function test failed: " << status.to_string(); + check_function_all_arg_comb(func_name, input_types, data_set); } } @@ -108,8 +106,7 @@ TEST(FunctionMapConcatTest, TestEdgeCases) { } }; - Status status = check_function(func_name, input_types, data_set); - EXPECT_TRUE(status.ok()) << "Function test failed one: " << status.to_string(); + check_function_all_arg_comb(func_name, input_types, data_set); } // Test key conflicts (later map should override earlier) @@ -132,8 +129,7 @@ TEST(FunctionMapConcatTest, TestEdgeCases) { } }; - Status status = check_function(func_name, input_types, data_set); - EXPECT_TRUE(status.ok()) << "Function test failed two: " << status.to_string(); + check_function_all_arg_comb(func_name, input_types, data_set); } // Test multiple maps (more than 2) @@ -158,8 +154,7 @@ TEST(FunctionMapConcatTest, TestEdgeCases) { } }; - Status status = check_function(func_name, input_types, data_set); - EXPECT_TRUE(status.ok()) << "Function test failed three: " << status.to_string(); + check_function_all_arg_comb(func_name, input_types, data_set); } // Test different value types @@ -183,8 +178,7 @@ TEST(FunctionMapConcatTest, TestEdgeCases) { } }; - Status status = check_function(func_name, input_types, data_set); - EXPECT_TRUE(status.ok()) << "Function test failed four: " << status.to_string(); + check_function_all_arg_comb(func_name, input_types, data_set); } @@ -233,8 +227,7 @@ TEST(FunctionMapConcatTest, TestWithNULL){ } }; - Status status = check_function(func_name, input_types, data_set); - EXPECT_TRUE(status.ok()) << "Function test failed seven: " << status.to_string(); + check_function_all_arg_comb(func_name, input_types, data_set); } // Test with null values @@ -257,8 +250,7 @@ TEST(FunctionMapConcatTest, TestWithNULL){ } }; - Status status = check_function(func_name, input_types, data_set); - EXPECT_TRUE(status.ok()) << "Function test failed five: " << status.to_string(); + check_function_all_arg_comb(func_name, input_types, data_set); } } @@ -293,8 +285,7 @@ TEST(FunctionMapConcatTest, TestComplexTypes) { } }; - Status status = check_function(func_name, input_types, data_set); - EXPECT_TRUE(status.ok()) << "Function test failed 1: " << status.to_string(); + check_function_all_arg_comb(func_name, input_types, data_set); } // Test with map as value @@ -325,8 +316,7 @@ TEST(FunctionMapConcatTest, TestComplexTypes) { } }; - Status status = check_function(func_name, input_types, data_set); - EXPECT_TRUE(status.ok()) << "Function test failed 2: " << status.to_string(); + check_function_all_arg_comb(func_name, input_types, data_set); } } } From d42447c30d8e75c1b6971934c74473edbf1235c8 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Thu, 11 Dec 2025 01:56:47 +0800 Subject: [PATCH 12/41] feat: Test with double, float and decimalv2 --- .../vec/function/function_map_concat_test.cpp | 85 ++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/be/test/vec/function/function_map_concat_test.cpp b/be/test/vec/function/function_map_concat_test.cpp index c6e80bbd52d6c6..a61418db9646d4 100644 --- a/be/test/vec/function/function_map_concat_test.cpp +++ b/be/test/vec/function/function_map_concat_test.cpp @@ -207,7 +207,6 @@ TEST(FunctionMapConcatTest, TestWithNULL){ const std::string func_name = "map_concat"; // Test with null map (one map is null) { - // TODO cxr FIX Null InputTypeSet input_types = { PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING @@ -318,5 +317,89 @@ TEST(FunctionMapConcatTest, TestComplexTypes) { check_function_all_arg_comb(func_name, input_types, data_set); } + + // Test with double as value + { + InputTypeSet input_types = { + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_DOUBLE, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_DOUBLE + }; + DataSet data_set = { + { + TestArray({ + TestArray({ + std::string("key1"), 1.5, + std::string("key2"), 2.7 + }), + TestArray({ + std::string("key3"), 3.9 + }) + }), + TestArray({ + std::string("key1"), 1.5, + std::string("key2"), 2.7, + std::string("key3"), 3.9 + }) + } + }; + + check_function_all_arg_comb(func_name, input_types, data_set); + } + + // Test with float as value + { + InputTypeSet input_types = { + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_FLOAT, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_FLOAT + }; + DataSet data_set = { + { + TestArray({ + TestArray({ + std::string("key1"), 1.5f, + std::string("key2"), 2.7f + }), + TestArray({ + std::string("key3"), 3.9f + }) + }), + TestArray({ + std::string("key1"), 1.5f, + std::string("key2"), 2.7f, + std::string("key3"), 3.9f + }) + } + }; + + check_function_all_arg_comb(func_name, input_types, data_set); + } + + // Test with decimalv2 as value + { + InputTypeSet input_types = { + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_DECIMALV2, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_DECIMALV2 + }; + DataSet data_set = { + { + TestArray({ + TestArray({ + std::string("key1"), ut_type::DECIMALV2(1.5), + std::string("key2"), ut_type::DECIMALV2(2.7) + }), + TestArray({ + std::string("key3"), ut_type::DECIMALV2(3.9) + }) + }), + TestArray({ + std::string("key1"), ut_type::DECIMALV2(1.5), + std::string("key2"), ut_type::DECIMALV2(2.7), + std::string("key3"), ut_type::DECIMALV2(3.9) + }) + } + }; + + check_function_all_arg_comb(func_name, input_types, data_set); + } } } From fc221f6adc87da45127ef86c23627c72ad54b9f1 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Fri, 12 Dec 2025 21:53:01 +0800 Subject: [PATCH 13/41] feat: Test single map --- be/test/vec/function/function_map_concat_test.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/be/test/vec/function/function_map_concat_test.cpp b/be/test/vec/function/function_map_concat_test.cpp index a61418db9646d4..c6e61fa5925ff3 100644 --- a/be/test/vec/function/function_map_concat_test.cpp +++ b/be/test/vec/function/function_map_concat_test.cpp @@ -197,9 +197,8 @@ TEST(FunctionMapConcatTest, TestEdgeCases) { mp_dest_array } }; - any_cast(src[0]); - // Status status = check_function(func_name, input_types, data_set); - // EXPECT_TRUE(status.ok()) << "Function test failed six : " << status.to_string(); + + check_function_all_arg_comb(func_name, input_types, data_set); } } From 053760a411f97dfd790c68d21440ee06ab0b8ec5 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Fri, 12 Dec 2025 21:54:41 +0800 Subject: [PATCH 14/41] feat: extend map_concat to handle struct-typed values --- .../functions/scalar/MapConcat.java | 55 ++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java index b877b631741bc4..1ab2a27aa4f3db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java @@ -17,26 +17,37 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; +import org.apache.doris.nereids.types.DataType; import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.MapType; import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.TypeCoercionUtils; +import org.apache.doris.thrift.BackendService.AsyncProcessor.publish_cluster_state; import org.apache.doris.nereids.types.MapType; +import com.amazonaws.services.glue.model.Datatype; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import java.security.cert.PKIXRevocationChecker.Option; +import java.util.ArrayList; import java.util.List; +import java.util.Optional; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; /** * ScalarFunction 'map_concat' */ public class MapConcat extends ScalarFunction implements ExplicitlyCastableSignature, PropagateNullable { + private static final Logger LOG = LogManager.getLogger(MapConcat.class); public static final List SIGNATURES = ImmutableList.of( FunctionSignature @@ -64,6 +75,38 @@ public MapConcat withChildren(List children) { return new MapConcat(getFunctionParams(children)); } + @Override + public DataType getDataType() { + if (arity() >= 1){ + List keyTypes = new ArrayList<>(); + List valueTypes = new ArrayList<>(); + for (int i = 0; i < children.size(); i++){ + DataType argType = children.get(i).getDataType(); + if (!(argType instanceof MapType)){ + return MapType.SYSTEM_DEFAULT; + } + MapType mapType = (MapType) argType; + keyTypes.add(mapType.getKeyType()); + valueTypes.add(mapType.getValueType()); + } + + Optional commonKeyType = TypeCoercionUtils.findWiderCommonType(keyTypes, true, true); + Optional commonValueType = TypeCoercionUtils.findWiderCommonType(valueTypes, true, true); + if (commonKeyType.isPresent() && commonValueType.isPresent()){ + DataType keyType = commonKeyType.get(); + DataType valueType = commonValueType.get(); + return MapType.of(keyType, valueType); + } + if (!commonKeyType.isPresent()) { + throw new AnalysisException("mapconcat cannot find the common key type of " + this.toSql()); + } + if (!commonValueType.isPresent()) { + throw new AnalysisException("mapconcat cannot find the common value type of " + this.toSql()); + } + } + throw new RuntimeException("unreachable"); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitMapConcat(this, context); @@ -71,6 +114,16 @@ public R accept(ExpressionVisitor visitor, C context) { @Override public List getSignatures() { - return SIGNATURES; + if (arity() == 0){ + return SIGNATURES; + } else { + List signatures = ImmutableList.of( + FunctionSignature.of(getDataType(), + children.stream() + .map(ExpressionTrait::getDataType) + .collect(ImmutableList.toImmutableList()) + )); + return signatures; + } } } From 3c585c6a7a1cfcdf4be2b5d8c5a778ea1e7b895b Mon Sep 17 00:00:00 2001 From: nagisa <1434936049@qq.com> Date: Sat, 13 Dec 2025 00:58:57 +0800 Subject: [PATCH 15/41] fix: compile error --- .../nereids/trees/expressions/functions/scalar/MapConcat.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java index 1ab2a27aa4f3db..a1ffd0a6cb56da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; From 174cab11f47516126e634b87ca3e0ff2b841c17a Mon Sep 17 00:00:00 2001 From: nagisa <1434936049@qq.com> Date: Sun, 14 Dec 2025 03:27:13 +0800 Subject: [PATCH 16/41] feat: test_map_concat --- .../data/function_p0/test_map_concat.out | 7 + .../suites/function_p0/test_map_concat.groovy | 233 ++++++++++++++++++ 2 files changed, 240 insertions(+) create mode 100644 regression-test/data/function_p0/test_map_concat.out create mode 100644 regression-test/suites/function_p0/test_map_concat.groovy diff --git a/regression-test/data/function_p0/test_map_concat.out b/regression-test/data/function_p0/test_map_concat.out new file mode 100644 index 00000000000000..c6cc8341a03d68 --- /dev/null +++ b/regression-test/data/function_p0/test_map_concat.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 {"a":"apple", "b":"banana", "extra_key":"extra_value"} +2 {"c":"cherry", "extra_key":"extra_value"} +3 {"extra_key":"extra_value"} +4 \N + diff --git a/regression-test/suites/function_p0/test_map_concat.groovy b/regression-test/suites/function_p0/test_map_concat.groovy new file mode 100644 index 00000000000000..860b4ed845f2c4 --- /dev/null +++ b/regression-test/suites/function_p0/test_map_concat.groovy @@ -0,0 +1,233 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_map_concat") { + // Enable Nereids planner for testing + sql """ set enable_nereids_planner=true; """ + sql """ set enable_fallback_to_original_planner=false; """ + + def testTable = "test_map_concat_table" + + try { + // Drop table if exists + sql """ + drop table if exists ${testTable}; + """ + + // Create test table with MAP columns + sql """ + CREATE TABLE ${testTable} ( + id INT, + map1 MAP, + map2 MAP, + map3 MAP + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + // Insert test data + sql """ + insert into ${testTable} values + (1, {'a': 'apple', 'b': 'banana'}, {'x': 10, 'y': 20}, {1: 'one', 2: 'two'}), + (2, {'c': 'cherry'}, {'z': 30}, {3: 'three'}), + (3, {}, {}, {}), + (4, NULL, NULL, NULL); + """ + + // ============================================ + // Group 1: Basic functionality tests + // ============================================ + + // Test 1.1: Basic map_concat with two maps (all rows) + qt_sql """ + select + id, + map_concat(map1, {'extra_key': 'extra_value'}) as merged_map + from ${testTable} + order by id; + """ + + // Test 1.2: Concatenating three columns from table + qt_sql """ + select + id, + map_concat(map1, map2, map3) as all_maps_merged + from ${testTable} + where id = 1 + order by id; + """ + + // Test 1.3: Concatenating with literal maps (no table dependency) + qt_sql """ + select + map_concat( + {'a': 'apple'}, + {'b': 'banana'}, + {'c': 'cherry'} + ) as literal_maps_merged; + """ + + // ============================================ + // Group 2: Edge cases and special values + // ============================================ + + // Test 2.1: Concatenating with empty maps + qt_sql """ + select + id, + map_concat(map1, {}) as merged_with_empty1, + map_concat({}, map2) as merged_with_empty2, + map_concat({}, {}) as empty_with_empty + from ${testTable} + where id = 3 + order by id; + """ + + // Test 2.2: Concatenating with NULL maps + qt_sql """ + select + id, + map_concat(map1, NULL) as map_with_null, + map_concat(NULL, map2) as null_with_map, + map_concat(NULL, NULL) as null_with_null + from ${testTable} + where id = 4 + order by id; + """ + + // Test 2.3: Mixed NULL and empty maps + qt_sql """ + select + map_concat(NULL, {}) as null_with_empty, + map_concat({}, NULL) as empty_with_null; + """ + + // ============================================ + // Group 3: Key conflict and overwrite behavior + // ============================================ + + // Test 3.1: Key conflict (later value overwrites earlier) + qt_sql """ + select + map_concat( + {'a': 'apple', 'b': 'banana'}, + {'b': 'blueberry', 'c': 'cherry'} + ) as conflict_resolution; + """ + + // Test 3.2: Multiple key conflicts across three maps + qt_sql """ + select + map_concat( + {'a': 'first', 'b': 'first'}, + {'b': 'second', 'c': 'second'}, + {'c': 'third', 'd': 'third'} + ) as multi_conflict; + """ + + // ============================================ + // Group 4: Function composition and expressions + // ============================================ + + // Test 4.1: Using map_concat in expressions with map_size + qt_sql """ + select + id, + map_size(map_concat(map1, map1)) as self_concat_size, + map_size(map_concat(map1, map2)) as two_maps_size + from ${testTable} + where id = 1 + order by id; + """ + + // Test 4.2: Nested map_concat operations + qt_sql """ + select + map_concat( + map_concat({'a': 1}, {'b': 2}), + map_concat({'c': 3}, {'d': 4}) + ) as nested_concat; + """ + + // Test 4.3: map_concat with other map functions + qt_sql """ + select + map_keys(map_concat({'x': 10}, {'y': 20})) as keys_result, + map_values(map_concat({'x': 10}, {'y': 20})) as values_result; + """ + + // ============================================ + // Group 5: Type handling and error cases + // ============================================ + + // Test 5.1: Concatenating maps with same value types (should work) + qt_sql """ + select + map_concat( + CAST({'a': '1'} AS MAP), + CAST({'b': '2'} AS MAP) + ) as same_type_maps; + """ + + // Test 5.2: Testing with different key types (if supported) + qt_sql """ + select + map_concat( + {1: 'one', 2: 'two'}, + {3: 'three', 4: 'four'} + ) as int_key_maps; + """ + + // Test 5.3: map_concat with many parameters (stress test) + qt_sql """ + select + map_concat( + {'p1': 'v1'}, {'p2': 'v2'}, {'p3': 'v3'}, + {'p4': 'v4'}, {'p5': 'v5'}, {'p6': 'v6'}, + {'p7': 'v7'}, {'p8': 'v8'}, {'p9': 'v9'} + ) as many_maps; + """ + + // Test 5.4: Error case - type mismatch (expected to fail) + try { + qt_sql """ + select + map_concat( + CAST({'a': 1} AS MAP), + CAST({'b': '2'} AS MAP) + ) as type_mismatch; + """ + println "WARNING: Type mismatch test passed unexpectedly" + } catch (Exception e) { + println "Test 5.4 expected to fail for type mismatch: ${e.message}" + } + + } finally { + // Clean up + try { + sql """ + drop table if exists ${testTable}; + """ + } catch (Exception e) { + println "Error cleaning up table ${testTable}: ${e.message}" + } + } +} From 65de5e5aaaefba1664574c102ce0596b6953b4b4 Mon Sep 17 00:00:00 2001 From: nagisa <1434936049@qq.com> Date: Sun, 14 Dec 2025 03:28:38 +0800 Subject: [PATCH 17/41] fix: handle null values in arguments to prevent crash --- .../functions/scalar/MapConcat.java | 28 +++++++----- .../doris/nereids/util/TypeCoercionUtils.java | 44 +++++++++++++++++++ 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java index a1ffd0a6cb56da..f5fbbf172f613d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java @@ -17,26 +17,23 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; -import org.apache.doris.nereids.types.DataType; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.MapType; +import org.apache.doris.nereids.types.NullType; import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.TypeCoercionUtils; -import org.apache.doris.thrift.BackendService.AsyncProcessor.publish_cluster_state; -import org.apache.doris.nereids.types.MapType; -import com.amazonaws.services.glue.model.Datatype; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import java.security.cert.PKIXRevocationChecker.Option; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -84,26 +81,33 @@ public DataType getDataType() { for (int i = 0; i < children.size(); i++){ DataType argType = children.get(i).getDataType(); if (!(argType instanceof MapType)){ - return MapType.SYSTEM_DEFAULT; + if (!(argType instanceof NullType)){ + throw new AnalysisException("mapconcat function cannot process non-map and non-null child elements. Invalid SQL: " + this.toSql()); + } + continue; } MapType mapType = (MapType) argType; keyTypes.add(mapType.getKeyType()); valueTypes.add(mapType.getValueType()); } + if (keyTypes.isEmpty() && valueTypes.isEmpty()) { + return MapType.of(NullType.INSTANCE, NullType.INSTANCE); + } + Optional commonKeyType = TypeCoercionUtils.findWiderCommonType(keyTypes, true, true); Optional commonValueType = TypeCoercionUtils.findWiderCommonType(valueTypes, true, true); - if (commonKeyType.isPresent() && commonValueType.isPresent()){ - DataType keyType = commonKeyType.get(); - DataType valueType = commonValueType.get(); - return MapType.of(keyType, valueType); - } + if (!commonKeyType.isPresent()) { throw new AnalysisException("mapconcat cannot find the common key type of " + this.toSql()); } if (!commonValueType.isPresent()) { throw new AnalysisException("mapconcat cannot find the common value type of " + this.toSql()); } + + DataType keyType = commonKeyType.get(); + DataType valueType = commonValueType.get(); + return MapType.of(keyType, valueType); } throw new RuntimeException("unreachable"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index e02a292567470b..d46e48147a0691 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -49,6 +49,7 @@ import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; import org.apache.doris.nereids.trees.expressions.functions.scalar.Array; import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateMap; +import org.apache.doris.nereids.trees.expressions.functions.scalar.MapConcat; import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; @@ -710,10 +711,53 @@ public static Expression processBoundFunction(BoundFunction boundFunction) { return processCreateMap((CreateMap) boundFunction); } + if (boundFunction instanceof MapConcat) { + return processMapConcat((MapConcat) boundFunction); + } + // type coercion return implicitCastInputTypes(boundFunction, boundFunction.expectedInputTypes()); } + private static Expression processMapConcat(MapConcat mapConcat) { + List keyTypes = new ArrayList<>(); + List valueTypes = new ArrayList<>(); + List children = mapConcat.children(); + for (int i = 0; i < children.size(); i++){ + DataType argType = children.get(i).getDataType(); + if (!(argType instanceof MapType)){ + if (!(argType instanceof NullType)){ + throw new AnalysisException("mapconcat function cannot process non-map and non-null child elements. Invalid SQL: " + mapConcat.toSql()); + } + continue; + } + MapType mapType = (MapType) argType; + keyTypes.add(mapType.getKeyType()); + valueTypes.add(mapType.getValueType()); + } + Optional commonKeyType = TypeCoercionUtils.findWiderCommonType(keyTypes, true, true); + Optional commonValueType = TypeCoercionUtils.findWiderCommonType(valueTypes, true, true); + if (!commonKeyType.isPresent()) { + throw new AnalysisException("mapconcat cannot find the common key type of " + mapConcat.toSql()); + } + if (!commonValueType.isPresent()) { + throw new AnalysisException("mapconcat cannot find the common value type of " + mapConcat.toSql()); + } + DataType keyType = commonKeyType.get(); + DataType valueType = commonValueType.get(); + DataType targetMapType = MapType.of(keyType, valueType); + ImmutableList.Builder newChildren = ImmutableList.builder(); + for (int i = 0; i < mapConcat.arity(); i++) { + DataType argType = children.get(i).getDataType(); + if (!(argType instanceof MapType)){ + newChildren.add(mapConcat.child(i)); + } else { + newChildren.add(castIfNotSameType(mapConcat.child(i), targetMapType)); + } + } + return mapConcat.withChildren(newChildren.build()); + } + private static Expression processCreateMap(CreateMap createMap) { if (createMap.arity() == 0) { return new MapLiteral(); From d428f86a224ba44b4ffbb26f1232b9a83b220779 Mon Sep 17 00:00:00 2001 From: nagisa <1434936049@qq.com> Date: Sun, 14 Dec 2025 19:23:22 +0800 Subject: [PATCH 18/41] fix: test_map_concat.out --- .../data/function_p0/test_map_concat.out | 54 +++++++++++++++++++ .../suites/function_p0/test_map_concat.groovy | 24 +++------ 2 files changed, 61 insertions(+), 17 deletions(-) diff --git a/regression-test/data/function_p0/test_map_concat.out b/regression-test/data/function_p0/test_map_concat.out index c6cc8341a03d68..ab961057e37327 100644 --- a/regression-test/data/function_p0/test_map_concat.out +++ b/regression-test/data/function_p0/test_map_concat.out @@ -5,3 +5,57 @@ 3 {"extra_key":"extra_value"} 4 \N +-- !sql -- +1 {"a":"apple", "b":"banana", "x":"10", "y":"20", "1":"one", "2":"two"} +2 {"c":"cherry", "z":"30", "3":"three"} +3 {} +4 \N + +-- !sql -- +{"a":"apple", "b":"banana", "c":"cherry"} + +-- !sql -- +1 {"a":"apple", "b":"banana"} {"x":10, "y":20} {} +2 {"c":"cherry"} {"z":30} {} +3 {} {} {} +4 \N \N {} + +-- !sql -- +1 \N \N \N +2 \N \N \N +3 \N \N \N +4 \N \N \N + +-- !sql -- +\N \N + +-- !sql -- +{"a":"apple", "b":"blueberry", "c":"cherry"} + +-- !sql -- +{"a":"first", "b":"second", "c":"third", "d":"third"} + +-- !sql -- +1 2 4 +2 1 2 +3 0 0 +4 \N \N + +-- !sql -- +{"a":1, "b":2, "c":3, "d":4} + +-- !sql -- +["x", "y"] [10, 20] + +-- !sql -- +{"a":"1", "b":"2"} + +-- !sql -- +{1:"one", 2:"two", 3:"three", 4:"four"} + +-- !sql -- +{"p1":"v1", "p2":"v2", "p3":"v3", "p4":"v4", "p5":"v5", "p6":"v6", "p7":"v7", "p8":"v8", "p9":"v9"} + +-- !sql -- +{"a":"1", "b":"2"} + diff --git a/regression-test/suites/function_p0/test_map_concat.groovy b/regression-test/suites/function_p0/test_map_concat.groovy index 860b4ed845f2c4..1720526d20da24 100644 --- a/regression-test/suites/function_p0/test_map_concat.groovy +++ b/regression-test/suites/function_p0/test_map_concat.groovy @@ -71,7 +71,6 @@ suite("test_map_concat") { id, map_concat(map1, map2, map3) as all_maps_merged from ${testTable} - where id = 1 order by id; """ @@ -97,7 +96,6 @@ suite("test_map_concat") { map_concat({}, map2) as merged_with_empty2, map_concat({}, {}) as empty_with_empty from ${testTable} - where id = 3 order by id; """ @@ -109,7 +107,6 @@ suite("test_map_concat") { map_concat(NULL, map2) as null_with_map, map_concat(NULL, NULL) as null_with_null from ${testTable} - where id = 4 order by id; """ @@ -154,7 +151,6 @@ suite("test_map_concat") { map_size(map_concat(map1, map1)) as self_concat_size, map_size(map_concat(map1, map2)) as two_maps_size from ${testTable} - where id = 1 order by id; """ @@ -207,19 +203,13 @@ suite("test_map_concat") { """ // Test 5.4: Error case - type mismatch (expected to fail) - try { - qt_sql """ - select - map_concat( - CAST({'a': 1} AS MAP), - CAST({'b': '2'} AS MAP) - ) as type_mismatch; - """ - println "WARNING: Type mismatch test passed unexpectedly" - } catch (Exception e) { - println "Test 5.4 expected to fail for type mismatch: ${e.message}" - } - + qt_sql """ + select + map_concat( + CAST({'a': 1} AS MAP), + CAST({'b': '2'} AS MAP) + ) as type_mismatch; + """ } finally { // Clean up try { From 41c7f69e25ef6b8ba25458a934aaaed662e1cd29 Mon Sep 17 00:00:00 2001 From: nagisa <1434936049@qq.com> Date: Sun, 14 Dec 2025 21:26:47 +0800 Subject: [PATCH 19/41] test: expand test coverage for mapconcat function --- .../data/function_p0/test_map_concat.out | 82 ++++++ .../suites/function_p0/test_map_concat.groovy | 243 ++++++++++++++++++ 2 files changed, 325 insertions(+) diff --git a/regression-test/data/function_p0/test_map_concat.out b/regression-test/data/function_p0/test_map_concat.out index ab961057e37327..c8442e95d9fc74 100644 --- a/regression-test/data/function_p0/test_map_concat.out +++ b/regression-test/data/function_p0/test_map_concat.out @@ -59,3 +59,85 @@ -- !sql -- {"a":"1", "b":"2"} +-- !sql -- +{"single":"argument"} + +-- !sql -- +{"a":"value", "b":null, "c":"another", "d":null} + +-- !sql -- +\N + +-- !sql -- +{"tiny":127, "small":32767, "int":2147483647} + +-- !sql -- +{"flag1":1, "flag2":0, "flag3":1} + +-- !sql -- +{"date1":"2023-01-01", "date2":"2023-12-31"} + +-- !sql -- +{"arr1":[1, 2, 3], "arr2":[4, 5, 6]} + +-- !sql -- +{"k01":"v01", "k02":"v02", "k03":"v03", "k04":"v04", "k05":"v05", "k06":"v06", "k07":"v07", "k08":"v08", "k09":"v09", "k10":"v10", "k11":"v11", "k12":"v12"} + +-- !sql -- +{"a":1, "b":2, "c":3, "d":4, "e":5, "f":6} + +-- !sql -- +true false + +-- !sql -- +1 {"a":"apple", "b":"banana", "x":"10", "y":"20"} +2 {"c":"cherry", "z":"30"} + +-- !sql -- +["z", "a", "m"] ["a", "z", "m"] + +-- !sql -- +[{"key":"a", "value":1}, {"key":"b", "value":2}, {"key":"c", "value":3}, {"key":"d", "value":4}] + +-- !sql -- +true false + +-- !sql -- +{"中文键":"中文值", "key with emoji 🔥":"value with emoji 🚀", "key with accents café":"value with accents naïve"} + +-- !sql -- +{"English key":"value1", "日本語キー":"値1", "한국어 키":"값1", "русский ключ":"значение1"} + +-- !sql -- +{"a":1, "b":2, "c":3, "d":4, "e":5, "f":6, "g":7, "h":8, "i":9, "j":10} + +-- !sql -- +{"1":"one", "2":"two", "3":"three", "4":"four", "five":"5", "six":"6", "seven":"7", "eight":"8"} + +-- !sql -- +1 {"static":"value", "a":"apple", "b":"banana"} +2 {"static":"value", "c":"cherry"} +3 {"static":"value"} +4 \N + +-- !sql -- +1 {"odd":"true", "id":"1"} +2 {"even":"true", "id":"2"} +3 {"odd":"true", "id":"3"} +4 {"even":"true", "id":"4"} + +-- !sql -- +{"key with "double quotes"":"value with 'single quotes'", "key with \\backslashes\\":"value with \\n newline", "key with , comma":"value with ; semicolon", "key with {} braces":"value with [] brackets"} + +-- !sql -- +{"empty value":"", "":"another empty key", "key":""} + +-- !sql -- +{"very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_":"very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_", "another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_":"another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_"} + +-- !sql -- +{"a":"apple", "b":"banana", "x":"10", "y":"20"} +{"c":"cherry", "z":"30"} +{} +{} + diff --git a/regression-test/suites/function_p0/test_map_concat.groovy b/regression-test/suites/function_p0/test_map_concat.groovy index 1720526d20da24..92579d03f24a18 100644 --- a/regression-test/suites/function_p0/test_map_concat.groovy +++ b/regression-test/suites/function_p0/test_map_concat.groovy @@ -210,6 +210,249 @@ suite("test_map_concat") { CAST({'b': '2'} AS MAP) ) as type_mismatch; """ + + // ============================================ + // Group 6: Additional edge cases and extended testing + // ============================================ + + // Test 6.1: Single argument (should return the same map) + qt_sql """ + select + map_concat({'single': 'argument'}) as single_argument; + """ + + // Test 6.2: Map with NULL values inside + qt_sql """ + select + map_concat( + {'a': 'value', 'b': NULL}, + {'c': 'another', 'd': NULL} + ) as maps_with_null_values; + """ + + // Test 6.3: Mixed NULL maps and empty maps + qt_sql """ + select + map_concat(NULL, {'a': 1}, {}, NULL, {'b': 2}) as mixed_nulls_empties; + """ + + // Test 6.4: More numeric types testing + qt_sql """ + select + map_concat( + CAST({'tiny': 127} AS MAP), + CAST({'small': 32767} AS MAP), + CAST({'int': 2147483647} AS MAP) + ) as more_numeric_types; + """ + + // Test 6.5: Boolean type values + qt_sql """ + select + map_concat( + {'flag1': true, 'flag2': false}, + {'flag3': true} + ) as boolean_values; + """ + + // Test 6.6: Date type values + qt_sql """ + select + map_concat( + {'date1': DATE '2023-01-01'}, + {'date2': DATE '2023-12-31'} + ) as date_values; + """ + + // Test 6.7: Array as map values + qt_sql """ + select + map_concat( + {'arr1': [1, 2, 3]}, + {'arr2': [4, 5, 6]} + ) as array_values; + """ + + // Test 6.8: Very many parameters (more than 9) + qt_sql """ + select + map_concat( + {'k01': 'v01'}, {'k02': 'v02'}, {'k03': 'v03'}, + {'k04': 'v04'}, {'k05': 'v05'}, {'k06': 'v06'}, + {'k07': 'v07'}, {'k08': 'v08'}, {'k09': 'v09'}, + {'k10': 'v10'}, {'k11': 'v11'}, {'k12': 'v12'} + ) as many_parameters; + """ + + // Test 6.9: Nested map_concat operations (deep nesting) + qt_sql """ + select + map_concat( + map_concat({'a': 1}, {'b': 2}), + map_concat( + map_concat({'c': 3}, {'d': 4}), + map_concat({'e': 5}, {'f': 6}) + ) + ) as deeply_nested; + """ + + // Test 6.10: map_concat with map_contains_key combination + qt_sql """ + select + map_contains_key( + map_concat({'a': 1, 'b': 2}, {'c': 3, 'd': 4}), + 'c' + ) as contains_key_c, + map_contains_key( + map_concat({'a': 1, 'b': 2}, {'c': 3, 'd': 4}), + 'e' + ) as contains_key_e; + """ + + // Test 6.11: map_concat in WHERE clause + qt_sql """ + select + id, + map_concat(map1, map2) as merged + from ${testTable} + where map_size(map_concat(map1, map2)) > 0 + order by id; + """ + + // Test 6.12: Key order consistency test + qt_sql """ + select + map_keys(map_concat({'z': 1, 'a': 2}, {'m': 3})) as keys_order1, + map_keys(map_concat({'a': 2, 'z': 1}, {'m': 3})) as keys_order2; + """ + + // ============================================ + // Group 7: Advanced edge cases - interface, charset, and nested scenarios + // ============================================ + + // Test 7.1: Interface-related test - map_concat with map_entries + qt_sql """ + select + map_entries(map_concat({'a': 1, 'b': 2}, {'c': 3, 'd': 4})) as entries_result; + """ + + // Test 7.2: Interface-related test - map_concat with map_contains_value + qt_sql """ + select + map_contains_value(map_concat({'a': 1, 'b': 2}, {'c': 3, 'd': 4}), 3) as contains_value_3, + map_contains_value(map_concat({'a': 1, 'b': 2}, {'c': 3, 'd': 4}), 5) as contains_value_5; + """ + + // Test 7.3: Different charset test - UTF-8 special characters + qt_sql """ + select + map_concat( + {'中文键': '中文值', 'key with emoji 🔥': 'value with emoji 🚀'}, + {'key with accents café': 'value with accents naïve'} + ) as utf8_charset_test; + """ + + // Test 7.4: Different charset test - mixed language keys + qt_sql """ + select + map_concat( + {'English key': 'value1', '日本語キー': '値1'}, + {'한국어 키': '값1', 'русский ключ': 'значение1'} + ) as mixed_language_test; + """ + + // Test 7.5: Nested map_concat - complex nesting + qt_sql """ + select + map_concat( + map_concat( + map_concat({'a': 1}, {'b': 2}), + map_concat({'c': 3}, {'d': 4}) + ), + map_concat( + map_concat( + map_concat({'e': 5}, {'f': 6}), + map_concat({'g': 7}, {'h': 8}) + ), + map_concat({'i': 9}, {'j': 10}) + ) + ) as complex_nested_concat; + """ + + // Test 7.6: Nested map_concat with different key types + qt_sql """ + select + map_concat( + map_concat({1: 'one', 2: 'two'}, {3: 'three', 4: 'four'}), + map_concat({'five': 5, 'six': 6}, {'seven': 7, 'eight': 8}) + ) as mixed_key_types_nested; + """ + + // Test 7.7: Interface test - map_concat in subquery + qt_sql """ + select + id, + (select map_concat({'static': 'value'}, map1) from ${testTable} where id = t.id) as subquery_concat + from ${testTable} t + order by id; + """ + + // Test 7.8: Interface test - map_concat with CASE expression + qt_sql """ + select + id, + map_concat( + CASE WHEN id % 2 = 0 THEN {'even': 'true'} ELSE {'odd': 'true'} END, + CASE WHEN id = 1 THEN {'id': '1'} + WHEN id = 2 THEN {'id': '2'} + WHEN id = 3 THEN {'id': '3'} + ELSE {'id': '4'} END + ) as case_expr_concat + from ${testTable} + order by id; + """ + + // Test 7.9: Special characters in keys and values + qt_sql """ + select + map_concat( + {'key with "double quotes"': 'value with ''single quotes'''}, + {'key with \\\\backslashes\\\\': 'value with \\\\n newline'}, + {'key with , comma': 'value with ; semicolon'}, + {'key with {} braces': 'value with [] brackets'} + ) as special_chars_test; + """ + + // Test 7.10: Empty string as key and value + qt_sql """ + select + map_concat( + {'': 'empty key', 'empty value': ''}, + {'': 'another empty key', 'key': ''} + ) as empty_string_test; + """ + + // Test 7.11: Very long strings as keys and values + qt_sql """ + select + map_concat( + {'very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_': + 'very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_'}, + {'another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_': + 'another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_'} + ) as long_strings_test; + """ + + // Test 7.12: map_concat with COALESCE to handle NULLs + qt_sql """ + select + map_concat( + COALESCE(map1, {}), + COALESCE(map2, {}) + ) as coalesce_handled + from ${testTable} + order by id; + """ } finally { // Clean up try { From 4332c227146ad917bcadbe72b040a24033eaf7d5 Mon Sep 17 00:00:00 2001 From: nagisa <1434936049@qq.com> Date: Tue, 16 Dec 2025 20:56:08 +0800 Subject: [PATCH 20/41] fix: style issue --- .../doris/catalog/BuiltinScalarFunctions.java | 2 +- .../functions/scalar/MapConcat.java | 51 ++++++++++--------- .../visitor/ScalarFunctionVisitor.java | 3 +- .../doris/nereids/util/TypeCoercionUtils.java | 12 +++-- 4 files changed, 36 insertions(+), 32 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index 37324f4c49d7df..dfe2fb87deeff3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -318,6 +318,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.MakeDate; import org.apache.doris.nereids.trees.expressions.functions.scalar.MakeSet; import org.apache.doris.nereids.trees.expressions.functions.scalar.MakeTime; +import org.apache.doris.nereids.trees.expressions.functions.scalar.MapConcat; import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsEntry; import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsKey; import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsValue; @@ -550,7 +551,6 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsDiff; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsSub; -import org.apache.doris.nereids.trees.expressions.functions.scalar.MapConcat; import com.google.common.collect.ImmutableList; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java index f5fbbf172f613d..259cd2030487bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java @@ -1,17 +1,17 @@ // Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file +// or more contributor license agreements. See the NOTICE file // distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file +// regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at +// with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the +// KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. @@ -33,26 +33,26 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; import java.util.ArrayList; import java.util.List; import java.util.Optional; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; /** * ScalarFunction 'map_concat' */ public class MapConcat extends ScalarFunction implements ExplicitlyCastableSignature, PropagateNullable { - private static final Logger LOG = LogManager.getLogger(MapConcat.class); - public static final List SIGNATURES = ImmutableList.of( - FunctionSignature - .ret(MapType.of(new AnyDataType(0), new AnyDataType(1))) - .varArgs(MapType.of(new AnyDataType(0), new AnyDataType(1))) + FunctionSignature + .ret(MapType.of(new AnyDataType(0), new AnyDataType(1))) + .varArgs(MapType.of(new AnyDataType(0), new AnyDataType(1))) ); + private static final Logger LOG = LogManager.getLogger(MapConcat.class); + /** * constructor with more than 0 arguments. */ @@ -75,14 +75,15 @@ public MapConcat withChildren(List children) { @Override public DataType getDataType() { - if (arity() >= 1){ + if (arity() >= 1) { List keyTypes = new ArrayList<>(); List valueTypes = new ArrayList<>(); - for (int i = 0; i < children.size(); i++){ + for (int i = 0; i < children.size(); i++) { DataType argType = children.get(i).getDataType(); - if (!(argType instanceof MapType)){ - if (!(argType instanceof NullType)){ - throw new AnalysisException("mapconcat function cannot process non-map and non-null child elements. Invalid SQL: " + this.toSql()); + if (!(argType instanceof MapType)) { + if (!(argType instanceof NullType)) { + throw new AnalysisException("mapconcat function cannot process non-map and non-null child " + + "elements. Invalid SQL: " + this.toSql()); } continue; } @@ -97,14 +98,14 @@ public DataType getDataType() { Optional commonKeyType = TypeCoercionUtils.findWiderCommonType(keyTypes, true, true); Optional commonValueType = TypeCoercionUtils.findWiderCommonType(valueTypes, true, true); - + if (!commonKeyType.isPresent()) { throw new AnalysisException("mapconcat cannot find the common key type of " + this.toSql()); } if (!commonValueType.isPresent()) { throw new AnalysisException("mapconcat cannot find the common value type of " + this.toSql()); } - + DataType keyType = commonKeyType.get(); DataType valueType = commonValueType.get(); return MapType.of(keyType, valueType); @@ -119,15 +120,15 @@ public R accept(ExpressionVisitor visitor, C context) { @Override public List getSignatures() { - if (arity() == 0){ + if (arity() == 0) { return SIGNATURES; } else { List signatures = ImmutableList.of( - FunctionSignature.of(getDataType(), - children.stream() - .map(ExpressionTrait::getDataType) - .collect(ImmutableList.toImmutableList()) - )); + FunctionSignature.of(getDataType(), + children.stream() + .map(ExpressionTrait::getDataType) + .collect(ImmutableList.toImmutableList()) + )); return signatures; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index 3810c435ed7a75..ac5da481263345 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -326,6 +326,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.MakeDate; import org.apache.doris.nereids.trees.expressions.functions.scalar.MakeSet; import org.apache.doris.nereids.trees.expressions.functions.scalar.MakeTime; +import org.apache.doris.nereids.trees.expressions.functions.scalar.MapConcat; import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsEntry; import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsKey; import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsValue; @@ -553,7 +554,6 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.YearWeek; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsDiff; -import org.apache.doris.nereids.trees.expressions.functions.scalar.MapConcat; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsSub; import org.apache.doris.nereids.trees.expressions.functions.udf.AliasUdf; import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdf; @@ -2722,6 +2722,7 @@ default R visitPeriodAdd(PeriodAdd periodAdd, C context) { default R visitPeriodDiff(PeriodDiff periodDiff, C context) { return visitScalarFunction(periodDiff, context); } + default R visitMapConcat(MapConcat mapConcat, C context) { return visitScalarFunction(mapConcat, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index d46e48147a0691..51553ad095ea8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -723,11 +723,13 @@ private static Expression processMapConcat(MapConcat mapConcat) { List keyTypes = new ArrayList<>(); List valueTypes = new ArrayList<>(); List children = mapConcat.children(); - for (int i = 0; i < children.size(); i++){ + for (int i = 0; i < children.size(); i++) { DataType argType = children.get(i).getDataType(); - if (!(argType instanceof MapType)){ - if (!(argType instanceof NullType)){ - throw new AnalysisException("mapconcat function cannot process non-map and non-null child elements. Invalid SQL: " + mapConcat.toSql()); + if (!(argType instanceof MapType)) { + if (!(argType instanceof NullType)) { + throw new AnalysisException("mapconcat function cannot process" + + "non-map and non-null child elements. " + + "Invalid SQL: " + mapConcat.toSql()); } continue; } @@ -749,7 +751,7 @@ private static Expression processMapConcat(MapConcat mapConcat) { ImmutableList.Builder newChildren = ImmutableList.builder(); for (int i = 0; i < mapConcat.arity(); i++) { DataType argType = children.get(i).getDataType(); - if (!(argType instanceof MapType)){ + if (!(argType instanceof MapType)) { newChildren.add(mapConcat.child(i)); } else { newChildren.add(castIfNotSameType(mapConcat.child(i), targetMapType)); From 94bd981de8ab1ef9abc678bd98b6573bbd81c8ad Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Tue, 16 Dec 2025 22:13:55 +0800 Subject: [PATCH 21/41] fix: stype issue --- .../data_type_date_or_datetime_v2.h | 2 +- be/src/vec/functions/function_map.cpp | 42 +- .../vec/function/function_map_concat_test.cpp | 484 +++++++----------- be/test/vec/function/function_test_util.cpp | 13 +- be/test/vec/function/function_test_util.h | 30 +- 5 files changed, 218 insertions(+), 353 deletions(-) diff --git a/be/src/vec/data_types/data_type_date_or_datetime_v2.h b/be/src/vec/data_types/data_type_date_or_datetime_v2.h index 3922f33de9472d..2e4e36c0640ba5 100644 --- a/be/src/vec/data_types/data_type_date_or_datetime_v2.h +++ b/be/src/vec/data_types/data_type_date_or_datetime_v2.h @@ -191,7 +191,7 @@ constexpr bool IsDataTypeDateTimeV2 = false; template <> inline constexpr bool IsDataTypeDateTimeV2 = true; -template +template constexpr bool IsDataTypeMap = false; template <> inline constexpr bool IsDataTypeMap = true; diff --git a/be/src/vec/functions/function_map.cpp b/be/src/vec/functions/function_map.cpp index 5f822309c32790..95ef882f6399d6 100644 --- a/be/src/vec/functions/function_map.cpp +++ b/be/src/vec/functions/function_map.cpp @@ -91,9 +91,10 @@ class FunctionMap : public IFunction { DCHECK(arguments.size() % 2 == 0) << "function: " << get_name() << ", arguments should not be even number"; LOG(INFO) << "[FunctionMap.execute_impl] input_rows_count: " << input_rows_count; - for (size_t i =0;i < arguments.size();++i) { + for (size_t i = 0; i < arguments.size(); ++i) { auto& col = block.get_by_position(arguments[i]).column; - LOG(INFO) << "[FunctionMap.execute_impl] argument " << i << " column type: " << col->get_name()<<" data:"<dump_structure(); + LOG(INFO) << "[FunctionMap.execute_impl] argument " << i + << " column type: " << col->get_name() << " data:" << col->dump_structure(); } size_t num_element = arguments.size(); @@ -798,16 +799,16 @@ class FunctionDeduplicateMap : public IFunction { private: }; -class FunctionMapConcat : public IFunction{ +class FunctionMapConcat : public IFunction { public: static constexpr auto name = "map_concat"; - static FunctionPtr create() {return std::make_shared();} + static FunctionPtr create() { return std::make_shared(); } String get_name() const override { return name; } bool is_variadic() const override { return true; } size_t get_number_of_arguments() const override { return 1; } DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { - DCHECK(arguments.size()>0) - <<"function: "< 0) + << "function: " << get_name() << ", arguments should not be empty"; return arguments[0]; } Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, @@ -815,9 +816,10 @@ class FunctionMapConcat : public IFunction{ auto result_col = block.get_by_position(result).type->create_column(); ColumnMap* result_map_column = nullptr; ColumnNullable* result_nullable_column = nullptr; - if (result_col->is_nullable()){ + if (result_col->is_nullable()) { result_nullable_column = reinterpret_cast(result_col.get()); - result_map_column = check_and_get_column(result_nullable_column->get_nested_column()); + result_map_column = + check_and_get_column(result_nullable_column->get_nested_column()); } else { result_map_column = check_and_get_column(result_col.get()); } @@ -836,27 +838,29 @@ class FunctionMapConcat : public IFunction{ } size_t off = 0; - for(size_t row = 0; row < input_rows_count ; row++) { - for(size_t col: arguments){ + for (size_t row = 0; row < input_rows_count; row++) { + for (size_t col : arguments) { const ColumnMap* map_column = nullptr; - auto src_column = + auto src_column = block.get_by_position(col).column->convert_to_full_column_if_const(); - if (src_column->is_nullable()){ - auto nullable_column = reinterpret_cast(src_column.get()); - map_column = check_and_get_column(nullable_column->get_nested_column()); + if (src_column->is_nullable()) { + auto nullable_column = + reinterpret_cast(src_column.get()); + map_column = + check_and_get_column(nullable_column->get_nested_column()); } else { map_column = check_and_get_column(*src_column.get()); } - if (!map_column){ + if (!map_column) { return Status::RuntimeError("unsupported types for function {}({})", get_name(), - block.get_by_position(col).type->get_name()); + block.get_by_position(col).type->get_name()); } const auto& src_column_offsets = map_column->get_offsets(); const size_t length = src_column_offsets[row] - src_column_offsets[row - 1]; off += length; - for(size_t i=src_column_offsets[row-1];iget_keys(),i); - result_col_map_vals_data.insert_from(map_column->get_values(),i); + for (size_t i = src_column_offsets[row - 1]; i < src_column_offsets[row]; i++) { + result_col_map_keys_data.insert_from(map_column->get_keys(), i); + result_col_map_vals_data.insert_from(map_column->get_values(), i); } } column_offsets[row] = off; diff --git a/be/test/vec/function/function_map_concat_test.cpp b/be/test/vec/function/function_map_concat_test.cpp index c6e61fa5925ff3..390a97c3b72586 100644 --- a/be/test/vec/function/function_map_concat_test.cpp +++ b/be/test/vec/function/function_map_concat_test.cpp @@ -1,404 +1,262 @@ +#include + #include #include -#include -#include "vec/core/types.h" -#include "function_test_util.h" +#include "function_test_util.h" +#include "vec/core/types.h" namespace doris::vectorized { TEST(FunctionMapConcatTest, TestBase) { const std::string func_name = "map_concat"; { // simple case - two maps - InputTypeSet input_types = { - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING - }; - DataSet data_set = { - { - TestArray({ - TestArray({std::int32_t(1), std::string("A")}), - TestArray({std::int32_t(2), std::string("B")}) - }), - TestArray({ - std::int32_t(1), std::string("A"), - std::int32_t(2), std::string("B") - }) - } - }; - + InputTypeSet input_types = {PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, + PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, + PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING}; + DataSet data_set = {{TestArray({TestArray({std::int32_t(1), std::string("A")}), + TestArray({std::int32_t(2), std::string("B")})}), + TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), + std::string("B")})}}; + check_function_all_arg_comb(func_name, input_types, data_set); // EXPECT_TRUE(status.ok()) << "Function test failed: " << status.to_string(); } { // two maps concatenation with string keys - InputTypeSet input_types = { - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_STRING, - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_STRING - }; - DataSet data_set = { - { - TestArray({ - TestArray({std::string("A"), std::string("a")}), - TestArray({std::string("B"), std::string("b")}) - }), - TestArray({ - std::string("A"), std::string("a"), - std::string("B"), std::string("b") - }) - } - }; + InputTypeSet input_types = {PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, + PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_STRING}; + DataSet data_set = {{TestArray({TestArray({std::string("A"), std::string("a")}), + TestArray({std::string("B"), std::string("b")})}), + TestArray({std::string("A"), std::string("a"), std::string("B"), + std::string("b")})}}; check_function_all_arg_comb(func_name, input_types, data_set); } { // two maps concatenation with string keys - InputTypeSet input_types = { - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_STRING, - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_STRING - }; - DataSet data_set = { - { - TestArray({ - TestArray({std::string("A"), std::string("a")}), - TestArray({std::string("B"), std::string("b")}) - }), - TestArray({ - std::string("A"), std::string("a"), - std::string("B"), std::string("b") - }) - }, - { - TestArray({ - TestArray({std::string("C"), std::string("c")}), - TestArray({std::string("D"), std::string("d")}) - }), - TestArray({ - std::string("C"), std::string("c"), - std::string("D"), std::string("d") - }) - } - }; + InputTypeSet input_types = {PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, + PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_STRING}; + DataSet data_set = {{TestArray({TestArray({std::string("A"), std::string("a")}), + TestArray({std::string("B"), std::string("b")})}), + TestArray({std::string("A"), std::string("a"), std::string("B"), + std::string("b")})}, + {TestArray({TestArray({std::string("C"), std::string("c")}), + TestArray({std::string("D"), std::string("d")})}), + TestArray({std::string("C"), std::string("c"), std::string("D"), + std::string("d")})}}; check_function_all_arg_comb(func_name, input_types, data_set); } } TEST(FunctionMapConcatTest, TestEdgeCases) { const std::string func_name = "map_concat"; - + // Test empty maps { - InputTypeSet input_types = { - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING - }; + InputTypeSet input_types = {PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, + PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, + PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING}; DataSet data_set = { - { - TestArray({TestArray({}), TestArray({})}), - TestArray({}) - }, - { - TestArray({TestArray({std::int32_t(1), std::string("A")}), TestArray({})}), - TestArray({std::int32_t(1), std::string("A")}) - }, - { - TestArray({TestArray({}), TestArray({std::int32_t(2), std::string("B")})}), - TestArray({std::int32_t(2), std::string("B")}) - } - }; - + {TestArray({TestArray({}), TestArray({})}), TestArray({})}, + {TestArray({TestArray({std::int32_t(1), std::string("A")}), TestArray({})}), + TestArray({std::int32_t(1), std::string("A")})}, + {TestArray({TestArray({}), TestArray({std::int32_t(2), std::string("B")})}), + TestArray({std::int32_t(2), std::string("B")})}}; + check_function_all_arg_comb(func_name, input_types, data_set); } - + // Test key conflicts (later map should override earlier) { - InputTypeSet input_types = { - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING - }; - DataSet data_set = { - { - TestArray({ - TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), std::string("B")}), - TestArray({std::int32_t(1), std::string("C"), std::int32_t(3), std::string("D")}) - }), - TestArray({ - std::int32_t(2), std::string("B"), - std::int32_t(1), std::string("C"), - std::int32_t(3), std::string("D") - }) - } - }; - + InputTypeSet input_types = {PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, + PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, + PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING}; + DataSet data_set = {{TestArray({TestArray({std::int32_t(1), std::string("A"), + std::int32_t(2), std::string("B")}), + TestArray({std::int32_t(1), std::string("C"), + std::int32_t(3), std::string("D")})}), + TestArray({std::int32_t(2), std::string("B"), std::int32_t(1), + std::string("C"), std::int32_t(3), std::string("D")})}}; + check_function_all_arg_comb(func_name, input_types, data_set); } - + // Test multiple maps (more than 2) { InputTypeSet input_types = { - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING - }; - DataSet data_set = { - { - TestArray({ - TestArray({std::int32_t(1), std::string("A")}), - TestArray({std::int32_t(2), std::string("B")}), - TestArray({std::int32_t(3), std::string("C")}) - }), - TestArray({ - std::int32_t(1), std::string("A"), - std::int32_t(2), std::string("B"), - std::int32_t(3), std::string("C") - }) - } - }; - + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING}; + DataSet data_set = {{TestArray({TestArray({std::int32_t(1), std::string("A")}), + TestArray({std::int32_t(2), std::string("B")}), + TestArray({std::int32_t(3), std::string("C")})}), + TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), + std::string("B"), std::int32_t(3), std::string("C")})}}; + check_function_all_arg_comb(func_name, input_types, data_set); } - + // Test different value types { - InputTypeSet input_types = { - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT, - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT - }; - DataSet data_set = { - { - TestArray({ - TestArray({std::string("A"), std::int32_t(1), std::string("B"), std::int32_t(2)}), - TestArray({std::string("C"), std::int32_t(3), std::string("D"), std::int32_t(4)}) - }), - TestArray({ - std::string("A"), std::int32_t(1), - std::string("B"), std::int32_t(2), - std::string("C"), std::int32_t(3), - std::string("D"), std::int32_t(4) - }) - } - }; - + InputTypeSet input_types = {PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_INT, PrimitiveType::TYPE_MAP, + PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT}; + DataSet data_set = {{TestArray({TestArray({std::string("A"), std::int32_t(1), + std::string("B"), std::int32_t(2)}), + TestArray({std::string("C"), std::int32_t(3), + std::string("D"), std::int32_t(4)})}), + TestArray({std::string("A"), std::int32_t(1), std::string("B"), + std::int32_t(2), std::string("C"), std::int32_t(3), + std::string("D"), std::int32_t(4)})}}; + check_function_all_arg_comb(func_name, input_types, data_set); } - - + // Test single map (should return the same map) { - InputTypeSet input_types = { - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING - }; - TestArray mp_src_array = TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), std::string("B")}); + InputTypeSet input_types = {PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, + PrimitiveType::TYPE_STRING}; + TestArray mp_src_array = + TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), std::string("B")}); TestArray src; src.push_back(mp_src_array); - TestArray mp_dest_array = TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), std::string("B")}); - DataSet data_set = { - { - src, - mp_dest_array - } - }; + TestArray mp_dest_array = + TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), std::string("B")}); + DataSet data_set = {{src, mp_dest_array}}; check_function_all_arg_comb(func_name, input_types, data_set); } } -TEST(FunctionMapConcatTest, TestWithNULL){ +TEST(FunctionMapConcatTest, TestWithNULL) { const std::string func_name = "map_concat"; // Test with null map (one map is null) { - InputTypeSet input_types = { - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING - }; - DataSet data_set = { - { - TestArray({Null(), TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), std::string("B")})}), - Null() - }, - { - TestArray({TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), std::string("B")}), Null()}), - Null() - }, - { - TestArray({Null(), Null()}), - Null() - } - }; - + InputTypeSet input_types = {PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, + PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, + PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING}; + DataSet data_set = {{TestArray({Null(), TestArray({std::int32_t(1), std::string("A"), + std::int32_t(2), std::string("B")})}), + Null()}, + {TestArray({TestArray({std::int32_t(1), std::string("A"), + std::int32_t(2), std::string("B")}), + Null()}), + Null()}, + {TestArray({Null(), Null()}), Null()}}; + check_function_all_arg_comb(func_name, input_types, data_set); } // Test with null values { - InputTypeSet input_types = { - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING, - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING - }; + InputTypeSet input_types = {PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_INT, + PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, + PrimitiveType::TYPE_INT, PrimitiveType::TYPE_STRING}; DataSet data_set = { - { - TestArray({ - TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), Null()}), - TestArray({std::int32_t(2), std::string("B"), std::int32_t(3), std::string("C")}) - }), - TestArray({ - std::int32_t(1), std::string("A"), - std::int32_t(2), std::string("B"), - std::int32_t(3), std::string("C") - }) - } - }; - + {TestArray({TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), Null()}), + TestArray({std::int32_t(2), std::string("B"), std::int32_t(3), + std::string("C")})}), + TestArray({std::int32_t(1), std::string("A"), std::int32_t(2), std::string("B"), + std::int32_t(3), std::string("C")})}}; + check_function_all_arg_comb(func_name, input_types, data_set); } } TEST(FunctionMapConcatTest, TestComplexTypes) { const std::string func_name = "map_concat"; - + // Test with nested types (array as value) { - InputTypeSet input_types = { - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_ARRAY, PrimitiveType::TYPE_INT, - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_ARRAY, PrimitiveType::TYPE_INT - }; + InputTypeSet input_types = {PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_ARRAY, PrimitiveType::TYPE_INT, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_ARRAY, PrimitiveType::TYPE_INT}; DataSet data_set = { - { - TestArray({ - TestArray({ - std::string("A"), - TestArray({std::int32_t(1), std::int32_t(2)}), - std::string("B"), - TestArray({std::int32_t(3), std::int32_t(5)}) - }), - TestArray({ - std::string("C"), - TestArray({std::int32_t(4), std::int32_t(5)}) - }) - }), - TestArray({ - std::string("A"), TestArray({std::int32_t(1), std::int32_t(2)}), - std::string("B"), TestArray({std::int32_t(3), std::int32_t(5)}), - std::string("C"), TestArray({std::int32_t(4), std::int32_t(5)}) - }) - } - }; - + {TestArray({TestArray({std::string("A"), + TestArray({std::int32_t(1), std::int32_t(2)}), + std::string("B"), + TestArray({std::int32_t(3), std::int32_t(5)})}), + TestArray({std::string("C"), + TestArray({std::int32_t(4), std::int32_t(5)})})}), + TestArray({std::string("A"), TestArray({std::int32_t(1), std::int32_t(2)}), + std::string("B"), TestArray({std::int32_t(3), std::int32_t(5)}), + std::string("C"), TestArray({std::int32_t(4), std::int32_t(5)})})}}; + check_function_all_arg_comb(func_name, input_types, data_set); } - + // Test with map as value { - InputTypeSet input_types = { - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT, - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT - }; + InputTypeSet input_types = {PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_INT, PrimitiveType::TYPE_MAP, + PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_MAP, + PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_INT}; DataSet data_set = { - { - TestArray({ - TestArray({ - std::string("outer1"), - TestArray({TestArray({std::string("inner1"), std::int32_t(1)})}), - std::string("outer2"), - TestArray({TestArray({std::string("inner2"), std::int32_t(2)})}) - }), - TestArray({ - std::string("outer3"), - TestArray({TestArray({std::string("inner3"), std::int32_t(3)})}) - }) - }), - TestArray({ - std::string("outer1"), TestArray({TestArray({std::string("inner1"), std::int32_t(1)})}), - std::string("outer2"), TestArray({TestArray({std::string("inner2"), std::int32_t(2)})}), - std::string("outer3"), TestArray({TestArray({std::string("inner3"), std::int32_t(3)})}) - }) - } - }; - + {TestArray( + {TestArray( + {std::string("outer1"), + TestArray({TestArray({std::string("inner1"), std::int32_t(1)})}), + std::string("outer2"), + TestArray( + {TestArray({std::string("inner2"), std::int32_t(2)})})}), + TestArray({std::string("outer3"), + TestArray({TestArray( + {std::string("inner3"), std::int32_t(3)})})})}), + TestArray({std::string("outer1"), + TestArray({TestArray({std::string("inner1"), std::int32_t(1)})}), + std::string("outer2"), + TestArray({TestArray({std::string("inner2"), std::int32_t(2)})}), + std::string("outer3"), + TestArray({TestArray({std::string("inner3"), std::int32_t(3)})})})}}; + check_function_all_arg_comb(func_name, input_types, data_set); } - + // Test with double as value { - InputTypeSet input_types = { - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_DOUBLE, - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_DOUBLE - }; + InputTypeSet input_types = {PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_DOUBLE, PrimitiveType::TYPE_MAP, + PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_DOUBLE}; DataSet data_set = { - { - TestArray({ - TestArray({ - std::string("key1"), 1.5, - std::string("key2"), 2.7 - }), - TestArray({ - std::string("key3"), 3.9 - }) - }), - TestArray({ - std::string("key1"), 1.5, - std::string("key2"), 2.7, - std::string("key3"), 3.9 - }) - } - }; - + {TestArray({TestArray({std::string("key1"), 1.5, std::string("key2"), 2.7}), + TestArray({std::string("key3"), 3.9})}), + TestArray({std::string("key1"), 1.5, std::string("key2"), 2.7, std::string("key3"), + 3.9})}}; + check_function_all_arg_comb(func_name, input_types, data_set); } - + // Test with float as value { - InputTypeSet input_types = { - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_FLOAT, - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_FLOAT - }; + InputTypeSet input_types = {PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_FLOAT, PrimitiveType::TYPE_MAP, + PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_FLOAT}; DataSet data_set = { - { - TestArray({ - TestArray({ - std::string("key1"), 1.5f, - std::string("key2"), 2.7f - }), - TestArray({ - std::string("key3"), 3.9f - }) - }), - TestArray({ - std::string("key1"), 1.5f, - std::string("key2"), 2.7f, - std::string("key3"), 3.9f - }) - } - }; - + {TestArray({TestArray({std::string("key1"), 1.5f, std::string("key2"), 2.7f}), + TestArray({std::string("key3"), 3.9f})}), + TestArray({std::string("key1"), 1.5f, std::string("key2"), 2.7f, + std::string("key3"), 3.9f})}}; + check_function_all_arg_comb(func_name, input_types, data_set); } - + // Test with decimalv2 as value { - InputTypeSet input_types = { - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_DECIMALV2, - PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_DECIMALV2 - }; - DataSet data_set = { - { - TestArray({ - TestArray({ - std::string("key1"), ut_type::DECIMALV2(1.5), - std::string("key2"), ut_type::DECIMALV2(2.7) - }), - TestArray({ - std::string("key3"), ut_type::DECIMALV2(3.9) - }) - }), - TestArray({ - std::string("key1"), ut_type::DECIMALV2(1.5), - std::string("key2"), ut_type::DECIMALV2(2.7), - std::string("key3"), ut_type::DECIMALV2(3.9) - }) - } - }; - + InputTypeSet input_types = {PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, + PrimitiveType::TYPE_DECIMALV2, PrimitiveType::TYPE_MAP, + PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_DECIMALV2}; + DataSet data_set = {{TestArray({TestArray({std::string("key1"), ut_type::DECIMALV2(1.5), + std::string("key2"), ut_type::DECIMALV2(2.7)}), + TestArray({std::string("key3"), ut_type::DECIMALV2(3.9)})}), + TestArray({std::string("key1"), ut_type::DECIMALV2(1.5), + std::string("key2"), ut_type::DECIMALV2(2.7), + std::string("key3"), ut_type::DECIMALV2(3.9)})}}; + check_function_all_arg_comb(func_name, input_types, data_set); } } -} +} // namespace doris::vectorized diff --git a/be/test/vec/function/function_test_util.cpp b/be/test/vec/function/function_test_util.cpp index 155e9a4d53c880..e161e6e2f7f936 100644 --- a/be/test/vec/function/function_test_util.cpp +++ b/be/test/vec/function/function_test_util.cpp @@ -353,20 +353,20 @@ bool insert_array_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const Any } bool insert_map_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const AnyType& cell, - bool datetime_is_string_format) { + bool datetime_is_string_format) { auto origin_input_array = any_cast(cell); DataTypePtr key_type = assert_cast(type_ptr.get())->get_key_type(); DataTypePtr value_type = assert_cast(type_ptr.get())->get_value_type(); MutableColumnPtr key_column = key_type->create_column(); MutableColumnPtr value_column = value_type->create_column(); - for(size_t i=0;iis_nullable()){ - auto*data_type_nullable = assert_cast(descs[0].data_type.get()); - map_type = check_and_get_data_type(data_type_nullable->get_nested_type().get()); + auto get_key_value_type = [&]() { + const DataTypeMap* map_type = nullptr; + if (descs[0].data_type->is_nullable()) { + auto* data_type_nullable = + assert_cast(descs[0].data_type.get()); + map_type = check_and_get_data_type( + data_type_nullable->get_nested_type().get()); } else { map_type = check_and_get_data_type(descs[0].data_type.get()); } assert(map_type); - return std::make_pair(map_type->get_key_type(),map_type->get_value_type()); + return std::make_pair(map_type->get_key_type(), map_type->get_value_type()); }; auto return_type = [&]() { if constexpr (IsDataTypeDecimal) { // decimal @@ -390,8 +392,9 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty : std::make_shared(real_scale); } else if constexpr (IsDataTypeMap) { auto [key_type, value_type] = get_key_value_type(); - return ResultNullable ? make_nullable(std::make_shared(key_type, value_type)) - : std::make_shared(key_type, value_type); + return ResultNullable + ? make_nullable(std::make_shared(key_type, value_type)) + : std::make_shared(key_type, value_type); } else { return ResultNullable ? make_nullable(std::make_shared()) : std::make_shared(); @@ -444,11 +447,12 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty } result_type_ptr = ResultNullable ? make_nullable(std::make_shared(real_scale)) : std::make_shared(real_scale); - } else if constexpr (IsDataTypeMap){ + } else if constexpr (IsDataTypeMap) { auto [key_type, value_type] = get_key_value_type(); - result_type_ptr = ResultNullable ? make_nullable(std::make_shared(key_type, value_type)) - : std::make_shared(key_type, value_type); - }else { + result_type_ptr = + ResultNullable ? make_nullable(std::make_shared(key_type, value_type)) + : std::make_shared(key_type, value_type); + } else { result_type_ptr = ResultNullable ? make_nullable(std::make_shared()) : std::make_shared(); } From 7bfec0f2495bd67dbfe654487e5179a324de7da5 Mon Sep 17 00:00:00 2001 From: nagisa <1434936049@qq.com> Date: Thu, 18 Dec 2025 01:38:13 +0800 Subject: [PATCH 22/41] fix: style issue --- .../functions/scalar/MapConcat.java | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java index 259cd2030487bc..f323cef4579249 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java @@ -27,14 +27,9 @@ import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.MapType; import org.apache.doris.nereids.types.NullType; -import org.apache.doris.nereids.types.coercion.AnyDataType; -import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.TypeCoercionUtils; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; import java.util.ArrayList; import java.util.List; @@ -46,18 +41,14 @@ public class MapConcat extends ScalarFunction implements ExplicitlyCastableSignature, PropagateNullable { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature - .ret(MapType.of(new AnyDataType(0), new AnyDataType(1))) - .varArgs(MapType.of(new AnyDataType(0), new AnyDataType(1))) - ); - - private static final Logger LOG = LogManager.getLogger(MapConcat.class); + FunctionSignature.ret(MapType.SYSTEM_DEFAULT).args() + ); /** * constructor with more than 0 arguments. */ - public MapConcat(Expression arg, Expression... varArgs) { - super("map_concat", ExpressionUtils.mergeArguments(arg, varArgs)); + public MapConcat(Expression... varArgs) { + super("map_concat", varArgs); } /** @@ -69,7 +60,6 @@ private MapConcat(ScalarFunctionParams functionParams) { @Override public MapConcat withChildren(List children) { - Preconditions.checkArgument(children.size() >= 1); return new MapConcat(getFunctionParams(children)); } @@ -110,7 +100,7 @@ public DataType getDataType() { DataType valueType = commonValueType.get(); return MapType.of(keyType, valueType); } - throw new RuntimeException("unreachable"); + return MapType.SYSTEM_DEFAULT; } @Override From 958eac901c05177305251ac2aee98f4922777e6c Mon Sep 17 00:00:00 2001 From: nagisa <1434936049@qq.com> Date: Thu, 18 Dec 2025 01:38:45 +0800 Subject: [PATCH 23/41] fix: style issue --- be/src/vec/functions/function_map.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/be/src/vec/functions/function_map.cpp b/be/src/vec/functions/function_map.cpp index 95ef882f6399d6..3f3c1c8782720e 100644 --- a/be/src/vec/functions/function_map.cpp +++ b/be/src/vec/functions/function_map.cpp @@ -805,7 +805,7 @@ class FunctionMapConcat : public IFunction { static FunctionPtr create() { return std::make_shared(); } String get_name() const override { return name; } bool is_variadic() const override { return true; } - size_t get_number_of_arguments() const override { return 1; } + size_t get_number_of_arguments() const override { return 0; } DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { DCHECK(arguments.size() > 0) << "function: " << get_name() << ", arguments should not be empty"; @@ -823,15 +823,11 @@ class FunctionMapConcat : public IFunction { } else { result_map_column = check_and_get_column(result_col.get()); } - // map keys column auto& result_col_map_keys_data = result_map_column->get_keys(); - // map values column auto& result_col_map_vals_data = result_map_column->get_values(); ColumnArray::Offsets64& column_offsets = result_map_column->get_offsets(); column_offsets.resize(input_rows_count); - // Initialize null map if result is nullable - // reference to ColumnNullable::size() if (result_nullable_column) { auto& null_map_data = result_nullable_column->get_null_map_data(); null_map_data.resize_fill(input_rows_count, 0); From fb58202ead6fe9c2ff3a766a9c727873c150ebba Mon Sep 17 00:00:00 2001 From: nagisa <1434936049@qq.com> Date: Thu, 18 Dec 2025 01:39:16 +0800 Subject: [PATCH 24/41] =?UTF-8?q?fix:=20crash=20when=20=E2=80=9Cselect=20m?= =?UTF-8?q?ap=5Fconcat();=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/org/apache/doris/nereids/util/TypeCoercionUtils.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 51553ad095ea8e..b71e557461ff61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -720,6 +720,9 @@ public static Expression processBoundFunction(BoundFunction boundFunction) { } private static Expression processMapConcat(MapConcat mapConcat) { + if (mapConcat.arity() == 0) { + return new MapLiteral(); + } List keyTypes = new ArrayList<>(); List valueTypes = new ArrayList<>(); List children = mapConcat.children(); From 109c9af8fe8c8872b7d68db57789072d207fa76c Mon Sep 17 00:00:00 2001 From: nagisa <1434936049@qq.com> Date: Thu, 18 Dec 2025 02:45:23 +0800 Subject: [PATCH 25/41] fix: optimize test --- .../data/function_p0/test_map_concat.out | 128 +--- .../suites/function_p0/test_map_concat.groovy | 553 ++++-------------- 2 files changed, 121 insertions(+), 560 deletions(-) diff --git a/regression-test/data/function_p0/test_map_concat.out b/regression-test/data/function_p0/test_map_concat.out index c8442e95d9fc74..d1d69a82a81481 100644 --- a/regression-test/data/function_p0/test_map_concat.out +++ b/regression-test/data/function_p0/test_map_concat.out @@ -1,143 +1,35 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !sql -- -1 {"a":"apple", "b":"banana", "extra_key":"extra_value"} -2 {"c":"cherry", "extra_key":"extra_value"} -3 {"extra_key":"extra_value"} -4 \N +{} -- !sql -- -1 {"a":"apple", "b":"banana", "x":"10", "y":"20", "1":"one", "2":"two"} -2 {"c":"cherry", "z":"30", "3":"three"} -3 {} -4 \N +{"single":"argument"} -- !sql -- {"a":"apple", "b":"banana", "c":"cherry"} -- !sql -- -1 {"a":"apple", "b":"banana"} {"x":10, "y":20} {} -2 {"c":"cherry"} {"z":30} {} -3 {} {} {} -4 \N \N {} - --- !sql -- -1 \N \N \N -2 \N \N \N -3 \N \N \N -4 \N \N \N +1 {"a":"apple", "b":"banana", "x":"10", "y":"20", "1":"one", "2":"two"} +2 {"c":"cherry", "z":"30", "3":"three"} +3 {} +4 \N -- !sql -- -\N \N +{"a":"apple", "b":"banana"} \N \N -- !sql -- {"a":"apple", "b":"blueberry", "c":"cherry"} -- !sql -- -{"a":"first", "b":"second", "c":"third", "d":"third"} - --- !sql -- -1 2 4 -2 1 2 -3 0 0 -4 \N \N - --- !sql -- -{"a":1, "b":2, "c":3, "d":4} - --- !sql -- -["x", "y"] [10, 20] +4 ["x", "y"] -- !sql -- -{"a":"1", "b":"2"} +{1:"one", 2:"two", 3:"three", 4:"four"} {"bigint1":10000000000, "bigint2":20000000000, "bigint3":30000000000} {"double1":1.2345678901, "double2":2.3456789012, "double3":3.4567890123} {"flag1":1, "flag2":0} {"arr1":[1, 2, 3], "arr2":[4, 5, 6]} {"decimal1":123.456, "decimal2":789.012, "decimal3":345.678} {"date1":"2023-01-01 00:00:00", "date2":"2023-12-31 00:00:00", "timestamp1":"2023-01-01 12:00:00"} {"int_val":100, "bigint_val":200} {"decimal_val":123.456, "double_val":789.0119999999999} {"中文键":"中文值", "key with emoji 🔥":"value with emoji 🚀", "key with accents café":"value with accents naïve"} -- !sql -- -{1:"one", 2:"two", 3:"three", 4:"four"} - --- !sql -- -{"p1":"v1", "p2":"v2", "p3":"v3", "p4":"v4", "p5":"v5", "p6":"v6", "p7":"v7", "p8":"v8", "p9":"v9"} - --- !sql -- -{"a":"1", "b":"2"} - --- !sql -- -{"single":"argument"} - --- !sql -- -{"a":"value", "b":null, "c":"another", "d":null} - --- !sql -- -\N - --- !sql -- -{"tiny":127, "small":32767, "int":2147483647} - --- !sql -- -{"flag1":1, "flag2":0, "flag3":1} - --- !sql -- -{"date1":"2023-01-01", "date2":"2023-12-31"} - --- !sql -- -{"arr1":[1, 2, 3], "arr2":[4, 5, 6]} - --- !sql -- -{"k01":"v01", "k02":"v02", "k03":"v03", "k04":"v04", "k05":"v05", "k06":"v06", "k07":"v07", "k08":"v08", "k09":"v09", "k10":"v10", "k11":"v11", "k12":"v12"} - --- !sql -- -{"a":1, "b":2, "c":3, "d":4, "e":5, "f":6} - --- !sql -- -true false +{"a":1, "b":2, "c":3, "d":4} {"a":"1", "b":"2"} -- !sql -- 1 {"a":"apple", "b":"banana", "x":"10", "y":"20"} 2 {"c":"cherry", "z":"30"} --- !sql -- -["z", "a", "m"] ["a", "z", "m"] - --- !sql -- -[{"key":"a", "value":1}, {"key":"b", "value":2}, {"key":"c", "value":3}, {"key":"d", "value":4}] - --- !sql -- -true false - --- !sql -- -{"中文键":"中文值", "key with emoji 🔥":"value with emoji 🚀", "key with accents café":"value with accents naïve"} - --- !sql -- -{"English key":"value1", "日本語キー":"値1", "한국어 키":"값1", "русский ключ":"значение1"} - --- !sql -- -{"a":1, "b":2, "c":3, "d":4, "e":5, "f":6, "g":7, "h":8, "i":9, "j":10} - --- !sql -- -{"1":"one", "2":"two", "3":"three", "4":"four", "five":"5", "six":"6", "seven":"7", "eight":"8"} - --- !sql -- -1 {"static":"value", "a":"apple", "b":"banana"} -2 {"static":"value", "c":"cherry"} -3 {"static":"value"} -4 \N - --- !sql -- -1 {"odd":"true", "id":"1"} -2 {"even":"true", "id":"2"} -3 {"odd":"true", "id":"3"} -4 {"even":"true", "id":"4"} - --- !sql -- -{"key with "double quotes"":"value with 'single quotes'", "key with \\backslashes\\":"value with \\n newline", "key with , comma":"value with ; semicolon", "key with {} braces":"value with [] brackets"} - --- !sql -- -{"empty value":"", "":"another empty key", "key":""} - --- !sql -- -{"very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_":"very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_", "another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_":"another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_"} - --- !sql -- -{"a":"apple", "b":"banana", "x":"10", "y":"20"} -{"c":"cherry", "z":"30"} -{} -{} - diff --git a/regression-test/suites/function_p0/test_map_concat.groovy b/regression-test/suites/function_p0/test_map_concat.groovy index 92579d03f24a18..592eb9531af363 100644 --- a/regression-test/suites/function_p0/test_map_concat.groovy +++ b/regression-test/suites/function_p0/test_map_concat.groovy @@ -16,451 +16,120 @@ // under the License. suite("test_map_concat") { - // Enable Nereids planner for testing sql """ set enable_nereids_planner=true; """ sql """ set enable_fallback_to_original_planner=false; """ def testTable = "test_map_concat_table" - try { - // Drop table if exists - sql """ - drop table if exists ${testTable}; - """ - - // Create test table with MAP columns - sql """ - CREATE TABLE ${testTable} ( - id INT, - map1 MAP, - map2 MAP, - map3 MAP - ) ENGINE=OLAP - DUPLICATE KEY(id) - DISTRIBUTED BY HASH(id) BUCKETS 1 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - ); - """ - - // Insert test data - sql """ - insert into ${testTable} values - (1, {'a': 'apple', 'b': 'banana'}, {'x': 10, 'y': 20}, {1: 'one', 2: 'two'}), - (2, {'c': 'cherry'}, {'z': 30}, {3: 'three'}), - (3, {}, {}, {}), - (4, NULL, NULL, NULL); - """ - - // ============================================ - // Group 1: Basic functionality tests - // ============================================ - - // Test 1.1: Basic map_concat with two maps (all rows) - qt_sql """ - select - id, - map_concat(map1, {'extra_key': 'extra_value'}) as merged_map - from ${testTable} - order by id; - """ - - // Test 1.2: Concatenating three columns from table - qt_sql """ - select - id, - map_concat(map1, map2, map3) as all_maps_merged - from ${testTable} - order by id; - """ - - // Test 1.3: Concatenating with literal maps (no table dependency) - qt_sql """ - select - map_concat( - {'a': 'apple'}, - {'b': 'banana'}, - {'c': 'cherry'} - ) as literal_maps_merged; - """ - - // ============================================ - // Group 2: Edge cases and special values - // ============================================ - - // Test 2.1: Concatenating with empty maps - qt_sql """ - select - id, - map_concat(map1, {}) as merged_with_empty1, - map_concat({}, map2) as merged_with_empty2, - map_concat({}, {}) as empty_with_empty - from ${testTable} - order by id; - """ - - // Test 2.2: Concatenating with NULL maps - qt_sql """ - select - id, - map_concat(map1, NULL) as map_with_null, - map_concat(NULL, map2) as null_with_map, - map_concat(NULL, NULL) as null_with_null - from ${testTable} - order by id; - """ - - // Test 2.3: Mixed NULL and empty maps - qt_sql """ - select - map_concat(NULL, {}) as null_with_empty, - map_concat({}, NULL) as empty_with_null; - """ - - // ============================================ - // Group 3: Key conflict and overwrite behavior - // ============================================ - - // Test 3.1: Key conflict (later value overwrites earlier) - qt_sql """ - select - map_concat( - {'a': 'apple', 'b': 'banana'}, - {'b': 'blueberry', 'c': 'cherry'} - ) as conflict_resolution; - """ - - // Test 3.2: Multiple key conflicts across three maps - qt_sql """ - select - map_concat( - {'a': 'first', 'b': 'first'}, - {'b': 'second', 'c': 'second'}, - {'c': 'third', 'd': 'third'} - ) as multi_conflict; - """ - - // ============================================ - // Group 4: Function composition and expressions - // ============================================ - - // Test 4.1: Using map_concat in expressions with map_size - qt_sql """ - select - id, - map_size(map_concat(map1, map1)) as self_concat_size, - map_size(map_concat(map1, map2)) as two_maps_size - from ${testTable} - order by id; - """ - - // Test 4.2: Nested map_concat operations - qt_sql """ - select - map_concat( - map_concat({'a': 1}, {'b': 2}), - map_concat({'c': 3}, {'d': 4}) - ) as nested_concat; - """ - - // Test 4.3: map_concat with other map functions - qt_sql """ - select - map_keys(map_concat({'x': 10}, {'y': 20})) as keys_result, - map_values(map_concat({'x': 10}, {'y': 20})) as values_result; - """ - - // ============================================ - // Group 5: Type handling and error cases - // ============================================ - - // Test 5.1: Concatenating maps with same value types (should work) - qt_sql """ - select - map_concat( - CAST({'a': '1'} AS MAP), - CAST({'b': '2'} AS MAP) - ) as same_type_maps; - """ - - // Test 5.2: Testing with different key types (if supported) - qt_sql """ - select - map_concat( - {1: 'one', 2: 'two'}, - {3: 'three', 4: 'four'} - ) as int_key_maps; - """ - - // Test 5.3: map_concat with many parameters (stress test) - qt_sql """ - select - map_concat( - {'p1': 'v1'}, {'p2': 'v2'}, {'p3': 'v3'}, - {'p4': 'v4'}, {'p5': 'v5'}, {'p6': 'v6'}, - {'p7': 'v7'}, {'p8': 'v8'}, {'p9': 'v9'} - ) as many_maps; - """ - - // Test 5.4: Error case - type mismatch (expected to fail) - qt_sql """ - select - map_concat( - CAST({'a': 1} AS MAP), - CAST({'b': '2'} AS MAP) - ) as type_mismatch; - """ - - // ============================================ - // Group 6: Additional edge cases and extended testing - // ============================================ - - // Test 6.1: Single argument (should return the same map) - qt_sql """ - select - map_concat({'single': 'argument'}) as single_argument; - """ - - // Test 6.2: Map with NULL values inside - qt_sql """ - select - map_concat( - {'a': 'value', 'b': NULL}, - {'c': 'another', 'd': NULL} - ) as maps_with_null_values; - """ - - // Test 6.3: Mixed NULL maps and empty maps - qt_sql """ - select - map_concat(NULL, {'a': 1}, {}, NULL, {'b': 2}) as mixed_nulls_empties; - """ - - // Test 6.4: More numeric types testing - qt_sql """ - select - map_concat( - CAST({'tiny': 127} AS MAP), - CAST({'small': 32767} AS MAP), - CAST({'int': 2147483647} AS MAP) - ) as more_numeric_types; - """ - - // Test 6.5: Boolean type values - qt_sql """ - select - map_concat( - {'flag1': true, 'flag2': false}, - {'flag3': true} - ) as boolean_values; - """ - - // Test 6.6: Date type values - qt_sql """ - select - map_concat( - {'date1': DATE '2023-01-01'}, - {'date2': DATE '2023-12-31'} - ) as date_values; - """ - - // Test 6.7: Array as map values - qt_sql """ - select - map_concat( - {'arr1': [1, 2, 3]}, - {'arr2': [4, 5, 6]} - ) as array_values; - """ - - // Test 6.8: Very many parameters (more than 9) - qt_sql """ - select - map_concat( - {'k01': 'v01'}, {'k02': 'v02'}, {'k03': 'v03'}, - {'k04': 'v04'}, {'k05': 'v05'}, {'k06': 'v06'}, - {'k07': 'v07'}, {'k08': 'v08'}, {'k09': 'v09'}, - {'k10': 'v10'}, {'k11': 'v11'}, {'k12': 'v12'} - ) as many_parameters; - """ - - // Test 6.9: Nested map_concat operations (deep nesting) - qt_sql """ - select - map_concat( - map_concat({'a': 1}, {'b': 2}), - map_concat( - map_concat({'c': 3}, {'d': 4}), - map_concat({'e': 5}, {'f': 6}) - ) - ) as deeply_nested; - """ - - // Test 6.10: map_concat with map_contains_key combination - qt_sql """ - select - map_contains_key( - map_concat({'a': 1, 'b': 2}, {'c': 3, 'd': 4}), - 'c' - ) as contains_key_c, - map_contains_key( - map_concat({'a': 1, 'b': 2}, {'c': 3, 'd': 4}), - 'e' - ) as contains_key_e; - """ - - // Test 6.11: map_concat in WHERE clause - qt_sql """ - select - id, - map_concat(map1, map2) as merged - from ${testTable} - where map_size(map_concat(map1, map2)) > 0 - order by id; - """ - - // Test 6.12: Key order consistency test - qt_sql """ - select - map_keys(map_concat({'z': 1, 'a': 2}, {'m': 3})) as keys_order1, - map_keys(map_concat({'a': 2, 'z': 1}, {'m': 3})) as keys_order2; - """ - - // ============================================ - // Group 7: Advanced edge cases - interface, charset, and nested scenarios - // ============================================ - - // Test 7.1: Interface-related test - map_concat with map_entries - qt_sql """ - select - map_entries(map_concat({'a': 1, 'b': 2}, {'c': 3, 'd': 4})) as entries_result; - """ - - // Test 7.2: Interface-related test - map_concat with map_contains_value - qt_sql """ - select - map_contains_value(map_concat({'a': 1, 'b': 2}, {'c': 3, 'd': 4}), 3) as contains_value_3, - map_contains_value(map_concat({'a': 1, 'b': 2}, {'c': 3, 'd': 4}), 5) as contains_value_5; - """ - - // Test 7.3: Different charset test - UTF-8 special characters - qt_sql """ - select - map_concat( - {'中文键': '中文值', 'key with emoji 🔥': 'value with emoji 🚀'}, - {'key with accents café': 'value with accents naïve'} - ) as utf8_charset_test; - """ - - // Test 7.4: Different charset test - mixed language keys - qt_sql """ - select - map_concat( - {'English key': 'value1', '日本語キー': '値1'}, - {'한국어 키': '값1', 'русский ключ': 'значение1'} - ) as mixed_language_test; - """ - - // Test 7.5: Nested map_concat - complex nesting - qt_sql """ - select - map_concat( - map_concat( - map_concat({'a': 1}, {'b': 2}), - map_concat({'c': 3}, {'d': 4}) - ), - map_concat( - map_concat( - map_concat({'e': 5}, {'f': 6}), - map_concat({'g': 7}, {'h': 8}) - ), - map_concat({'i': 9}, {'j': 10}) - ) - ) as complex_nested_concat; - """ - - // Test 7.6: Nested map_concat with different key types - qt_sql """ - select - map_concat( - map_concat({1: 'one', 2: 'two'}, {3: 'three', 4: 'four'}), - map_concat({'five': 5, 'six': 6}, {'seven': 7, 'eight': 8}) - ) as mixed_key_types_nested; - """ - - // Test 7.7: Interface test - map_concat in subquery - qt_sql """ - select - id, - (select map_concat({'static': 'value'}, map1) from ${testTable} where id = t.id) as subquery_concat - from ${testTable} t - order by id; - """ - - // Test 7.8: Interface test - map_concat with CASE expression - qt_sql """ - select - id, - map_concat( - CASE WHEN id % 2 = 0 THEN {'even': 'true'} ELSE {'odd': 'true'} END, - CASE WHEN id = 1 THEN {'id': '1'} - WHEN id = 2 THEN {'id': '2'} - WHEN id = 3 THEN {'id': '3'} - ELSE {'id': '4'} END - ) as case_expr_concat - from ${testTable} - order by id; - """ - - // Test 7.9: Special characters in keys and values - qt_sql """ - select - map_concat( - {'key with "double quotes"': 'value with ''single quotes'''}, - {'key with \\\\backslashes\\\\': 'value with \\\\n newline'}, - {'key with , comma': 'value with ; semicolon'}, - {'key with {} braces': 'value with [] brackets'} - ) as special_chars_test; - """ - - // Test 7.10: Empty string as key and value - qt_sql """ - select - map_concat( - {'': 'empty key', 'empty value': ''}, - {'': 'another empty key', 'key': ''} - ) as empty_string_test; - """ - - // Test 7.11: Very long strings as keys and values - qt_sql """ - select - map_concat( - {'very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_very_long_key_': - 'very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_very_long_value_'}, - {'another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_another_long_key_': - 'another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_another_long_value_'} - ) as long_strings_test; - """ - - // Test 7.12: map_concat with COALESCE to handle NULLs - qt_sql """ - select - map_concat( - COALESCE(map1, {}), - COALESCE(map2, {}) - ) as coalesce_handled - from ${testTable} - order by id; - """ - } finally { - // Clean up - try { - sql """ - drop table if exists ${testTable}; - """ - } catch (Exception e) { - println "Error cleaning up table ${testTable}: ${e.message}" - } - } + sql """ + drop table if exists ${testTable}; + """ + + sql """ + CREATE TABLE ${testTable} ( + id INT, + map1 MAP, + map2 MAP, + map3 MAP + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """ + insert into ${testTable} values + (1, {'a': 'apple', 'b': 'banana'}, {'x': 10, 'y': 20}, {1: 'one', 2: 'two'}), + (2, {'c': 'cherry'}, {'z': 30}, {3: 'three'}), + (3, {}, {}, {}), + (4, NULL, NULL, NULL); + """ + + qt_sql """ + select map_concat() as empty_map; + """ + + qt_sql """ + select map_concat(map('single', 'argument')) as single_argument; + """ + + qt_sql """ + select map_concat({'a': 'apple'}, {'b': 'banana'}, {'c': 'cherry'}) as literal_maps_merged; + """ + + qt_sql """ + select id, map_concat(map1, map2, map3) as all_maps_merged from ${testTable} order by id; + """ + + qt_sql """ + select + map_concat(map1, {}) as merged_with_empty, + map_concat(map1, NULL) as map_with_null, + map_concat(NULL, NULL) as null_with_null + from ${testTable} where id = 1; + """ + + qt_sql """ + select map_concat({'a': 'apple', 'b': 'banana'}, {'b': 'blueberry', 'c': 'cherry'}) as conflict_resolution; + """ + + qt_sql """ + select + map_size(map_concat(map1, map2)) as two_maps_size, + map_keys(map_concat({'x': 10}, {'y': 20})) as keys_result + from ${testTable} where id = 1; + """ + + qt_sql """ + select + map_concat({1: 'one', 2: 'two'}, {3: 'three', 4: 'four'}) as int_key_maps, + map_concat( + CAST({'bigint1': 10000000000, 'bigint2': 20000000000} AS MAP), + CAST({'bigint3': 30000000000} AS MAP) + ) as bigint_values, + map_concat( + CAST({'double1': 1.2345678901, 'double2': 2.3456789012} AS MAP), + CAST({'double3': 3.4567890123} AS MAP) + ) as double_values, + map_concat({'flag1': true}, {'flag2': false}) as boolean_values, + map_concat({'arr1': [1, 2, 3]}, {'arr2': [4, 5, 6]}) as array_values, + map_concat( + CAST({'decimal1': 123.456, 'decimal2': 789.012} AS MAP), + CAST({'decimal3': 345.678} AS MAP) + ) as decimal_values, + map_concat({'date1': DATE '2023-01-01', 'date2': DATE '2023-12-31'}, + {'timestamp1': TIMESTAMP '2023-01-01 12:00:00'}) as timestamp_values, + -- 类型混合测试：兼容类型间的混合 + map_concat( + CAST({'int_val': 100} AS MAP), + CAST({'bigint_val': 200} AS MAP) + ) as int_bigint_mixed, + map_concat( + CAST({'decimal_val': 123.456} AS MAP), + CAST({'double_val': 789.012} AS MAP) + ) as decimal_double_mixed, + map_concat({'中文键': '中文值', 'key with emoji 🔥': 'value with emoji 🚀'}, + {'key with accents café': 'value with accents naïve'}) as utf8_charset_test; + """ + + qt_sql """ + select + map_concat( + map_concat({'a': 1}, {'b': 2}), + map_concat({'c': 3}, {'d': 4}) + ) as nested_concat, + map_concat( + CAST({'a': 1} AS MAP), + CAST({'b': '2'} AS MAP) + ) as type_mismatch; + """ + + qt_sql """ + select id, map_concat(map1, map2) as merged + from ${testTable} + where map_size(map_concat(map1, map2)) > 0 + order by id; + """ } From 41ed41f089d7783102c7099ae6ea2e71c2e3d346 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Thu, 18 Dec 2025 13:18:21 +0800 Subject: [PATCH 26/41] fix: license issue --- .../vec/function/function_map_concat_test.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/be/test/vec/function/function_map_concat_test.cpp b/be/test/vec/function/function_map_concat_test.cpp index 390a97c3b72586..ec6c0bfca7153c 100644 --- a/be/test/vec/function/function_map_concat_test.cpp +++ b/be/test/vec/function/function_map_concat_test.cpp @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include From b28ec86b1b4790f78e5e752df719ad89921f3e77 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Thu, 18 Dec 2025 13:32:39 +0800 Subject: [PATCH 27/41] fix: int -> size_t --- be/test/vec/function/function_test_util.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/test/vec/function/function_test_util.cpp b/be/test/vec/function/function_test_util.cpp index e161e6e2f7f936..af9db8e5f15925 100644 --- a/be/test/vec/function/function_test_util.cpp +++ b/be/test/vec/function/function_test_util.cpp @@ -366,7 +366,7 @@ bool insert_map_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const AnyTy insert_cell(value_column, value_type, value, datetime_is_string_format); } Array key_array, value_array; - for (int i = 0; i < origin_input_array.size() / 2; i++) { + for (size_t i = 0; i < origin_input_array.size() / 2; i++) { key_array.push_back((*key_column)[i]); value_array.push_back((*value_column)[i]); } From 207cade58d82aff1cf014aafebb63a89e03846d8 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Fri, 19 Dec 2025 19:20:42 +0800 Subject: [PATCH 28/41] fix: remove info log --- be/src/vec/functions/function_map.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/be/src/vec/functions/function_map.cpp b/be/src/vec/functions/function_map.cpp index 3f3c1c8782720e..9f1118ec5c821c 100644 --- a/be/src/vec/functions/function_map.cpp +++ b/be/src/vec/functions/function_map.cpp @@ -90,12 +90,6 @@ class FunctionMap : public IFunction { uint32_t result, size_t input_rows_count) const override { DCHECK(arguments.size() % 2 == 0) << "function: " << get_name() << ", arguments should not be even number"; - LOG(INFO) << "[FunctionMap.execute_impl] input_rows_count: " << input_rows_count; - for (size_t i = 0; i < arguments.size(); ++i) { - auto& col = block.get_by_position(arguments[i]).column; - LOG(INFO) << "[FunctionMap.execute_impl] argument " << i - << " column type: " << col->get_name() << " data:" << col->dump_structure(); - } size_t num_element = arguments.size(); auto result_col = block.get_by_position(result).type->create_column(); From 4d7e3e2f78382ff230dc0a14e09b8b9024d7847b Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Fri, 19 Dec 2025 19:21:37 +0800 Subject: [PATCH 29/41] fix: remove chinese comment --- regression-test/suites/function_p0/test_map_concat.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/regression-test/suites/function_p0/test_map_concat.groovy b/regression-test/suites/function_p0/test_map_concat.groovy index 592eb9531af363..3fb13627b0c434 100644 --- a/regression-test/suites/function_p0/test_map_concat.groovy +++ b/regression-test/suites/function_p0/test_map_concat.groovy @@ -101,7 +101,6 @@ suite("test_map_concat") { ) as decimal_values, map_concat({'date1': DATE '2023-01-01', 'date2': DATE '2023-12-31'}, {'timestamp1': TIMESTAMP '2023-01-01 12:00:00'}) as timestamp_values, - -- 类型混合测试：兼容类型间的混合 map_concat( CAST({'int_val': 100} AS MAP), CAST({'bigint_val': 200} AS MAP) From e3a0e06525ad37fd9d9dce96403ac92f60cbb3d4 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Fri, 26 Dec 2025 22:42:14 +0800 Subject: [PATCH 30/41] feat: assert_cast --- be/src/vec/functions/function_map.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/vec/functions/function_map.cpp b/be/src/vec/functions/function_map.cpp index 9f1118ec5c821c..52610df2b22b88 100644 --- a/be/src/vec/functions/function_map.cpp +++ b/be/src/vec/functions/function_map.cpp @@ -811,7 +811,7 @@ class FunctionMapConcat : public IFunction { ColumnMap* result_map_column = nullptr; ColumnNullable* result_nullable_column = nullptr; if (result_col->is_nullable()) { - result_nullable_column = reinterpret_cast(result_col.get()); + result_nullable_column = assert_cast(result_col.get()); result_map_column = check_and_get_column(result_nullable_column->get_nested_column()); } else { @@ -835,7 +835,7 @@ class FunctionMapConcat : public IFunction { block.get_by_position(col).column->convert_to_full_column_if_const(); if (src_column->is_nullable()) { auto nullable_column = - reinterpret_cast(src_column.get()); + assert_cast(src_column.get()); map_column = check_and_get_column(nullable_column->get_nested_column()); } else { From c85ec749c54eec7d45ca5db7ce91d076229ee798 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Sat, 27 Dec 2025 15:00:56 +0800 Subject: [PATCH 31/41] add: test about map_concat(map3,map1) --- be/src/vec/functions/function_map.cpp | 3 +-- regression-test/data/function_p0/test_map_concat.out | 6 ++++++ regression-test/suites/function_p0/test_map_concat.groovy | 6 ++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/be/src/vec/functions/function_map.cpp b/be/src/vec/functions/function_map.cpp index 52610df2b22b88..1c188bad28c3b1 100644 --- a/be/src/vec/functions/function_map.cpp +++ b/be/src/vec/functions/function_map.cpp @@ -834,8 +834,7 @@ class FunctionMapConcat : public IFunction { auto src_column = block.get_by_position(col).column->convert_to_full_column_if_const(); if (src_column->is_nullable()) { - auto nullable_column = - assert_cast(src_column.get()); + auto nullable_column = assert_cast(src_column.get()); map_column = check_and_get_column(nullable_column->get_nested_column()); } else { diff --git a/regression-test/data/function_p0/test_map_concat.out b/regression-test/data/function_p0/test_map_concat.out index d1d69a82a81481..6b79850984676f 100644 --- a/regression-test/data/function_p0/test_map_concat.out +++ b/regression-test/data/function_p0/test_map_concat.out @@ -29,6 +29,12 @@ -- !sql -- {"a":1, "b":2, "c":3, "d":4} {"a":"1", "b":"2"} +-- !sql -- +1 {"1":"one", "2":"two", "a":"apple", "b":"banana"} +2 {"3":"three", "c":"cherry"} +3 {} +4 \N + -- !sql -- 1 {"a":"apple", "b":"banana", "x":"10", "y":"20"} 2 {"c":"cherry", "z":"30"} diff --git a/regression-test/suites/function_p0/test_map_concat.groovy b/regression-test/suites/function_p0/test_map_concat.groovy index 3fb13627b0c434..43586da6d75aae 100644 --- a/regression-test/suites/function_p0/test_map_concat.groovy +++ b/regression-test/suites/function_p0/test_map_concat.groovy @@ -125,6 +125,12 @@ suite("test_map_concat") { ) as type_mismatch; """ + qt_sql """ + select id, map_concat(map3, map1) as merged_different_key_types + from ${testTable} + order by id; + """ + qt_sql """ select id, map_concat(map1, map2) as merged from ${testTable} From 92ebc69de6532e7c2436aadf807b706860b34e5a Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Fri, 9 Jan 2026 02:19:01 +0800 Subject: [PATCH 32/41] fix: assert_cast --- be/src/vec/functions/function_map.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/be/src/vec/functions/function_map.cpp b/be/src/vec/functions/function_map.cpp index 1c188bad28c3b1..c5b9f0cbc5450a 100644 --- a/be/src/vec/functions/function_map.cpp +++ b/be/src/vec/functions/function_map.cpp @@ -812,10 +812,9 @@ class FunctionMapConcat : public IFunction { ColumnNullable* result_nullable_column = nullptr; if (result_col->is_nullable()) { result_nullable_column = assert_cast(result_col.get()); - result_map_column = - check_and_get_column(result_nullable_column->get_nested_column()); + result_map_column = assert_cast(result_nullable_column->get_nested_column_ptr().get()); } else { - result_map_column = check_and_get_column(result_col.get()); + result_map_column = assert_cast(result_col.get()); } auto& result_col_map_keys_data = result_map_column->get_keys(); auto& result_col_map_vals_data = result_map_column->get_values(); @@ -835,10 +834,9 @@ class FunctionMapConcat : public IFunction { block.get_by_position(col).column->convert_to_full_column_if_const(); if (src_column->is_nullable()) { auto nullable_column = assert_cast(src_column.get()); - map_column = - check_and_get_column(nullable_column->get_nested_column()); + map_column = assert_cast(nullable_column->get_nested_column_ptr().get()); } else { - map_column = check_and_get_column(*src_column.get()); + map_column = assert_cast(src_column.get()); } if (!map_column) { return Status::RuntimeError("unsupported types for function {}({})", get_name(), From 75284461aac76a4b6b43779ef716a74a58e438d7 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Fri, 9 Jan 2026 03:14:25 +0800 Subject: [PATCH 33/41] fix: format --- be/src/vec/functions/function_map.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/be/src/vec/functions/function_map.cpp b/be/src/vec/functions/function_map.cpp index c5b9f0cbc5450a..6e08195f6045fb 100644 --- a/be/src/vec/functions/function_map.cpp +++ b/be/src/vec/functions/function_map.cpp @@ -812,7 +812,8 @@ class FunctionMapConcat : public IFunction { ColumnNullable* result_nullable_column = nullptr; if (result_col->is_nullable()) { result_nullable_column = assert_cast(result_col.get()); - result_map_column = assert_cast(result_nullable_column->get_nested_column_ptr().get()); + result_map_column = + assert_cast(result_nullable_column->get_nested_column_ptr().get()); } else { result_map_column = assert_cast(result_col.get()); } @@ -834,7 +835,8 @@ class FunctionMapConcat : public IFunction { block.get_by_position(col).column->convert_to_full_column_if_const(); if (src_column->is_nullable()) { auto nullable_column = assert_cast(src_column.get()); - map_column = assert_cast(nullable_column->get_nested_column_ptr().get()); + map_column = assert_cast( + nullable_column->get_nested_column_ptr().get()); } else { map_column = assert_cast(src_column.get()); } From 36637240ea2ef543c53e2cf4c0fe558ad3b71859 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Sun, 11 Jan 2026 05:08:01 +0800 Subject: [PATCH 34/41] fix: process map_concat() by be --- be/src/vec/functions/function_map.cpp | 5 + .../functions/scalar/MapConcat.java | 110 ++++++++++-------- .../doris/nereids/util/TypeCoercionUtils.java | 47 -------- 3 files changed, 67 insertions(+), 95 deletions(-) diff --git a/be/src/vec/functions/function_map.cpp b/be/src/vec/functions/function_map.cpp index 6e08195f6045fb..6235841158e786 100644 --- a/be/src/vec/functions/function_map.cpp +++ b/be/src/vec/functions/function_map.cpp @@ -801,6 +801,11 @@ class FunctionMapConcat : public IFunction { bool is_variadic() const override { return true; } size_t get_number_of_arguments() const override { return 0; } DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + if (arguments.empty()) { + return std::make_shared( + make_nullable(std::make_shared()), + make_nullable(std::make_shared())); + } DCHECK(arguments.size() > 0) << "function: " << get_name() << ", arguments should not be empty"; return arguments[0]; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java index f323cef4579249..e0760a29dad6ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; @@ -63,46 +62,6 @@ public MapConcat withChildren(List children) { return new MapConcat(getFunctionParams(children)); } - @Override - public DataType getDataType() { - if (arity() >= 1) { - List keyTypes = new ArrayList<>(); - List valueTypes = new ArrayList<>(); - for (int i = 0; i < children.size(); i++) { - DataType argType = children.get(i).getDataType(); - if (!(argType instanceof MapType)) { - if (!(argType instanceof NullType)) { - throw new AnalysisException("mapconcat function cannot process non-map and non-null child " - + "elements. Invalid SQL: " + this.toSql()); - } - continue; - } - MapType mapType = (MapType) argType; - keyTypes.add(mapType.getKeyType()); - valueTypes.add(mapType.getValueType()); - } - - if (keyTypes.isEmpty() && valueTypes.isEmpty()) { - return MapType.of(NullType.INSTANCE, NullType.INSTANCE); - } - - Optional commonKeyType = TypeCoercionUtils.findWiderCommonType(keyTypes, true, true); - Optional commonValueType = TypeCoercionUtils.findWiderCommonType(valueTypes, true, true); - - if (!commonKeyType.isPresent()) { - throw new AnalysisException("mapconcat cannot find the common key type of " + this.toSql()); - } - if (!commonValueType.isPresent()) { - throw new AnalysisException("mapconcat cannot find the common value type of " + this.toSql()); - } - - DataType keyType = commonKeyType.get(); - DataType valueType = commonValueType.get(); - return MapType.of(keyType, valueType); - } - return MapType.SYSTEM_DEFAULT; - } - @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitMapConcat(this, context); @@ -113,13 +72,68 @@ public List getSignatures() { if (arity() == 0) { return SIGNATURES; } else { - List signatures = ImmutableList.of( - FunctionSignature.of(getDataType(), - children.stream() - .map(ExpressionTrait::getDataType) - .collect(ImmutableList.toImmutableList()) - )); - return signatures; + return computeNonEmptySignatures(); + } + } + + /** + * Compute signatures when arity > 0. + * Extract key and value types, find common type, and construct signatures. + */ + private List computeNonEmptySignatures() { + List children = children(); + + List keyTypes = new ArrayList<>(); + List valueTypes = new ArrayList<>(); + + for (Expression child : children) { + DataType argType = child.getDataType(); + if (argType instanceof MapType) { + MapType mapType = (MapType) argType; + keyTypes.add(mapType.getKeyType()); + valueTypes.add(mapType.getValueType()); + } else if (!(argType instanceof NullType)) { + throw new AnalysisException("mapconcat function cannot process" + + "non-map and non-null child elements. " + + "Invalid SQL: " + toSql()); + } + } + + Optional commonKeyType = TypeCoercionUtils.findWiderCommonType( + keyTypes, true, true); + Optional commonValueType = TypeCoercionUtils.findWiderCommonType( + valueTypes, true, true); + + if (!commonKeyType.isPresent()) { + throw new AnalysisException("mapconcat cannot find the common key type of " + toSql()); + } + if (!commonValueType.isPresent()) { + throw new AnalysisException("mapconcat cannot find the common value type of " + toSql()); + } + + // Build result map type and signatures + DataType retMapType = MapType.of(commonKeyType.get(), commonValueType.get()); + List retArgTypes = buildArgTypes(children, retMapType); + + return ImmutableList.of(FunctionSignature.of(retMapType, retArgTypes)); + } + + /** + * Build argument types with type coercion. + * Map arguments are coerced to the result map type. + */ + private List buildArgTypes(List children, DataType retMapType) { + ImmutableList.Builder retArgTypes = ImmutableList.builder(); + + for (int i = 0; i < children.size(); i++) { + DataType argType = children.get(i).getDataType(); + if (argType instanceof MapType) { + retArgTypes.add(retMapType); + } else { + retArgTypes.add(argType); + } } + + return retArgTypes.build(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 59c307d35da1aa..5c0341cfb945a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -707,57 +707,10 @@ public static Expression processBoundFunction(BoundFunction boundFunction) { return processCreateMap((CreateMap) boundFunction); } - if (boundFunction instanceof MapConcat) { - return processMapConcat((MapConcat) boundFunction); - } - // type coercion return implicitCastInputTypes(boundFunction, boundFunction.expectedInputTypes()); } - private static Expression processMapConcat(MapConcat mapConcat) { - if (mapConcat.arity() == 0) { - return new MapLiteral(); - } - List keyTypes = new ArrayList<>(); - List valueTypes = new ArrayList<>(); - List children = mapConcat.children(); - for (int i = 0; i < children.size(); i++) { - DataType argType = children.get(i).getDataType(); - if (!(argType instanceof MapType)) { - if (!(argType instanceof NullType)) { - throw new AnalysisException("mapconcat function cannot process" - + "non-map and non-null child elements. " - + "Invalid SQL: " + mapConcat.toSql()); - } - continue; - } - MapType mapType = (MapType) argType; - keyTypes.add(mapType.getKeyType()); - valueTypes.add(mapType.getValueType()); - } - Optional commonKeyType = TypeCoercionUtils.findWiderCommonType(keyTypes, true, true); - Optional commonValueType = TypeCoercionUtils.findWiderCommonType(valueTypes, true, true); - if (!commonKeyType.isPresent()) { - throw new AnalysisException("mapconcat cannot find the common key type of " + mapConcat.toSql()); - } - if (!commonValueType.isPresent()) { - throw new AnalysisException("mapconcat cannot find the common value type of " + mapConcat.toSql()); - } - DataType keyType = commonKeyType.get(); - DataType valueType = commonValueType.get(); - DataType targetMapType = MapType.of(keyType, valueType); - ImmutableList.Builder newChildren = ImmutableList.builder(); - for (int i = 0; i < mapConcat.arity(); i++) { - DataType argType = children.get(i).getDataType(); - if (!(argType instanceof MapType)) { - newChildren.add(mapConcat.child(i)); - } else { - newChildren.add(castIfNotSameType(mapConcat.child(i), targetMapType)); - } - } - return mapConcat.withChildren(newChildren.build()); - } private static Expression processCreateMap(CreateMap createMap) { if (createMap.arity() == 0) { From 105c5a68861fe61ad3301db938b324c86b52b575 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Sun, 11 Jan 2026 05:26:41 +0800 Subject: [PATCH 35/41] fix: style --- .../expressions/functions/scalar/MapConcat.java | 15 +++++++-------- .../doris/nereids/util/TypeCoercionUtils.java | 3 --- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java index e0760a29dad6ef..cef8fe23154613 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java @@ -82,10 +82,10 @@ public List getSignatures() { */ private List computeNonEmptySignatures() { List children = children(); - + List keyTypes = new ArrayList<>(); List valueTypes = new ArrayList<>(); - + for (Expression child : children) { DataType argType = child.getDataType(); if (argType instanceof MapType) { @@ -98,23 +98,22 @@ private List computeNonEmptySignatures() { + "Invalid SQL: " + toSql()); } } - Optional commonKeyType = TypeCoercionUtils.findWiderCommonType( keyTypes, true, true); Optional commonValueType = TypeCoercionUtils.findWiderCommonType( valueTypes, true, true); - + if (!commonKeyType.isPresent()) { throw new AnalysisException("mapconcat cannot find the common key type of " + toSql()); } if (!commonValueType.isPresent()) { throw new AnalysisException("mapconcat cannot find the common value type of " + toSql()); } - + // Build result map type and signatures DataType retMapType = MapType.of(commonKeyType.get(), commonValueType.get()); List retArgTypes = buildArgTypes(children, retMapType); - + return ImmutableList.of(FunctionSignature.of(retMapType, retArgTypes)); } @@ -124,7 +123,7 @@ private List computeNonEmptySignatures() { */ private List buildArgTypes(List children, DataType retMapType) { ImmutableList.Builder retArgTypes = ImmutableList.builder(); - + for (int i = 0; i < children.size(); i++) { DataType argType = children.get(i).getDataType(); if (argType instanceof MapType) { @@ -133,7 +132,7 @@ private List buildArgTypes(List children, DataType retMapT retArgTypes.add(argType); } } - + return retArgTypes.build(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 5c0341cfb945a1..5429108be1c280 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -45,7 +45,6 @@ import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; import org.apache.doris.nereids.trees.expressions.functions.scalar.Array; import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateMap; -import org.apache.doris.nereids.trees.expressions.functions.scalar.MapConcat; import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; @@ -707,11 +706,9 @@ public static Expression processBoundFunction(BoundFunction boundFunction) { return processCreateMap((CreateMap) boundFunction); } - // type coercion return implicitCastInputTypes(boundFunction, boundFunction.expectedInputTypes()); } - private static Expression processCreateMap(CreateMap createMap) { if (createMap.arity() == 0) { return new MapLiteral(); From be768a6b980ee28c0ced256ce603d9160e3302f2 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Sat, 17 Jan 2026 01:55:21 +0800 Subject: [PATCH 36/41] fix: comment --- .../java/org/apache/doris/nereids/util/TypeCoercionUtils.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 5429108be1c280..641dd4f76dcb46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -706,6 +706,7 @@ public static Expression processBoundFunction(BoundFunction boundFunction) { return processCreateMap((CreateMap) boundFunction); } + // type coercion return implicitCastInputTypes(boundFunction, boundFunction.expectedInputTypes()); } From 88045c4ce3eec661f94cb5cbb4280a4116568d3b Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Sat, 17 Jan 2026 03:10:17 +0800 Subject: [PATCH 37/41] refactor --- .../functions/scalar/MapConcat.java | 27 ++----------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java index cef8fe23154613..0a497f259b9265 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapConcat.java @@ -39,9 +39,6 @@ */ public class MapConcat extends ScalarFunction implements ExplicitlyCastableSignature, PropagateNullable { - public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(MapType.SYSTEM_DEFAULT).args() - ); /** * constructor with more than 0 arguments. @@ -70,17 +67,9 @@ public R accept(ExpressionVisitor visitor, C context) { @Override public List getSignatures() { if (arity() == 0) { - return SIGNATURES; - } else { - return computeNonEmptySignatures(); + return ImmutableList.of(FunctionSignature.ret(MapType.SYSTEM_DEFAULT).args()); } - } - /** - * Compute signatures when arity > 0. - * Extract key and value types, find common type, and construct signatures. - */ - private List computeNonEmptySignatures() { List children = children(); List keyTypes = new ArrayList<>(); @@ -110,20 +99,8 @@ private List computeNonEmptySignatures() { throw new AnalysisException("mapconcat cannot find the common value type of " + toSql()); } - // Build result map type and signatures DataType retMapType = MapType.of(commonKeyType.get(), commonValueType.get()); - List retArgTypes = buildArgTypes(children, retMapType); - - return ImmutableList.of(FunctionSignature.of(retMapType, retArgTypes)); - } - - /** - * Build argument types with type coercion. - * Map arguments are coerced to the result map type. - */ - private List buildArgTypes(List children, DataType retMapType) { ImmutableList.Builder retArgTypes = ImmutableList.builder(); - for (int i = 0; i < children.size(); i++) { DataType argType = children.get(i).getDataType(); if (argType instanceof MapType) { @@ -133,6 +110,6 @@ private List buildArgTypes(List children, DataType retMapT } } - return retArgTypes.build(); + return ImmutableList.of(FunctionSignature.of(retMapType, retArgTypes.build())); } } From 5c33069c89633cdf685e21f01fc4898e09013bad Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Sat, 17 Jan 2026 17:29:51 +0800 Subject: [PATCH 38/41] fix: comment --- .../java/org/apache/doris/nereids/util/TypeCoercionUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 641dd4f76dcb46..814a63d7649a4e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -705,7 +705,7 @@ public static Expression processBoundFunction(BoundFunction boundFunction) { if (boundFunction instanceof CreateMap) { return processCreateMap((CreateMap) boundFunction); } - + // type coercion return implicitCastInputTypes(boundFunction, boundFunction.expectedInputTypes()); } From c35513dc7cf7c3f9f9f4a369f632740eb3bb8f7d Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Sat, 17 Jan 2026 17:33:25 +0800 Subject: [PATCH 39/41] fix: comment --- .../java/org/apache/doris/nereids/util/TypeCoercionUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 814a63d7649a4e..641dd4f76dcb46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -705,7 +705,7 @@ public static Expression processBoundFunction(BoundFunction boundFunction) { if (boundFunction instanceof CreateMap) { return processCreateMap((CreateMap) boundFunction); } - + // type coercion return implicitCastInputTypes(boundFunction, boundFunction.expectedInputTypes()); } From 49bfdd68f4a896813013aa2289882a776dc2eb06 Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Sun, 18 Jan 2026 23:09:16 +0800 Subject: [PATCH 40/41] fix: ut --- be/test/vec/function/function_map_concat_test.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/be/test/vec/function/function_map_concat_test.cpp b/be/test/vec/function/function_map_concat_test.cpp index ec6c0bfca7153c..6243aa799458e3 100644 --- a/be/test/vec/function/function_map_concat_test.cpp +++ b/be/test/vec/function/function_map_concat_test.cpp @@ -266,12 +266,15 @@ TEST(FunctionMapConcatTest, TestComplexTypes) { InputTypeSet input_types = {PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_DECIMALV2, PrimitiveType::TYPE_MAP, PrimitiveType::TYPE_STRING, PrimitiveType::TYPE_DECIMALV2}; - DataSet data_set = {{TestArray({TestArray({std::string("key1"), ut_type::DECIMALV2(1.5), - std::string("key2"), ut_type::DECIMALV2(2.7)}), - TestArray({std::string("key3"), ut_type::DECIMALV2(3.9)})}), - TestArray({std::string("key1"), ut_type::DECIMALV2(1.5), - std::string("key2"), ut_type::DECIMALV2(2.7), - std::string("key3"), ut_type::DECIMALV2(3.9)})}}; + DataSet data_set = { + {TestArray( + {TestArray({std::string("key1"), ut_type::DECIMALV2VALUEFROMDOUBLE(1.5), + std::string("key2"), ut_type::DECIMALV2VALUEFROMDOUBLE(2.7)}), + TestArray( + {std::string("key3"), ut_type::DECIMALV2VALUEFROMDOUBLE(3.9)})}), + TestArray({std::string("key1"), ut_type::DECIMALV2VALUEFROMDOUBLE(1.5), + std::string("key2"), ut_type::DECIMALV2VALUEFROMDOUBLE(2.7), + std::string("key3"), ut_type::DECIMALV2VALUEFROMDOUBLE(3.9)})}}; check_function_all_arg_comb(func_name, input_types, data_set); } From 6336b48e997499766c829b3bd25eb1b89ebaf81d Mon Sep 17 00:00:00 2001 From: nagisa-kun <1434936049@qq.com> Date: Thu, 29 Jan 2026 01:59:59 +0800 Subject: [PATCH 41/41] fix: format --- be/test/vec/function/function_test_util.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/test/vec/function/function_test_util.h b/be/test/vec/function/function_test_util.h index a77f51b356890d..1c34471a1ec365 100644 --- a/be/test/vec/function/function_test_util.h +++ b/be/test/vec/function/function_test_util.h @@ -59,8 +59,8 @@ #include "vec/data_types/data_type_struct.h" #include "vec/data_types/data_type_time.h" #include "vec/data_types/data_type_varbinary.h" -#include "vec/functions/function_helpers.h" #include "vec/exprs/function_context.h" +#include "vec/functions/function_helpers.h" #include "vec/functions/simple_function_factory.h" namespace doris::vectorized {