diff --git a/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp b/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp index 0744b5b542845b..a5429814cb4bd3 100644 --- a/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp @@ -199,7 +199,7 @@ Status VariantColumnReader::_new_iterator_with_flat_leaves(ColumnIteratorUPtr* i const auto* node = target_col.has_path_info() ? _subcolumn_readers->find_leaf(relative_path) : nullptr; if (!node) { - if (relative_path.get_path() == SPARSE_COLUMN_PATH) { + if (relative_path.get_path() == SPARSE_COLUMN_PATH && _sparse_column_reader != nullptr) { // read sparse column and filter extracted columns in subcolumn_path_map std::unique_ptr inner_iter; RETURN_IF_ERROR(_sparse_column_reader->new_iterator(&inner_iter, nullptr)); diff --git a/be/src/vec/common/schema_util.cpp b/be/src/vec/common/schema_util.cpp index 2cb18863395d90..0fb5a9901712b3 100644 --- a/be/src/vec/common/schema_util.cpp +++ b/be/src/vec/common/schema_util.cpp @@ -775,6 +775,14 @@ Status VariantCompactionUtil::check_path_stats(const std::vectortablet_schema()->num_variant_columns() == 0) { return Status::OK(); } + // check no extended schema in input rowsets + for (const auto& rowset : intputs) { + for (const auto& column : rowset->tablet_schema()->columns()) { + if (column->is_extracted_column()) { + return Status::OK(); + } + } + } // check no extended schema in output rowset for (const auto& column : output->tablet_schema()->columns()) { if (column->is_extracted_column()) { @@ -829,6 +837,12 @@ Status VariantCompactionUtil::check_path_stats(const std::vectortablet_id()); + } if (original_uid_to_path_stats.at(uid).at(path) != size) { return Status::InternalError( "Path stats not match for uid {} with path `{}`, input size {}, output " @@ -900,7 +914,7 @@ Status VariantCompactionUtil::get_compaction_nested_columns( return Status::OK(); } -void VariantCompactionUtil::get_compaction_subcolumns( +void VariantCompactionUtil::get_compaction_subcolumns_from_subpaths( TabletSchema::PathsSetInfo& paths_set_info, const TabletColumnPtr parent_column, const TabletSchemaSPtr& target, const PathToDataTypes& path_to_data_types, const std::unordered_set& sparse_paths, TabletSchemaSPtr& output_schema) { @@ -966,6 +980,34 @@ void VariantCompactionUtil::get_compaction_subcolumns( } } +void VariantCompactionUtil::get_compaction_subcolumns_from_data_types( + TabletSchema::PathsSetInfo& paths_set_info, const TabletColumnPtr parent_column, + const TabletSchemaSPtr& target, const PathToDataTypes& path_to_data_types, + TabletSchemaSPtr& output_schema) { + const auto& parent_indexes = target->inverted_indexs(parent_column->unique_id()); + for (const auto& [path, data_types] : path_to_data_types) { + if (data_types.empty() || path.empty() || path.has_nested_part()) { + continue; + } + DataTypePtr data_type; + get_least_supertype_jsonb(data_types, &data_type); + auto column_name = parent_column->name_lower_case() + "." + path.get_path(); + auto column_path = PathInData(column_name); + TabletColumn sub_column = get_column_by_type( + data_type, column_name, + vectorized::schema_util::ExtraInfo {.unique_id = -1, + .parent_unique_id = parent_column->unique_id(), + .path_info = column_path}); + vectorized::schema_util::inherit_column_attributes(*parent_column, sub_column); + TabletIndexes sub_column_indexes; + vectorized::schema_util::inherit_index(parent_indexes, sub_column_indexes, sub_column); + paths_set_info.subcolumn_indexes.emplace(path.get_path(), std::move(sub_column_indexes)); + output_schema->append_column(sub_column); + VLOG_DEBUG << "append sub column " << path.get_path() << " data type " + << data_type->get_name(); + } +} + // Build the temporary schema for compaction // 1. aggregate path stats and data types from all rowsets // 2. append typed columns and nested columns to the output schema @@ -986,7 +1028,9 @@ Status VariantCompactionUtil::get_extended_compaction_schema( output_schema->shawdow_copy_without_columns(*target); std::unordered_map uid_to_paths_set_info; for (const TabletColumnPtr& column : target->columns()) { - output_schema->append_column(*column); + if (!column->is_extracted_column()) { + output_schema->append_column(*column); + } if (!column->is_variant_type()) { continue; } @@ -1008,10 +1052,20 @@ Status VariantCompactionUtil::get_extended_compaction_schema( uid_to_paths_set_info[column->unique_id()]); // 4. append subcolumns - get_compaction_subcolumns( - uid_to_paths_set_info[column->unique_id()], column, target, - uid_to_variant_extended_info[column->unique_id()].path_to_data_types, - uid_to_variant_extended_info[column->unique_id()].sparse_paths, output_schema); + if (column->variant_max_subcolumns_count() > 0 || !column->get_sub_columns().empty()) { + get_compaction_subcolumns_from_subpaths( + uid_to_paths_set_info[column->unique_id()], column, target, + uid_to_variant_extended_info[column->unique_id()].path_to_data_types, + uid_to_variant_extended_info[column->unique_id()].sparse_paths, output_schema); + } + // variant_max_subcolumns_count == 0 and no typed paths materialized + // it means that all subcolumns are materialized, may be from old data + else { + get_compaction_subcolumns_from_data_types( + uid_to_paths_set_info[column->unique_id()], column, target, + uid_to_variant_extended_info[column->unique_id()].path_to_data_types, + output_schema); + } // append sparse column TabletColumn sparse_column = create_sparse_column(*column); diff --git a/be/src/vec/common/schema_util.h b/be/src/vec/common/schema_util.h index e6d9d25b5b69b9..9aed56024ea9c6 100644 --- a/be/src/vec/common/schema_util.h +++ b/be/src/vec/common/schema_util.h @@ -192,12 +192,15 @@ class VariantCompactionUtil { segment_v2::VariantStatisticsPB* stats, size_t row_pos, size_t num_rows); - static void get_compaction_subcolumns(TabletSchema::PathsSetInfo& paths_set_info, - const TabletColumnPtr parent_column, - const TabletSchemaSPtr& target, - const PathToDataTypes& path_to_data_types, - const std::unordered_set& sparse_paths, - TabletSchemaSPtr& output_schema); + static void get_compaction_subcolumns_from_subpaths( + TabletSchema::PathsSetInfo& paths_set_info, const TabletColumnPtr parent_column, + const TabletSchemaSPtr& target, const PathToDataTypes& path_to_data_types, + const std::unordered_set& sparse_paths, TabletSchemaSPtr& output_schema); + + static void get_compaction_subcolumns_from_data_types( + TabletSchema::PathsSetInfo& paths_set_info, const TabletColumnPtr parent_column, + const TabletSchemaSPtr& target, const PathToDataTypes& path_to_data_types, + TabletSchemaSPtr& output_schema); static Status get_compaction_typed_columns(const TabletSchemaSPtr& target, const std::unordered_set& typed_paths, diff --git a/be/test/vec/common/schema_util_test.cpp b/be/test/vec/common/schema_util_test.cpp index 3988ed1bb9a624..ab3a77df68cd5a 100644 --- a/be/test/vec/common/schema_util_test.cpp +++ b/be/test/vec/common/schema_util_test.cpp @@ -45,7 +45,7 @@ using namespace doris; class SchemaUtilTest : public testing::Test { public: SchemaUtilTest() = default; - virtual ~SchemaUtilTest() = default; + ~SchemaUtilTest() override = default; }; void construct_column(ColumnPB* column_pb, TabletIndexPB* tablet_index, int64_t index_id, @@ -1452,7 +1452,7 @@ TEST_F(SchemaUtilTest, get_compaction_nested_columns) { EXPECT_FALSE(st2.ok()); } -TEST_F(SchemaUtilTest, get_compaction_subcolumns) { +TEST_F(SchemaUtilTest, get_compaction_subcolumns_from_subpaths) { TabletColumn variant; variant.set_unique_id(30); variant.set_variant_max_subcolumns_count(3); @@ -1470,7 +1470,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns) { std::unordered_set sparse_paths; TabletSchemaSPtr output_schema = std::make_shared(); - schema_util::VariantCompactionUtil::get_compaction_subcolumns( + schema_util::VariantCompactionUtil::get_compaction_subcolumns_from_subpaths( paths_set_info, parent_column, schema, path_to_data_types, sparse_paths, output_schema); EXPECT_EQ(output_schema->num_columns(), 2); for (const auto& column : output_schema->columns()) { @@ -1483,7 +1483,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns) { std::make_shared()}; path_to_data_types[vectorized::PathInData("b")] = { std::make_shared()}; - schema_util::VariantCompactionUtil::get_compaction_subcolumns( + schema_util::VariantCompactionUtil::get_compaction_subcolumns_from_subpaths( paths_set_info, parent_column, schema, path_to_data_types, sparse_paths, output_schema); EXPECT_EQ(output_schema->num_columns(), 2); bool found_int = false, found_str = false; @@ -1500,7 +1500,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns) { output_schema = std::make_shared(); sparse_paths.insert("a"); - schema_util::VariantCompactionUtil::get_compaction_subcolumns( + schema_util::VariantCompactionUtil::get_compaction_subcolumns_from_subpaths( paths_set_info, parent_column, schema, path_to_data_types, sparse_paths, output_schema); EXPECT_EQ(output_schema->num_columns(), 2); for (const auto& column : output_schema->columns()) { @@ -1517,7 +1517,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns) { for (int i = 0; i < config::variant_max_sparse_column_statistics_size + 1; ++i) { sparse_paths.insert("dummy" + std::to_string(i)); } - schema_util::VariantCompactionUtil::get_compaction_subcolumns( + schema_util::VariantCompactionUtil::get_compaction_subcolumns_from_subpaths( paths_set_info, parent_column, schema, path_to_data_types, sparse_paths, output_schema); EXPECT_EQ(output_schema->num_columns(), 2); for (const auto& column : output_schema->columns()) { @@ -1554,7 +1554,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_advanced) { std::unordered_set sparse_paths; TabletSchemaSPtr output_schema = std::make_shared(); - schema_util::VariantCompactionUtil::get_compaction_subcolumns( + schema_util::VariantCompactionUtil::get_compaction_subcolumns_from_subpaths( paths_set_info, parent_column, schema, path_to_data_types, sparse_paths, output_schema); EXPECT_EQ(output_schema->num_columns(), 4); for (const auto& column : output_schema->columns()) { @@ -1571,7 +1571,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_advanced) { std::make_shared()}; path_to_data_types[vectorized::PathInData("b")] = { std::make_shared()}; - schema_util::VariantCompactionUtil::get_compaction_subcolumns( + schema_util::VariantCompactionUtil::get_compaction_subcolumns_from_subpaths( paths_set_info, parent_column, schema, path_to_data_types, sparse_paths, output_schema); EXPECT_EQ(output_schema->num_columns(), 4); bool found_int = false, found_str = false; @@ -1590,7 +1590,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_advanced) { output_schema = std::make_shared(); sparse_paths.insert("a"); - schema_util::VariantCompactionUtil::get_compaction_subcolumns( + schema_util::VariantCompactionUtil::get_compaction_subcolumns_from_subpaths( paths_set_info, parent_column, schema, path_to_data_types, sparse_paths, output_schema); EXPECT_EQ(output_schema->num_columns(), 4); for (const auto& column : output_schema->columns()) { @@ -1609,7 +1609,7 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_advanced) { for (int i = 0; i < config::variant_max_sparse_column_statistics_size + 1; ++i) { sparse_paths.insert("dummy" + std::to_string(i)); } - schema_util::VariantCompactionUtil::get_compaction_subcolumns( + schema_util::VariantCompactionUtil::get_compaction_subcolumns_from_subpaths( paths_set_info, parent_column, schema, path_to_data_types, sparse_paths, output_schema); EXPECT_EQ(output_schema->num_columns(), 4); for (const auto& column : output_schema->columns()) { @@ -1621,6 +1621,56 @@ TEST_F(SchemaUtilTest, get_compaction_subcolumns_advanced) { } } +TEST_F(SchemaUtilTest, get_compaction_subcolumns_from_data_types) { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + + construct_column(schema_pb.add_column(), schema_pb.add_index(), 20000, "v_index_alpha", 1, + "VARIANT", "v1", IndexType::INVERTED); + + TabletSchemaSPtr target = std::make_shared(); + target->init_from_pb(schema_pb); + + TabletColumnPtr parent_column = target->columns().front(); + // Build path -> data types + doris::vectorized::schema_util::PathToDataTypes path_to_data_types; + path_to_data_types[vectorized::PathInData("a")] = { + std::make_shared(), + std::make_shared()}; // -> BIGINT + path_to_data_types[vectorized::PathInData("b")] = { + std::make_shared()}; // -> STRING + + TabletSchemaSPtr output_schema = std::make_shared(); + TabletSchema::PathsSetInfo paths_set_info; + + schema_util::VariantCompactionUtil::get_compaction_subcolumns_from_data_types( + paths_set_info, parent_column, target, path_to_data_types, output_schema); + + EXPECT_EQ(output_schema->num_columns(), 2); + bool found_a = false, found_b = false; + for (const auto& col : output_schema->columns()) { + if (col->name() == "v1.a") { + found_a = true; + EXPECT_EQ(col->type(), FieldType::OLAP_FIELD_TYPE_BIGINT); + EXPECT_EQ(col->parent_unique_id(), 1); + EXPECT_EQ(col->path_info_ptr()->get_path(), "v1.a"); + } else if (col->name() == "v1.b") { + found_b = true; + EXPECT_EQ(col->type(), FieldType::OLAP_FIELD_TYPE_STRING); + EXPECT_EQ(col->parent_unique_id(), 1); + EXPECT_EQ(col->path_info_ptr()->get_path(), "v1.b"); + } + } + EXPECT_TRUE(found_a && found_b); + + ASSERT_TRUE(paths_set_info.subcolumn_indexes.find("a") != + paths_set_info.subcolumn_indexes.end()); + ASSERT_TRUE(paths_set_info.subcolumn_indexes.find("b") != + paths_set_info.subcolumn_indexes.end()); + EXPECT_EQ(paths_set_info.subcolumn_indexes["a"].size(), 1); + EXPECT_EQ(paths_set_info.subcolumn_indexes["b"].size(), 1); +} + // Test has_different_structure_in_same_path function indirectly through check_variant_has_no_ambiguous_paths TEST_F(SchemaUtilTest, has_different_structure_in_same_path_indirect) { // Test case 1: Same structure and same length - should not detect ambiguity diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 3ddb207e951a6b..3c6edbf569c397 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1059,10 +1059,6 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP throw new DdlException("Not supporting alter table add generated columns."); } - if (newColumn.getType().isVariantType() && olapTable.hasVariantColumns()) { - checkAddVariantColumnAllowed(olapTable, newColumn); - } - /* * add new column to indexes. * UNIQUE: @@ -3458,20 +3454,4 @@ private void checkOrder(List targetIndexSchema, List orderedColN nameSet.add(colName); } } - - private void checkAddVariantColumnAllowed(OlapTable olapTable, Column newColumn) throws DdlException { - int currentCount = newColumn.getVariantMaxSubcolumnsCount(); - for (Column column : olapTable.getBaseSchema()) { - if (column.getType().isVariantType()) { - if (currentCount == 0 && column.getVariantMaxSubcolumnsCount() != 0) { - throw new DdlException("The variant_max_subcolumns_count must either be 0 in all columns" - + " or greater than 0 in all columns"); - } - if (currentCount > 0 && column.getVariantMaxSubcolumnsCount() == 0) { - throw new DdlException("The variant_max_subcolumns_count must either be 0 in all columns" - + " or greater than 0 in all columns"); - } - } - } - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index 72829fe13cb6a5..4e6e8ea6e449be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -346,9 +346,7 @@ public void validate(ConnectContext ctx) { Preconditions.checkState(!Strings.isNullOrEmpty(ctlName), "catalog name is null or empty"); Preconditions.checkState(!Strings.isNullOrEmpty(dbName), "database name is null or empty"); - //check datatype: datev1, decimalv2, variant - boolean allZero = false; - boolean allPositive = false; + //check datatype: datev1, decimalv2 for (ColumnDefinition columnDef : columns) { String columnNameUpperCase = columnDef.getName().toUpperCase(); if (columnNameUpperCase.startsWith("__DORIS_")) { @@ -361,20 +359,6 @@ public void validate(ConnectContext ctx) { "Disable to create table of `VARIANT` type column named with a `.` character: " + columnNameUpperCase); } - VariantType variantType = (VariantType) columnDef.getType(); - if (variantType.getVariantMaxSubcolumnsCount() == 0) { - allZero = true; - if (allPositive) { - throw new AnalysisException("The variant_max_subcolumns_count must either be 0" - + " in all columns, or greater than 0 in all columns"); - } - } else { - allPositive = true; - if (allZero) { - throw new AnalysisException("The variant_max_subcolumns_count must either be 0" - + " in all columns, or greater than 0 in all columns"); - } - } } if (columnDef.getType().isDateType() && Config.disable_datev1) { throw new AnalysisException( diff --git a/regression-test/data/variant_p0/predefine/test_predefine_ddl.out b/regression-test/data/variant_p0/predefine/test_predefine_ddl.out index 617122ab75fbc9..a65c3ec1fb5c4b 100644 --- a/regression-test/data/variant_p0/predefine/test_predefine_ddl.out +++ b/regression-test/data/variant_p0/predefine/test_predefine_ddl.out @@ -7,5 +7,5 @@ var variant Yes false \N NON id bigint Yes true \N var variant Yes false \N NONE var2 variant Yes false \N NONE -var3 variant Yes false \N NONE +var3 variant Yes false \N NONE diff --git a/regression-test/data/variant_p0/predefine/variant_different_max_subcolumns_count.out b/regression-test/data/variant_p0/predefine/variant_different_max_subcolumns_count.out new file mode 100644 index 00000000000000..91980ae970961f --- /dev/null +++ b/regression-test/data/variant_p0/predefine/variant_different_max_subcolumns_count.out @@ -0,0 +1,75 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 hello 1.1 1 {"a":1,"b":"hello","c":1.1} +\N \N 2.2 2 {"c":2.2} +3 world 3.3 3 {"a":3,"b":"world","c":3.3} +\N world 4.4 4 {"b":"world","c":4.4} +5 \N 5.5 5 {"a":5,"c":5.5} + +-- !sql -- +1 hello 1.1 1 {"a":1,"b":"hello","c":1.1} +\N \N 2.2 2 {"c":2.2} +3 world 3.3 3 {"a":3,"b":"world","c":3.3} +\N world 4.4 4 {"b":"world","c":4.4} +5 \N 5.5 5 {"a":5,"c":5.5} + +-- !sql -- +1 hello 1.1 1 hello 1.1 1 hello 1.1 1 hello 1.1 1 {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":"1","b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} +\N \N 2.2 \N \N 2.2 \N \N 2.2 \N \N 2.2 2 {"c":2.2} {"c":2.2} {"c":2.2} {"c":2.2} +3 world 3.3 3 world 3.3 3 world 3.3 3 world 3.3 3 {"a":3,"b":"world","c":3.3} {"a":3,"b":"world","c":3.3} {"a":"3","b":"world","c":3.3} {"a":3,"b":"world","c":3.3} +\N world 4.4 \N world 4.4 \N world 4.4 \N world 4.4 4 {"b":"world","c":4.4} {"b":"world","c":4.4} {"b":"world","c":4.4} {"b":"world","c":4.4} +5 \N 5.5 5 \N 5.5 5 \N 5.5 5 \N 5.5 5 {"a":5,"c":5.5} {"a":5,"c":5.5} {"a":"5","c":5.5} {"a":5,"c":5.5} +5 world \N 5 world \N 5 world \N 5 world \N 6 {"a":5,"b":"world"} {"a":5,"b":"world"} {"a":"5","b":"world"} {"a":5,"b":"world"} +1 \N \N 1 \N \N 1 \N \N 1 \N \N 7 {"a":1} {"a":1} {"a":"1"} {"a":1} +\N 1 \N \N 1 \N \N 1 \N \N 1 \N 8 {"b":"1"} {"b":"1"} {"b":"1"} {"b":"1"} + +-- !sql -- +1 hello 1.1 1 hello 1.1 1 hello 1.1 1 hello 1.1 1 {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":"1","b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} +\N \N 2.2 \N \N 2.2 \N \N 2.2 \N \N 2.2 2 {"c":2.2} {"c":2.2} {"c":2.2} {"c":2.2} +3 world 3.3 3 world 3.3 3 world 3.3 3 world 3.3 3 {"a":3,"b":"world","c":3.3} {"a":3,"b":"world","c":3.3} {"a":"3","b":"world","c":3.3} {"a":3,"b":"world","c":3.3} +\N world 4.4 \N world 4.4 \N world 4.4 \N world 4.4 4 {"b":"world","c":4.4} {"b":"world","c":4.4} {"b":"world","c":4.4} {"b":"world","c":4.4} +5 \N 5.5 5 \N 5.5 5 \N 5.5 5 \N 5.5 5 {"a":5,"c":5.5} {"a":5,"c":5.5} {"a":"5","c":5.5} {"a":5,"c":5.5} +5 world \N 5 world \N 5 world \N 5 world \N 6 {"a":5,"b":"world"} {"a":5,"b":"world"} {"a":"5","b":"world"} {"a":5,"b":"world"} +1 \N \N 1 \N \N 1 \N \N 1 \N \N 7 {"a":1} {"a":1} {"a":"1"} {"a":1} +\N 1 \N \N 1 \N \N 1 \N \N 1 \N 8 {"b":"1"} {"b":"1"} {"b":"1"} {"b":"1"} + +-- !sql -- +1 hello 1.1 \N \N \N 1 {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} +1 hello 1.1 \N \N \N 1 {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} +\N \N 2.2 2.2 \N \N 2 {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} +\N \N 2.2 2.2 \N \N 2 {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} +\N \N \N \N 3 3.3 3 {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} +\N \N \N \N 3 3.3 3 {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} + +-- !sql -- +1 hello 1.1 \N \N \N 1 {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} +1 hello 1.1 \N \N \N 1 {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} +\N \N 2.2 2.2 \N \N 2 {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} +\N \N 2.2 2.2 \N \N 2 {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} +\N \N \N \N 3 3.3 3 {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} +\N \N \N \N 3 3.3 3 {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} + +-- !sql -- +1 hello 1.1 \N \N \N 1 {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} +1 hello 1.1 \N \N \N 1 {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} +\N \N 2.2 2.2 \N \N 2 {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} +\N \N 2.2 2.2 \N \N 2 {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} +\N \N \N \N 3 3.3 3 {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} +\N \N \N \N 3 3.3 3 {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} +1 hello 1.1 1.1 3 3.3 4 {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} +1 hello 1.1 1.1 3 3.3 5 {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} +1 hello 1.1 1.1 3 3.3 6 {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} +1 hello 1.1 1.1 3 3.3 7 {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} + +-- !sql -- +1 hello 1.1 \N \N \N 1 {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} +1 hello 1.1 \N \N \N 1 {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} {"a":1,"b":"hello","c":1.1} +\N \N 2.2 2.2 \N \N 2 {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} +\N \N 2.2 2.2 \N \N 2 {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} {"c":2.2,"d":2.2} +\N \N \N \N 3 3.3 3 {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} +\N \N \N \N 3 3.3 3 {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} {"e":"3","f":"world","g":3.3,"h":3.3} +1 hello 1.1 1.1 3 3.3 4 {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} +1 hello 1.1 1.1 3 3.3 5 {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} +1 hello 1.1 1.1 3 3.3 6 {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} +1 hello 1.1 1.1 3 3.3 7 {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} {"a":1,"b":"hello","c":1.1,"d":1.1,"e":"3","f":"world","g":3.3,"h":3.3} + diff --git a/regression-test/suites/variant_github_events_new_p0/load.groovy b/regression-test/suites/variant_github_events_new_p0/load.groovy index 9438cc441aa90a..6ce128c4ee5626 100644 --- a/regression-test/suites/variant_github_events_new_p0/load.groovy +++ b/regression-test/suites/variant_github_events_new_p0/load.groovy @@ -44,7 +44,6 @@ suite("regression_test_variant_github_events_p0", "p0"){ } } } - int max_subcolumns_count = Math.floor(Math.random() * 50) + 1 boolean enable_typed_paths_to_sparse = new Random().nextBoolean() def table_name = "github_events" sql """DROP TABLE IF EXISTS ${table_name}""" @@ -52,7 +51,7 @@ suite("regression_test_variant_github_events_p0", "p0"){ sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( k bigint, - v variant<'payload.pull_request.head.repo.topics' : array, properties("variant_max_subcolumns_count" = "${max_subcolumns_count}", "variant_enable_typed_paths_to_sparse" = "${enable_typed_paths_to_sparse}")>, + v variant<'payload.pull_request.head.repo.topics' : array, properties("variant_enable_typed_paths_to_sparse" = "${enable_typed_paths_to_sparse}")>, INDEX idx_var(v) USING INVERTED COMMENT '' ) DUPLICATE KEY(`k`) @@ -138,7 +137,7 @@ suite("regression_test_variant_github_events_p0", "p0"){ } } } - sql """ALTER TABLE github_events ADD COLUMN v2 variant DEFAULT NULL""" + sql """ALTER TABLE github_events ADD COLUMN v2 variant DEFAULT NULL""" for(int t = 0; t <= 10; t += 1){ long k = 9223372036854775107 + t sql """INSERT INTO github_events VALUES (${k}, '{"aaaa" : 1234, "bbbb" : "11ssss"}', '{"xxxx" : 1234, "yyyy" : [1.111]}')""" diff --git a/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy b/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy index 8a364e6bb8d553..6c632bcca4cc8d 100644 --- a/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy +++ b/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy @@ -72,6 +72,9 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ sql "set enable_variant_flatten_nested = true" table_name = "github_events" int rand_subcolumns_count = Math.floor(Math.random() * (611 - 511 + 1)) + 511 + if ((rand_subcolumns_count % 2) == 0) { + rand_subcolumns_count = 0 + } sql "set enable_variant_flatten_nested = true" sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( diff --git a/regression-test/suites/variant_p0/agg.groovy b/regression-test/suites/variant_p0/agg.groovy index 97999a122fffd0..cec9d0ba520bee 100644 --- a/regression-test/suites/variant_p0/agg.groovy +++ b/regression-test/suites/variant_p0/agg.groovy @@ -18,7 +18,7 @@ suite("regression_test_variant_agg"){ sql """DROP TABLE IF EXISTS var_agg""" - int max_subcolumns_count = Math.floor(Math.random() * 10) + 1 + int max_subcolumns_count = Math.floor(Math.random() * 10) def var = "variant replace" if (max_subcolumns_count % 2 == 0) { var = "variant <'d' : int, 'b.f' : int, 'xxxx' : string, 'point' : bigint, properties(\"variant_max_subcolumns_count\" = \"${max_subcolumns_count}\")> replace" diff --git a/regression-test/suites/variant_p0/predefine/delete_update.groovy b/regression-test/suites/variant_p0/predefine/delete_update.groovy index bd1c917862bb9a..39fac8dd4604e8 100644 --- a/regression-test/suites/variant_p0/predefine/delete_update.groovy +++ b/regression-test/suites/variant_p0/predefine/delete_update.groovy @@ -21,8 +21,7 @@ suite("regression_test_variant_predefine_delete_and_update", "variant_type"){ // MOR def table_name = "var_delete_update" sql "DROP TABLE IF EXISTS ${table_name}" - int max_subcolumns_count = Math.floor(Math.random() * 5) + 1 - def var = "variant <'a' : largeint, 'b' : array, 'c' : double, 'd' : text, properties(\"variant_max_subcolumns_count\" = \"${max_subcolumns_count}\")>" + def var = "variant <'a' : largeint, 'b' : array, 'c' : double, 'd' : text>" sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( k bigint, diff --git a/regression-test/suites/variant_p0/predefine/insert_into_select.groovy b/regression-test/suites/variant_p0/predefine/insert_into_select.groovy index c8c86949dea0f4..4d92d2822dca87 100644 --- a/regression-test/suites/variant_p0/predefine/insert_into_select.groovy +++ b/regression-test/suites/variant_p0/predefine/insert_into_select.groovy @@ -17,8 +17,6 @@ suite("regression_test_variant_predefine_insert_into_select", "variant_type"){ def table_name = "insert_into_select" - def max_subcolumns_count = new Random().nextInt(5) + 1 - sql "set default_variant_max_subcolumns_count = ${max_subcolumns_count}" sql "DROP TABLE IF EXISTS ${table_name}_var" sql "DROP TABLE IF EXISTS ${table_name}_str" sql """ diff --git a/regression-test/suites/variant_p0/predefine/load.groovy b/regression-test/suites/variant_p0/predefine/load.groovy index 1e07d3509469df..3df4136568f686 100644 --- a/regression-test/suites/variant_p0/predefine/load.groovy +++ b/regression-test/suites/variant_p0/predefine/load.groovy @@ -17,7 +17,7 @@ suite("regression_test_variant_predefine_schema", "p0"){ sql """DROP TABLE IF EXISTS test_predefine""" - def count = new Random().nextInt(10) + 1; + def count = new Random().nextInt(10); if (new Random().nextInt(100) < 50) { count = "1000" } diff --git a/regression-test/suites/variant_p0/predefine/multi_var.groovy b/regression-test/suites/variant_p0/predefine/multi_var.groovy index 00da5b0e3499ec..75b020323152c2 100644 --- a/regression-test/suites/variant_p0/predefine/multi_var.groovy +++ b/regression-test/suites/variant_p0/predefine/multi_var.groovy @@ -16,17 +16,13 @@ // under the License. suite("regression_test_variant_predefine_multi_var", "variant_type"){ - int max_subcolumns_count = Math.floor(Math.random() * 7) + 1 def table_name = "multi_variants" - boolean enable_typed_paths_to_sparse = new Random().nextBoolean() - sql "set default_variant_enable_typed_paths_to_sparse = ${enable_typed_paths_to_sparse}" + sql "DROP TABLE IF EXISTS ${table_name}" sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( k bigint, - v variant<'k1' : int, 'k2' : string, 'k3' : array, 'k4' : double, - properties("variant_max_subcolumns_count" = "${max_subcolumns_count}", "variant_enable_typed_paths_to_sparse" = "${enable_typed_paths_to_sparse}") - >, + v variant<'k1' : int, 'k2' : string, 'k3' : array, 'k4' : double> ) DUPLICATE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 4 @@ -36,21 +32,20 @@ suite("regression_test_variant_predefine_multi_var", "variant_type"){ sql """INSERT INTO ${table_name} SELECT *, '{"k7":123, "k8": "elden ring", "k9" : 1.1112, "k10" : [1.12], "k11" : ["moon"]}' FROM numbers("number" = "203") where number > 100""" sql """INSERT INTO ${table_name} SELECT *, '{"k7":123, "k8": "elden ring", "k9" : 1.1112, "k10" : [1.12], "k11" : ["moon"]}' FROM numbers("number" = "411") where number > 200""" trigger_and_wait_compaction(table_name, "cumulative") - sql """alter table ${table_name} add column v2 variant<'k1' : int, 'k2' : string, 'k3' : array, 'k4' : double , - properties(\"variant_max_subcolumns_count\" = \"${max_subcolumns_count}\", \"variant_enable_typed_paths_to_sparse\" = \"${enable_typed_paths_to_sparse}\")> default null""" - sql """INSERT INTO ${table_name} select k, v, v from ${table_name}""" - sql """alter table ${table_name} add column v3 variant<'k1' : int, 'k2' : string, 'k3' : array, 'k4' : double, - properties(\"variant_max_subcolumns_count\" = \"${max_subcolumns_count}\", \"variant_enable_typed_paths_to_sparse\" = \"${enable_typed_paths_to_sparse}\")> default null""" - sql """INSERT INTO ${table_name} select k, v, v, v from ${table_name}""" + sql """alter table ${table_name} add column v2 variant<'k1' : int, 'k2' : string, 'k3' : array, 'k4' : double> default null""" + sql """INSERT INTO ${table_name} select k, v, cast(v as string) from ${table_name}""" + + sql """alter table ${table_name} add column v3 variant<'k1' : int, 'k2' : string, 'k3' : array, 'k4' : double> default null""" + sql """INSERT INTO ${table_name} select k, v, cast(v as string), cast(v as string) from ${table_name}""" sql "alter table ${table_name} add column ss string default null" - sql """INSERT INTO ${table_name} select k, v, v, v, v from ${table_name}""" + sql """INSERT INTO ${table_name} select k, v, cast(v as string), cast(v as string), cast(v as string) from ${table_name}""" sql """DELETE FROM ${table_name} where k = 1""" trigger_and_wait_compaction(table_name, "cumulative") qt_sql """select cast(v["k1"] as tinyint), cast(v2["k2"] as text), cast(v3["k3"] as string), cast(v["k7"] as tinyint), cast(v2["k8"] as text), cast(v3["k9"] as double) from ${table_name} order by k, 1, 2, 3, 4, 5, 6 limit 10""" qt_sql """select cast(v["k1"] as tinyint), cast(v2["k2"] as text), cast(v3["k3"] as string), cast(v["k7"] as tinyint), cast(v2["k8"] as text), cast(v3["k9"] as double) from ${table_name} where k > 200 order by k, 1, 2, 3, 4, 5, 6 limit 10""" qt_sql """select cast(v["k1"] as tinyint), cast(v2["k2"] as text), cast(v3["k3"] as string), cast(v["k7"] as tinyint), cast(v2["k8"] as text), cast(v3["k9"] as double) from ${table_name} where k > 300 order by k, 1, 2, 3, 4, 5, 6 limit 10""" - sql "alter table ${table_name} add column v4 variant default null" + sql "alter table ${table_name} add column v4 variant<'k1' : int, 'k2' : string, 'k3' : array, 'k4' : double> default null" for (int i = 0; i < 20; i++) { sql """insert into ${table_name} values (1, '{"a" : 1}', '{"a" : 1}', '{"a" : 1}', '{"a" : 1}', '{"a" : 1}')""" } diff --git a/regression-test/suites/variant_p0/predefine/test_multi_index_nonCurrent.groovy b/regression-test/suites/variant_p0/predefine/test_multi_index_nonCurrent.groovy index 01fc6d56e89591..1b422fd244e852 100644 --- a/regression-test/suites/variant_p0/predefine/test_multi_index_nonCurrent.groovy +++ b/regression-test/suites/variant_p0/predefine/test_multi_index_nonCurrent.groovy @@ -212,7 +212,7 @@ suite("test_variant_multi_index_nonCurrent", "p0, nonConcurrent") { } tableName = "test_variant_predefine_types_with_multi_indexes" sql "DROP TABLE IF EXISTS ${tableName}" - def max_subcolumns_count = new Random().nextInt(10) + 1 + def max_subcolumns_count = new Random().nextInt(10) sql "set default_variant_max_subcolumns_count = ${max_subcolumns_count}" sql """ CREATE TABLE ${tableName} ( diff --git a/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy b/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy index 71f49276c8d205..99ec6d84088471 100644 --- a/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy +++ b/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy @@ -220,7 +220,7 @@ suite("test_predefine_ddl", "p0") { INDEX idx_ab_2 (var) USING INVERTED PROPERTIES("field_pattern"="ab") COMMENT '' ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" - exception("column: var cannot have multiple inverted indexes with field pattern: ab") + exception("column: var cannot have multiple inverted indexes of the same type with field pattern: ab") } sql "DROP TABLE IF EXISTS ${tableName}" @@ -245,7 +245,7 @@ suite("test_predefine_ddl", "p0") { INDEX idx_ab_2 (var) USING INVERTED PROPERTIES("field_pattern"="ab", "parser"="unicode", "support_phrase" = "true") COMMENT '' ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" - exception("column: var cannot have multiple inverted indexes with field pattern: ab") + exception("column: var cannot have multiple inverted indexes of the same type with field pattern: ab") } test { @@ -259,7 +259,7 @@ suite("test_predefine_ddl", "p0") { INDEX idx_ab_2 (var) USING INVERTED PROPERTIES("field_pattern"="ab") COMMENT '' ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" - exception("column: var cannot have multiple inverted indexes with field pattern: ab") + exception("column: var cannot have multiple inverted indexes of the same type with field pattern: ab") } test { @@ -273,7 +273,7 @@ suite("test_predefine_ddl", "p0") { INDEX idx_ab_2 (var) USING INVERTED PROPERTIES("field_pattern"="ab") COMMENT '' ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" - exception("column: var cannot have multiple inverted indexes with field pattern: ab") + exception("column: var cannot have multiple inverted indexes of the same type with field pattern: ab") } test { @@ -311,18 +311,6 @@ suite("test_predefine_ddl", "p0") { ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1")""" - - test { - sql "DROP TABLE IF EXISTS ${tableName}" - sql """CREATE TABLE ${tableName} ( - `id` bigint NULL, - `var1` variant NULL, - `var2` variant NULL - ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) - BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1")""" - exception("The variant_max_subcolumns_count must either be 0 in all columns, or greater than 0 in all columns") - } - sql "DROP TABLE IF EXISTS ${tableName}" sql "set default_variant_max_subcolumns_count = 10" sql "set default_variant_enable_typed_paths_to_sparse = false" @@ -348,12 +336,7 @@ suite("test_predefine_ddl", "p0") { sql """alter table ${tableName} add column var2 variant NULL""" wait_for_latest_op_on_table_finish("${tableName}", timeout) - test { - sql """alter table ${tableName} add column var3 variant NULL""" - exception("The variant_max_subcolumns_count must either be 0 in all columns or greater than 0 in all columns") - } - - sql "alter table ${tableName} add column var3 variant NULL" + sql """alter table ${tableName} add column var3 variant NULL""" wait_for_latest_op_on_table_finish("${tableName}", timeout) qt_sql "desc ${tableName}" diff --git a/regression-test/suites/variant_p0/predefine/test_predefine_pattern.groovy b/regression-test/suites/variant_p0/predefine/test_predefine_pattern.groovy index b39fa53bf2c14a..382dd6c0e769cf 100644 --- a/regression-test/suites/variant_p0/predefine/test_predefine_pattern.groovy +++ b/regression-test/suites/variant_p0/predefine/test_predefine_pattern.groovy @@ -20,7 +20,7 @@ suite("test_variant_predefine_base", "p0"){ sql """ set enable_match_without_inverted_index = false """ sql """ set enable_common_expr_pushdown = true """ sql """ set default_variant_enable_typed_paths_to_sparse = false """ - def count = new Random().nextInt(5) + 1 + def count = new Random().nextInt(5) def tableName = "base_match_name_variant_test" sql "DROP TABLE IF EXISTS ${tableName}" sql """CREATE TABLE ${tableName} ( diff --git a/regression-test/suites/variant_p0/predefine/variant_different_max_subcolumns_count.groovy b/regression-test/suites/variant_p0/predefine/variant_different_max_subcolumns_count.groovy new file mode 100644 index 00000000000000..4fd39346f02b87 --- /dev/null +++ b/regression-test/suites/variant_p0/predefine/variant_different_max_subcolumns_count.groovy @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("variant_different_max_subcolumns_count", "p0") { + + + def table_name = "variant_different_max_subcolumns_count" + sql "DROP TABLE IF EXISTS ${table_name}" + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + k bigint, + v variant<'a' : int, 'b' : string, properties("variant_max_subcolumns_count" = "0", "variant_enable_typed_paths_to_sparse" = "false")>, + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 + properties("replication_num" = "1", "disable_auto_compaction" = "true"); + """ + + sql """INSERT INTO ${table_name} values(1, '{"a": "1", "b": "hello", "c": 1.1}'), (2, '{"c": 2.2}')""" + sql """INSERT INTO ${table_name} values(3, '{"a": "3", "b": "world", "c": 3.3}')""" + sql """INSERT INTO ${table_name} values(4, '{"b": "world", "c": 4.4}')""" + sql """INSERT INTO ${table_name} values(5, '{"a": "5", "c": 5.5}')""" + + qt_sql "select v['a'], v['b'], v['c'], * from ${table_name} order by k" + trigger_and_wait_compaction(table_name, "full") + qt_sql "select v['a'], v['b'], v['c'], * from ${table_name} order by k" + + sql "DROP TABLE IF EXISTS ${table_name}" + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + k bigint, + v variant<'a' : int, 'b' : string, properties("variant_max_subcolumns_count" = "0", "variant_enable_typed_paths_to_sparse" = "false")>, + v2 variant<'a' : int, 'b' : string, properties("variant_max_subcolumns_count" = "1", "variant_enable_typed_paths_to_sparse" = "false")>, + v3 variant, + v4 variant<'a' : int, 'b' : string, properties("variant_max_subcolumns_count" = "1", "variant_enable_typed_paths_to_sparse" = "true")> + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 + properties("replication_num" = "1", "disable_auto_compaction" = "true"); + """ + + sql """INSERT INTO ${table_name} values(1, '{"a": "1", "b": "hello", "c": 1.1}', '{"a": "1", "b": "hello", "c": 1.1}', '{"a": "1", "b": "hello", "c": 1.1}', '{"a": "1", "b": "hello", "c": 1.1}')""" + sql """INSERT INTO ${table_name} values(2, '{"c": 2.2}', '{"c": 2.2}', '{"c": 2.2}', '{"c": 2.2}')""" + sql """INSERT INTO ${table_name} values(3, '{"a": "3", "b": "world", "c": 3.3}', '{"a": "3", "b": "world", "c": 3.3}', '{"a": "3", "b": "world", "c": 3.3}', '{"a": "3", "b": "world", "c": 3.3}')""" + sql """INSERT INTO ${table_name} values(4, '{"b": "world", "c": 4.4}', '{"b": "world", "c": 4.4}', '{"b": "world", "c": 4.4}', '{"b": "world", "c": 4.4}')""" + sql """INSERT INTO ${table_name} values(5, '{"a": "5", "c": 5.5}', '{"a": "5", "c": 5.5}', '{"a": "5", "c": 5.5}', '{"a": "5", "c": 5.5}')""" + sql """INSERT INTO ${table_name} values(6, '{"a" : "5", "b" : "world"}', '{"a" : "5", "b" : "world"}', '{"a" : "5", "b" : "world"}', '{"a" : "5", "b" : "world"}')""" + sql """INSERT INTO ${table_name} values(7, '{"a" : "1"}', '{"a" : "1"}', '{"a" : "1"}', '{"a" : "1"}')""" + sql """INSERT INTO ${table_name} values(8, '{"b" : "1"}', '{"b" : "1"}', '{"b" : "1"}', '{"b" : "1"}')""" + + qt_sql "select v['a'], v['b'], v['c'], v2['a'], v2['b'], v2['c'], v3['a'], v3['b'], v3['c'], v4['a'], v4['b'], v4['c'], * from ${table_name} order by k" + + trigger_and_wait_compaction(table_name, "full") + qt_sql "select v['a'], v['b'], v['c'], v2['a'], v2['b'], v2['c'], v3['a'], v3['b'], v3['c'], v4['a'], v4['b'], v4['c'], * from ${table_name} order by k" + + sql "DROP TABLE IF EXISTS ${table_name}" + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + k bigint, + v variant<'a' : int, 'b' : string, properties("variant_max_subcolumns_count" = "0")>, + v2 variant<'a' : int, 'b' : string, properties("variant_max_subcolumns_count" = "1")>, + v3 variant<'a' : int, 'b' : string, properties("variant_max_subcolumns_count" = "3")>, + v4 variant<'a' : int, 'b' : string, properties("variant_max_subcolumns_count" = "5")>, + v5 variant<'a' : int, 'b' : string, properties("variant_max_subcolumns_count" = "7")> + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 + properties("replication_num" = "1", "disable_auto_compaction" = "true"); + """ + sql """INSERT INTO ${table_name} values(1, '{"a": "1", "b": "hello", "c": 1.1}', '{"a": "1", "b": "hello", "c": 1.1}', '{"a": "1", "b": "hello", "c": 1.1}', '{"a": "1", "b": "hello", "c": 1.1}', '{"a": "1", "b": "hello", "c": 1.1}')""" + sql """INSERT INTO ${table_name} values(2, '{"c": 2.2, "d": 2.2}', '{"c": 2.2, "d": 2.2}', '{"c": 2.2, "d": 2.2}', '{"c": 2.2, "d": 2.2}', '{"c": 2.2, "d": 2.2}')""" + sql """INSERT INTO ${table_name} values(3, '{"e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"e": "3", "f": "world", "g": 3.3, "h": 3.3}')""" + sql """INSERT INTO ${table_name} values(1, '{"a": "1", "b": "hello", "c": 1.1}', '{"a": "1", "b": "hello", "c": 1.1}', '{"a": "1", "b": "hello", "c": 1.1}', '{"a": "1", "b": "hello", "c": 1.1}', '{"a": "1", "b": "hello", "c": 1.1}')""" + sql """INSERT INTO ${table_name} values(2, '{"c": 2.2, "d": 2.2}', '{"c": 2.2, "d": 2.2}', '{"c": 2.2, "d": 2.2}', '{"c": 2.2, "d": 2.2}', '{"c": 2.2, "d": 2.2}')""" + sql """INSERT INTO ${table_name} values(3, '{"e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"e": "3", "f": "world", "g": 3.3, "h": 3.3}')""" + + qt_sql "select v['a'], v2['b'], v3['c'], v4['d'], v5['e'], v5['g'], * from ${table_name} order by k" + + trigger_and_wait_compaction(table_name, "full") + qt_sql "select v['a'], v2['b'], v3['c'], v4['d'], v5['e'], v5['g'], * from ${table_name} order by k" + + sql """ INSERT INTO ${table_name} values(4, '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}')""" + sql """ INSERT INTO ${table_name} values(5, '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}')""" + sql """ INSERT INTO ${table_name} values(6, '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}')""" + sql """ INSERT INTO ${table_name} values(7, '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}', '{"a": "1", "b": "hello", "c": 1.1, "d": 1.1, "e": "3", "f": "world", "g": 3.3, "h": 3.3}')""" + + + qt_sql "select v['a'], v2['b'], v3['c'], v4['d'], v5['e'], v5['g'], * from ${table_name} order by k" + trigger_and_wait_compaction(table_name, "full") + qt_sql "select v['a'], v2['b'], v3['c'], v4['d'], v5['e'], v5['g'], * from ${table_name} order by k" + +} \ No newline at end of file diff --git a/regression-test/suites/variant_p0/predefine/variant_hirachinal.groovy b/regression-test/suites/variant_p0/predefine/variant_hirachinal.groovy index 032dfba2b7b91c..211b3363d48312 100644 --- a/regression-test/suites/variant_p0/predefine/variant_hirachinal.groovy +++ b/regression-test/suites/variant_p0/predefine/variant_hirachinal.groovy @@ -18,7 +18,7 @@ suite("regression_test_variant_predefine_hirachinal", "variant_type"){ def table_name = "var_rs" sql "DROP TABLE IF EXISTS ${table_name}" - int count = new Random().nextInt(10) + 1 + int count = new Random().nextInt(10) sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( diff --git a/regression-test/suites/variant_p0/schema_change/test_add_multi_var_mulit_indexes.groovy b/regression-test/suites/variant_p0/schema_change/test_add_multi_var_mulit_indexes.groovy index dc452e67bc5415..35e86f7076b00e 100644 --- a/regression-test/suites/variant_p0/schema_change/test_add_multi_var_mulit_indexes.groovy +++ b/regression-test/suites/variant_p0/schema_change/test_add_multi_var_mulit_indexes.groovy @@ -38,24 +38,18 @@ suite("regression_test_variant_add_multi_var_mulit_indexes", "variant_type"){ assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") } def table_name = "variant_add_multi_var_mulit_indexes" - int count = new Random().nextInt(10) + 3 - sql "set default_variant_max_subcolumns_count = ${count}" sql "set default_variant_enable_typed_paths_to_sparse = false" sql "DROP TABLE IF EXISTS ${table_name}" sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( k bigint, - v variant + v variant ) DUPLICATE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1", "disable_auto_compaction" = "true"); """ sql """insert into ${table_name} values (0, '{"a" : 12345,"b" : 2}')""" - test { - sql """alter table ${table_name} add column var2 variant NULL""" - exception("The variant_max_subcolumns_count must either be 0 in all columns or greater than 0 in all columns") - } sql """ alter table ${table_name} add column v2 variant<'a': string, 'b': string> NULL"""