diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index 86d3857e639cfa..4cae49219dea49 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -1029,26 +1029,6 @@ void TabletSchema::append_index(TabletIndex&& index) { } } -void TabletSchema::update_index(const TabletColumn& col, const IndexType& index_type, - std::vector&& indexes) { - int32_t col_unique_id = col.is_extracted_column() ? col.parent_unique_id() : col.unique_id(); - const std::string& suffix_path = escape_for_path_name(col.suffix_path()); - IndexKey key(index_type, col_unique_id, suffix_path); - auto iter = _col_id_suffix_to_index.find(key); - if (iter != _col_id_suffix_to_index.end()) { - if (iter->second.size() == indexes.size()) { - for (size_t i = 0; i < iter->second.size(); ++i) { - size_t pos = iter->second[i]; - if (pos < _indexes.size()) { - _indexes[pos] = std::make_shared(std::move(indexes[i])); - } - } - } - } - LOG(WARNING) << " failed to update_index: " << index_type << " " << col_unique_id << " " - << suffix_path; -} - void TabletSchema::replace_column(size_t pos, TabletColumn new_col) { CHECK_LT(pos, num_columns()) << " outof range"; _cols[pos] = std::make_shared(std::move(new_col)); diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index 9d3d740cedb2ee..884233f9db3f5a 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -508,9 +508,6 @@ class TabletSchema : public MetadataAdder { bool has_inverted_index_with_index_id(int64_t index_id) const; - void update_index(const TabletColumn& column, const IndexType& index_type, - std::vector&& indexes); - std::vector inverted_indexs(const TabletColumn& col) const; std::vector inverted_indexs(int32_t col_unique_id, diff --git a/be/src/vec/common/schema_util.cpp b/be/src/vec/common/schema_util.cpp index e9e6810ce248e9..500f13a0983d7f 100644 --- a/be/src/vec/common/schema_util.cpp +++ b/be/src/vec/common/schema_util.cpp @@ -438,21 +438,25 @@ void inherit_column_attributes(const TabletColumn& source, TabletColumn& target, } // 2. inverted index - std::vector indexes_to_update; + TabletIndexes indexes_to_add; auto source_indexes = (*target_schema)->inverted_indexs(source.unique_id()); - for (const auto& source_index_meta : source_indexes) { - TabletIndex index_info = *source_index_meta; - index_info.set_escaped_escaped_index_suffix_path(target.path_info_ptr()->get_path()); - indexes_to_update.emplace_back(std::move(index_info)); + // if target is variant type, we need to inherit all indexes + // because this schema is a read schema from fe + if (target.is_variant_type()) { + for (auto& index : source_indexes) { + auto index_info = std::make_shared(*index); + index_info->set_escaped_escaped_index_suffix_path(target.path_info_ptr()->get_path()); + indexes_to_add.emplace_back(std::move(index_info)); + } + } else { + inherit_index(source_indexes, indexes_to_add, target); } auto target_indexes = (*target_schema) ->inverted_indexs(target.parent_unique_id(), target.path_info_ptr()->get_path()); - if (!target_indexes.empty()) { - (*target_schema)->update_index(target, IndexType::INVERTED, std::move(indexes_to_update)); - } else { - for (auto& index_info : indexes_to_update) { - (*target_schema)->append_index(std::move(index_info)); + if (target_indexes.empty()) { + for (auto& index_info : indexes_to_add) { + (*target_schema)->append_index(std::move(*index_info)); } } diff --git a/be/test/olap/tablet_schema_index_test.cpp b/be/test/olap/tablet_schema_index_test.cpp index 763dda26a37628..33472e9c4f0807 100644 --- a/be/test/olap/tablet_schema_index_test.cpp +++ b/be/test/olap/tablet_schema_index_test.cpp @@ -146,42 +146,6 @@ TEST_F(TabletSchemaIndexTest, TestClearIndexes) { EXPECT_TRUE(_tablet_schema->inverted_indexes().empty()); } -TEST_F(TabletSchemaIndexTest, TestUpdateIndexMethod) { - TabletColumn col; - col.set_parent_unique_id(100); - col.set_path_info(vectorized::PathInData("v2")); - _tablet_schema->append_column(col); - - TabletIndex old_index = create_test_index(1, IndexType::INVERTED, {100}, "v2"); - _tablet_schema->append_index(std::move(old_index)); - - TabletIndex new_index = create_test_index(1, IndexType::INVERTED, {100}, "v2"); - new_index._properties["new_prop"] = "value"; - - _tablet_schema->update_index(col, IndexType::INVERTED, {std::move(new_index)}); - - auto updated_indexs = _tablet_schema->inverted_indexs(100, "v2"); - ASSERT_FALSE(updated_indexs.empty()); - EXPECT_EQ(updated_indexs[0]->index_id(), 1); - EXPECT_EQ(updated_indexs[0]->properties().at("new_prop"), "value"); - - auto key = std::make_tuple(IndexType::INVERTED, 100, "v2"); - EXPECT_NE(_tablet_schema->_col_id_suffix_to_index.find(key), - _tablet_schema->_col_id_suffix_to_index.end()); -} - -TEST_F(TabletSchemaIndexTest, TestUpdateIndexAddNewWhenNotExist) { - // Not exist, return nullptr - TabletColumn col; - col.set_unique_id(200); - - TabletIndex new_index = create_test_index(2, IndexType::INVERTED, {200}, "v3"); - _tablet_schema->update_index(col, IndexType::INVERTED, {std::move(new_index)}); - - auto indexs = _tablet_schema->inverted_indexs(200, "v3"); - ASSERT_TRUE(indexs.empty()); -} - TEST_F(TabletSchemaIndexTest, TestUpdateIndexWithMultipleColumns) { TabletColumn col1, col2; col1.set_unique_id(300); diff --git a/be/test/olap/tablet_schema_test.cpp b/be/test/olap/tablet_schema_test.cpp index e10ea56ad9fddd..6b9665d780a4f9 100644 --- a/be/test/olap/tablet_schema_test.cpp +++ b/be/test/olap/tablet_schema_test.cpp @@ -325,54 +325,6 @@ TEST_F(TabletSchemaTest, test_tablet_schema_append_index) { EXPECT_EQ(201, indexes[0]->index_id()); } -TEST_F(TabletSchemaTest, test_tablet_schema_update_indexs) { - TabletSchema schema; - - TabletColumn col; - col.set_unique_id(5001); - col.set_name("update_col"); - col.set_type(FieldType::OLAP_FIELD_TYPE_STRING); - schema.append_column(col); - - std::vector indexes; - for (int i = 0; i < 3; ++i) { - TabletIndex index; - TabletIndexPB index_pb; - index_pb.set_index_id(300 + i); - index_pb.set_index_name("update_idx_" + std::to_string(i)); - index_pb.set_index_type(IndexType::INVERTED); - index_pb.add_col_unique_id(5001); - index.init_from_pb(index_pb); - schema.append_index(std::move(index)); - } - - std::vector new_indexes; - - for (int i = 0; i < 3; ++i) { - TabletIndex index; - TabletIndexPB index_pb; - index_pb.set_index_id(300 + i); - index_pb.set_index_name("update_idx_" + std::to_string(i)); - index_pb.set_index_type(IndexType::INVERTED); - index_pb.add_col_unique_id(5001); - index.init_from_pb(index_pb); - new_indexes.push_back(std::move(index)); - } - - schema.update_index(col, IndexType::INVERTED, std::move(indexes)); - - EXPECT_TRUE(schema.has_inverted_index()); - auto all_indexes = schema.inverted_indexs(5001); - EXPECT_EQ(3, all_indexes.size()); - - std::set expected_names = {"update_idx_0", "update_idx_1", "update_idx_2"}; - std::set actual_names; - for (const auto* index : all_indexes) { - actual_names.insert(index->index_name()); - } - EXPECT_EQ(expected_names, actual_names); -} - TEST_F(TabletSchemaTest, test_tablet_column_protobuf_roundtrip) { TabletColumn original; original.set_unique_id(6001); diff --git a/be/test/vec/common/schema_util_test.cpp b/be/test/vec/common/schema_util_test.cpp index 3988ed1bb9a624..d5667d025bc024 100644 --- a/be/test/vec/common/schema_util_test.cpp +++ b/be/test/vec/common/schema_util_test.cpp @@ -303,61 +303,6 @@ TEST_F(SchemaUtilTest, test_multiple_index_inheritance) { } } -TEST_F(SchemaUtilTest, test_index_update_logic) { - TabletSchemaPB schema_pb; - schema_pb.set_keys_type(KeysType::DUP_KEYS); - schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); - - construct_column(schema_pb.add_column(), schema_pb.add_index(), 10000, "v1_index_orig1", 1, - "VARIANT", "v1", IndexType::INVERTED); - construct_column(schema_pb.add_column(), schema_pb.add_index(), 10001, "v1_index_orig2", 1, - "VARIANT", "v1", IndexType::INVERTED); - - TabletSchemaSPtr tablet_schema = std::make_shared(); - tablet_schema->init_from_pb(schema_pb); - std::vector subcolumns; - - construct_subcolumn(tablet_schema, FieldType::OLAP_FIELD_TYPE_STRING, 1, "v1.name", - &subcolumns); - vectorized::schema_util::inherit_column_attributes(tablet_schema); - - const auto& subcol = subcolumns[0]; - auto initial_indexes = tablet_schema->inverted_indexs(subcol); - ASSERT_EQ(initial_indexes.size(), 2); - EXPECT_EQ(initial_indexes[0]->index_name(), "v1_index_orig1"); - EXPECT_EQ(initial_indexes[1]->index_name(), "v1_index_orig2"); - - std::vector updated_indexes; - TabletIndexPB tablet_index_pb1; - tablet_index_pb1.set_index_id(10002); - tablet_index_pb1.set_index_name("v1_index_updated1"); - tablet_index_pb1.set_index_type(IndexType::INVERTED); - tablet_index_pb1.add_col_unique_id(1); - TabletIndex tablet_index1; - tablet_index1.init_from_pb(tablet_index_pb1); - updated_indexes.emplace_back(std::move(tablet_index1)); - - TabletIndexPB tablet_index_pb2; - tablet_index_pb2.set_index_id(10003); - tablet_index_pb2.set_index_name("v1_index_updated2"); - tablet_index_pb2.set_index_type(IndexType::INVERTED); - tablet_index_pb2.add_col_unique_id(1); - TabletIndex tablet_index2; - tablet_index2.init_from_pb(tablet_index_pb2); - updated_indexes.emplace_back(std::move(tablet_index2)); - - tablet_schema->update_index(tablet_schema->column(1), IndexType::INVERTED, - std::move(updated_indexes)); - - vectorized::schema_util::inherit_column_attributes(tablet_schema); - auto updated_subcol_indexes = tablet_schema->inverted_indexs(subcol); - - EXPECT_EQ(updated_subcol_indexes.size(), 2); - EXPECT_EQ(updated_subcol_indexes[0]->index_name(), "v1_index_updated1"); - EXPECT_EQ(updated_subcol_indexes[1]->index_name(), "v1_index_updated2"); - EXPECT_EQ(updated_subcol_indexes[0]->get_index_suffix(), "v1%2Ename"); -} - // static std::unordered_map construct_column_map_with_random_values( // auto& column_map, int key_size, int value_size, const std::string& prefix) { // std::unordered_map key_value_counts; diff --git a/regression-test/suites/variant_p0/with_index/var_index.groovy b/regression-test/suites/variant_p0/with_index/var_index.groovy index 21b2f5f6f007da..5642ad111bfb3c 100644 --- a/regression-test/suites/variant_p0/with_index/var_index.groovy +++ b/regression-test/suites/variant_p0/with_index/var_index.groovy @@ -156,4 +156,100 @@ suite("regression_test_variant_var_index", "p0, nonConcurrent"){ } } + + sql """ DROP TABLE IF EXISTS ${table_name} """ + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + k bigint, + v variant, + INDEX idx_var(v) USING INVERTED PROPERTIES("parser" = "english") COMMENT '', + INDEX idx_var2(v) USING INVERTED + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 + properties("replication_num" = "1", "disable_auto_compaction" = "true"); + """ + + sql """insert into var_index values(1, '{"name": "张三", "age": 18}')""" + sql """ select * from var_index """ + + def timeout = 60000 + def delta_time = 1000 + def alter_res = "null" + def useTime = 0 + + def wait_for_latest_op_on_table_finish = { table, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${table}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(3000) // wait change table state to normal + logger.info(table + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + + def get_indeces_count = { + def tablets = sql_return_maparray """ show tablets from ${table_name}; """ + + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + + String tablet_id = tablets[0].TabletId + String backend_id = tablets[0].BackendId + String ip = backendId_to_backendIP.get(backend_id) + String port = backendId_to_backendHttpPort.get(backend_id) + def (code, out, err) = http_client("GET", String.format("http://%s:%s/api/show_nested_index_file?tablet_id=%s", ip, port, tablet_id)) + logger.info("Run show_nested_index_file_on_tablet: code=" + code + ", out=" + out + ", err=" + err) + if (parseJson(out.trim()).status == "E-6003") { + return 0 + } + def rowset_count = parseJson(out.trim()).rowsets.size(); + def indices_count = 0 + for (def rowset in parseJson(out.trim()).rowsets) { + for (int i = 0; i < rowset.segments.size(); i++) { + def segment = rowset.segments[i] + assertEquals(i, segment.segment_id) + if (segment.indices != null) { + indices_count += segment.indices.size() + } + } + } + return indices_count + } + + assertEquals(3, get_indeces_count()) + + sql """ drop index idx_var2 on ${table_name} """ + wait_for_latest_op_on_table_finish(table_name, timeout) + sql """ insert into ${table_name} values(2, '{"name": "李四", "age": 20}') """ + sql """ select * from ${table_name} """ + if (isCloudMode()) { + assertEquals(4, get_indeces_count()) + } else { + assertEquals(5, get_indeces_count()) + } + + + sql """ insert into ${table_name} values(2, '{"name": "李四", "age": 20}') """ + sql """ insert into ${table_name} values(2, '{"name": "李四", "age": 20}') """ + sql """ insert into ${table_name} values(2, '{"name": "李四", "age": 20}') """ + sql """ select * from ${table_name} """ + if (isCloudMode()) { + assertEquals(10, get_indeces_count()) + } else { + assertEquals(11, get_indeces_count()) + } + + sql """ drop index idx_var on ${table_name} """ + wait_for_latest_op_on_table_finish(table_name, timeout) + sql """ insert into ${table_name} values(2, '{"name": "李四", "age": 20}') """ + sql """ select * from ${table_name} """ + trigger_and_wait_compaction(table_name, "full") + assertEquals(0, get_indeces_count()) } \ No newline at end of file