diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.h b/be/src/olap/rowset/segment_v2/inverted_index_writer.h index 06bc960bc330fe..3b4e5ba2709a7d 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_writer.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.h @@ -75,7 +75,7 @@ class InvertedIndexColumnWriter { // check if the column is valid for inverted index, some columns // are generated from variant, but not all of them are supported - static bool check_column_valid(const TabletColumn& column) { + static bool check_support_inverted_index(const TabletColumn& column) { // bellow types are not supported in inverted index for extracted columns static std::set invalid_types = { FieldType::OLAP_FIELD_TYPE_DOUBLE, diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index 820afaa2103ff3..d6b84e489fe2d5 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -216,9 +216,7 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co } // indexes for this column opts.indexes = schema->get_indexes_for_column(column); - if (!InvertedIndexColumnWriter::check_column_valid(column)) { - // skip inverted index if invalid - opts.indexes.clear(); + if (!InvertedIndexColumnWriter::check_support_inverted_index(column)) { opts.need_zone_map = false; opts.need_bloom_filter = false; opts.need_bitmap_index = false; diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index d763543e7d1dc2..625f506a7a2d97 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -179,9 +179,7 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo } // indexes for this column opts.indexes = tablet_schema->get_indexes_for_column(column); - if (!InvertedIndexColumnWriter::check_column_valid(column)) { - // skip inverted index if invalid - opts.indexes.clear(); + if (!InvertedIndexColumnWriter::check_support_inverted_index(column)) { opts.need_zone_map = false; opts.need_bloom_filter = false; opts.need_bitmap_index = false; diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index ed3016313eb7c0..d8bf40a11714b6 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -1289,6 +1289,10 @@ Result TabletSchema::column(const std::string& field_name) std::vector TabletSchema::get_indexes_for_column( const TabletColumn& col) const { std::vector indexes_for_column; + // Some columns (Float, Double, JSONB ...) from the variant do not support index, but they are listed in TabltetIndex. + if (!segment_v2::InvertedIndexColumnWriter::check_support_inverted_index(col)) { + return indexes_for_column; + } int32_t col_unique_id = col.is_extracted_column() ? col.parent_unique_id() : col.unique_id(); const std::string& suffix_path = col.has_path_info() ? escape_for_path_name(col.path_info_ptr()->get_path()) : ""; @@ -1371,7 +1375,13 @@ const TabletIndex* TabletSchema::get_inverted_index(int32_t col_unique_id, return nullptr; } -const TabletIndex* TabletSchema::get_inverted_index(const TabletColumn& col) const { +const TabletIndex* TabletSchema::get_inverted_index(const TabletColumn& col, + bool check_valid) const { + // With check_valid set to true by default + // Some columns(Float, Double, JSONB ...) from the variant do not support inverted index + if (check_valid && !segment_v2::InvertedIndexColumnWriter::check_support_inverted_index(col)) { + return nullptr; + } // TODO use more efficient impl // Use parent id if unique not assigned, this could happend when accessing subcolumns of variants int32_t col_unique_id = col.is_extracted_column() ? col.parent_unique_id() : col.unique_id(); diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index 262f66ba561ffe..ebfe06f3dc4282 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -372,7 +372,10 @@ class TabletSchema { bool has_inverted_index_with_index_id(int64_t index_id, const std::string& suffix_path) const; const TabletIndex* get_inverted_index_with_index_id(int64_t index_id, const std::string& suffix_name) const; - const TabletIndex* get_inverted_index(const TabletColumn& col) const; + // check_valid: check if this column supports inverted index + // Some columns (Float, Double, JSONB ...) from the variant do not support index, but they are listed in TabletIndex. + // If returned, the index file will not be found. + const TabletIndex* get_inverted_index(const TabletColumn& col, bool check_valid = true) const; const TabletIndex* get_inverted_index(int32_t col_unique_id, const std::string& suffix_path) const; bool has_ngram_bf_index(int32_t col_unique_id) const; diff --git a/be/src/olap/task/index_builder.cpp b/be/src/olap/task/index_builder.cpp index 28ab654b4131d3..b76012d8490f26 100644 --- a/be/src/olap/task/index_builder.cpp +++ b/be/src/olap/task/index_builder.cpp @@ -349,7 +349,7 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta continue; } auto column = output_rowset_schema->column(column_idx); - if (!InvertedIndexColumnWriter::check_column_valid(column)) { + if (!InvertedIndexColumnWriter::check_support_inverted_index(column)) { continue; } DCHECK(output_rowset_schema->has_inverted_index_with_index_id(index_id, "")); diff --git a/be/src/vec/common/schema_util.cpp b/be/src/vec/common/schema_util.cpp index 64f93f98c56680..9f3975576df766 100644 --- a/be/src/vec/common/schema_util.cpp +++ b/be/src/vec/common/schema_util.cpp @@ -389,7 +389,8 @@ void inherit_column_attributes(const TabletColumn& source, TabletColumn& target, // add index meta TabletIndex index_info = *source_index_meta; index_info.set_escaped_escaped_index_suffix_path(target.path_info_ptr()->get_path()); - const auto* target_index_meta = target_schema->get_inverted_index(target); + // get_inverted_index: No need to check, just inherit directly + const auto* target_index_meta = target_schema->get_inverted_index(target, false); if (target_index_meta != nullptr) { // already exist target_schema->update_index(target, index_info); diff --git a/regression-test/data/inverted_index_p0/test_variant_index_format_v1.out b/regression-test/data/inverted_index_p0/test_variant_index_format_v1.out new file mode 100644 index 00000000000000..e2a83121bafe0a --- /dev/null +++ b/regression-test/data/inverted_index_p0/test_variant_index_format_v1.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +\N +\N +\N +4748 + +-- !sql -- +4 + diff --git a/regression-test/suites/inverted_index_p0/test_variant_index_format_v1.groovy b/regression-test/suites/inverted_index_p0/test_variant_index_format_v1.groovy new file mode 100644 index 00000000000000..0cd5b9386964b4 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_variant_index_format_v1.groovy @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_variant_index_format_v1", "p0") { + + def calc_file_crc_on_tablet = { ip, port, tablet -> + return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) + } + def set_be_config = { key, value -> + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + + backend_id = backendId_to_backendIP.keySet()[0] + def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) + logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) + } + + def load_json_data = {table_name, file_name -> + // load the json data + streamLoad { + table "${table_name}" + + // set http request header params + set 'read_json_by_line', 'true' + set 'format', 'json' + set 'max_filter_ratio', '0.1' + file file_name // import json file + time 10000 // limit inflight 10s + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + logger.info("Stream load ${file_name} result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + // assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + } + + def table_name = "github_events" + sql """DROP TABLE IF EXISTS ${table_name}""" + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + k bigint, + v variant, + INDEX idx_var(v) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 + properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1"); + """ + + set_be_config.call("memory_limitation_per_thread_for_schema_change_bytes", "6294967296") + load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") + load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-1.json'}""") + load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-2.json'}""") + load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-3.json'}""") + load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2022-11-07-16.json'}""") + load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2022-11-07-10.json'}""") + load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2022-11-07-22.json'}""") + load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2022-11-07-23.json'}""") + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + + tablets = sql_return_maparray """ show tablets from ${table_name}; """ + String tablet_id = tablets[0].TabletId + String backend_id = tablets[0].BackendId + String ip = backendId_to_backendIP.get(backend_id) + String port = backendId_to_backendHttpPort.get(backend_id) + def (code_0, out_0, err_0) = calc_file_crc_on_tablet(ip, port, tablet_id) + logger.info("Run calc_file_crc_on_tablet: code=" + code_0 + ", out=" + out_0 + ", err=" + err_0) + assertTrue(code_0 == 0) + assertTrue(out_0.contains("crc_value")) + assertTrue(out_0.contains("used_time_ms")) + assertEquals("0", parseJson(out_0.trim()).start_version) + assertEquals("9", parseJson(out_0.trim()).end_version) + assertEquals("9", parseJson(out_0.trim()).rowset_count) + + qt_sql """select cast(v["payload"]["pull_request"]["additions"] as int) from github_events where cast(v["repo"]["name"] as string) = 'xpressengine/xe-core' order by 1;""" + qt_sql """select count() from github_events where cast(v["repo"]["name"] as string) = 'xpressengine/xe-core'""" + set_be_config.call("memory_limitation_per_thread_for_schema_change_bytes", "2147483648") +}