diff --git a/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer.h b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer.h index 8b7898f833702d..5e856ae86ce62d 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer.h +++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer.h @@ -22,7 +22,6 @@ #include #include "olap/rowset/segment_v2/inverted_index/token_stream.h" -#include "olap/rowset/segment_v2/inverted_index/util/reader.h" namespace doris::segment_v2::inverted_index { diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index bd23ed3f472de1..652b8bf6c25927 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -238,7 +238,7 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3); break; default: - schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); + schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3); break; } diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 32d96eeb06b54a..99103ae4244258 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -3039,9 +3039,9 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true, description = { "倒排索引默认存储格式", - "Default storage format of inverted index, the default value is V1." + "Default storage format of inverted index, the default value is V3." }) - public static String inverted_index_storage_format = "V2"; + public static String inverted_index_storage_format = "V3"; @ConfField(mutable = true, masterOnly = true, description = { "是否在unique表mow上开启delete语句写delete predicate。若开启，会提升delete语句的性能，" diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index 8bd43890ba5887..a8d61b39c27816 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -350,10 +350,10 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, } else if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.DEFAULT) { if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) { schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V1); - } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V3")) { - schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V3); - } else { + } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V2")) { schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2); + } else { + schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V3); } } else { throw new DdlException("invalid inverted index storage format"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 3da280320f9efe..61ef3ed4143056 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -1194,10 +1194,10 @@ public static TInvertedIndexFileStorageFormat analyzeInvertedIndexFileStorageFor } else { if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) { return TInvertedIndexFileStorageFormat.V1; - } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V3")) { - return TInvertedIndexFileStorageFormat.V3; - } else { + } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V2")) { return TInvertedIndexFileStorageFormat.V2; + } else { + return TInvertedIndexFileStorageFormat.V3; } } @@ -1210,10 +1210,10 @@ public static TInvertedIndexFileStorageFormat analyzeInvertedIndexFileStorageFor } else if (invertedIndexFileStorageFormat.equalsIgnoreCase("default")) { if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) { return TInvertedIndexFileStorageFormat.V1; - } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V3")) { - return TInvertedIndexFileStorageFormat.V3; - } else { + } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V2")) { return TInvertedIndexFileStorageFormat.V2; + } else { + return TInvertedIndexFileStorageFormat.V3; } } else { throw new AnalysisException("unknown inverted index storage format: " + invertedIndexFileStorageFormat); diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java index ddd813df376ba4..926d803f931e09 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java @@ -289,49 +289,55 @@ public void testStoragePageSize() throws AnalysisException { @Test public void testAnalyzeInvertedIndexFileStorageFormat() throws AnalysisException { - TInvertedIndexFileStorageFormat result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(null); - Assertions.assertEquals(TInvertedIndexFileStorageFormat.V2, result); - - Config.inverted_index_storage_format = "V1"; - result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(new HashMap<>()); - Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1, result); - - Map propertiesWithV1 = new HashMap<>(); - propertiesWithV1.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "v1"); - result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV1); - Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1, result); - - Map propertiesWithV2 = new HashMap<>(); - propertiesWithV2.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "v2"); - result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV2); - Assertions.assertEquals(TInvertedIndexFileStorageFormat.V2, result); - - Map propertiesWithV3 = new HashMap<>(); - propertiesWithV3.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "v3"); - result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV3); - Assertions.assertEquals(TInvertedIndexFileStorageFormat.V3, result); - - Config.inverted_index_storage_format = "V1"; - Map propertiesWithDefaultV1 = new HashMap<>(); - propertiesWithDefaultV1.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "default"); - result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithDefaultV1); - Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1, result); - - Config.inverted_index_storage_format = "V2"; - Map propertiesWithDefaultV2 = new HashMap<>(); - propertiesWithDefaultV2.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "default"); - result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithDefaultV2); - Assertions.assertEquals(TInvertedIndexFileStorageFormat.V2, result); - - Map propertiesWithUnknown = new HashMap<>(); - propertiesWithUnknown.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "unknown_format"); + String originFormat = Config.inverted_index_storage_format; try { - PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithUnknown); - Assertions.fail("Expected AnalysisException was not thrown"); - } catch (AnalysisException e) { - Assertions.assertEquals( - "errCode = 2, detailMessage = unknown inverted index storage format: unknown_format", - e.getMessage()); + Config.inverted_index_storage_format = "V3"; + TInvertedIndexFileStorageFormat result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(null); + Assertions.assertEquals(TInvertedIndexFileStorageFormat.V3, result); + + Config.inverted_index_storage_format = "V1"; + result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(new HashMap<>()); + Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1, result); + + Map propertiesWithV1 = new HashMap<>(); + propertiesWithV1.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "v1"); + result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV1); + Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1, result); + + Map propertiesWithV2 = new HashMap<>(); + propertiesWithV2.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "v2"); + result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV2); + Assertions.assertEquals(TInvertedIndexFileStorageFormat.V2, result); + + Map propertiesWithV3 = new HashMap<>(); + propertiesWithV3.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "v3"); + result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV3); + Assertions.assertEquals(TInvertedIndexFileStorageFormat.V3, result); + + Config.inverted_index_storage_format = "V1"; + Map propertiesWithDefaultV1 = new HashMap<>(); + propertiesWithDefaultV1.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "default"); + result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithDefaultV1); + Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1, result); + + Config.inverted_index_storage_format = "V2"; + Map propertiesWithDefaultV2 = new HashMap<>(); + propertiesWithDefaultV2.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "default"); + result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithDefaultV2); + Assertions.assertEquals(TInvertedIndexFileStorageFormat.V2, result); + + Map propertiesWithUnknown = new HashMap<>(); + propertiesWithUnknown.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "unknown_format"); + try { + PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithUnknown); + Assertions.fail("Expected AnalysisException was not thrown"); + } catch (AnalysisException e) { + Assertions.assertEquals( + "errCode = 2, detailMessage = unknown inverted index storage format: unknown_format", + e.getMessage()); + } + } finally { + Config.inverted_index_storage_format = originFormat; } } diff --git a/regression-test/data/inverted_index_p0/storage_format/test_storage_format_default.out b/regression-test/data/inverted_index_p0/storage_format/test_storage_format_default.out new file mode 100644 index 00000000000000..71cd5aaf2b008a --- /dev/null +++ b/regression-test/data/inverted_index_p0/storage_format/test_storage_format_default.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +863 + diff --git a/regression-test/data/query_p0/system/test_table_properties.out b/regression-test/data/query_p0/system/test_table_properties.out index 598a4ea0ea50d3..8d32e2f55dbd3f 100644 --- a/regression-test/data/query_p0/system/test_table_properties.out +++ b/regression-test/data/query_p0/system/test_table_properties.out @@ -20,7 +20,7 @@ internal test_table_properties_db duplicate_table file_cache_ttl_seconds 0 internal test_table_properties_db duplicate_table group_commit_data_bytes 134217728 internal test_table_properties_db duplicate_table group_commit_interval_ms 10000 internal test_table_properties_db duplicate_table in_memory false -internal test_table_properties_db duplicate_table inverted_index_storage_format V2 +internal test_table_properties_db duplicate_table inverted_index_storage_format V3 internal test_table_properties_db duplicate_table is_being_synced false internal test_table_properties_db duplicate_table light_schema_change true internal test_table_properties_db duplicate_table min_load_replica_num -1 @@ -55,7 +55,7 @@ internal test_table_properties_db listtable file_cache_ttl_seconds 0 internal test_table_properties_db listtable group_commit_data_bytes 134217728 internal test_table_properties_db listtable group_commit_interval_ms 10000 internal test_table_properties_db listtable in_memory false -internal test_table_properties_db listtable inverted_index_storage_format V2 +internal test_table_properties_db listtable inverted_index_storage_format V3 internal test_table_properties_db listtable is_being_synced false internal test_table_properties_db listtable light_schema_change true internal test_table_properties_db listtable min_load_replica_num -1 @@ -90,7 +90,7 @@ internal test_table_properties_db unique_table file_cache_ttl_seconds 0 internal test_table_properties_db unique_table group_commit_data_bytes 134217728 internal test_table_properties_db unique_table group_commit_interval_ms 10000 internal test_table_properties_db unique_table in_memory false -internal test_table_properties_db unique_table inverted_index_storage_format V2 +internal test_table_properties_db unique_table inverted_index_storage_format V3 internal test_table_properties_db unique_table is_being_synced false internal test_table_properties_db unique_table light_schema_change true internal test_table_properties_db unique_table min_load_replica_num -1 @@ -127,7 +127,7 @@ internal test_table_properties_db duplicate_table file_cache_ttl_seconds 0 internal test_table_properties_db duplicate_table group_commit_data_bytes 134217728 internal test_table_properties_db duplicate_table group_commit_interval_ms 10000 internal test_table_properties_db duplicate_table in_memory false -internal test_table_properties_db duplicate_table inverted_index_storage_format V2 +internal test_table_properties_db duplicate_table inverted_index_storage_format V3 internal test_table_properties_db duplicate_table is_being_synced false internal test_table_properties_db duplicate_table light_schema_change true internal test_table_properties_db duplicate_table min_load_replica_num -1 @@ -162,7 +162,7 @@ internal test_table_properties_db unique_table file_cache_ttl_seconds 0 internal test_table_properties_db unique_table group_commit_data_bytes 134217728 internal test_table_properties_db unique_table group_commit_interval_ms 10000 internal test_table_properties_db unique_table in_memory false -internal test_table_properties_db unique_table inverted_index_storage_format V2 +internal test_table_properties_db unique_table inverted_index_storage_format V3 internal test_table_properties_db unique_table is_being_synced false internal test_table_properties_db unique_table light_schema_change true internal test_table_properties_db unique_table min_load_replica_num -1 @@ -201,7 +201,7 @@ internal test_table_properties_db duplicate_table file_cache_ttl_seconds 0 internal test_table_properties_db duplicate_table group_commit_data_bytes 134217728 internal test_table_properties_db duplicate_table group_commit_interval_ms 10000 internal test_table_properties_db duplicate_table in_memory false -internal test_table_properties_db duplicate_table inverted_index_storage_format V2 +internal test_table_properties_db duplicate_table inverted_index_storage_format V3 internal test_table_properties_db duplicate_table is_being_synced false internal test_table_properties_db duplicate_table light_schema_change true internal test_table_properties_db duplicate_table min_load_replica_num -1 diff --git a/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out b/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out index fe47469aeb5e47..7929b470514276 100644 --- a/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out +++ b/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out @@ -1,6 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !show -- -show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !select -- 1 1 30 @@ -36,11 +36,11 @@ show_create_table_and_views_nereids_view CREATE VIEW `show_create_table_and_view 300 1 -- !show -- -show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !show -- -show_create_table_and_views_nereids_like CREATE TABLE `show_create_table_and_views_nereids_like` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_nereids_like CREATE TABLE `show_create_table_and_views_nereids_like` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !show -- -show_create_table_and_views_nereids_like_with_rollup CREATE TABLE `show_create_table_and_views_nereids_like_with_rollup` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_nereids_like_with_rollup CREATE TABLE `show_create_table_and_views_nereids_like_with_rollup` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); diff --git a/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_renamed_column.groovy b/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_renamed_column.groovy index c685f2d121519f..c495e207a0cd12 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_renamed_column.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_renamed_column.groovy @@ -97,7 +97,7 @@ suite("test_index_change_on_renamed_column") { String backend_id = tablets[0].BackendId String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) - check_nested_index_file(ip, port, tablet_id, 3, 1, "V2") + check_nested_index_file(ip, port, tablet_id, 3, 1, "V3") // drop inverted index on renamed column sql """ alter table ${tableName} drop index idx_s; """ diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy index 6af721010ad4c4..24bcfb46ef06d2 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy @@ -213,7 +213,7 @@ suite("test_cumulative_compaction_with_format_v2", "inverted_index_format_v2") { assertTrue(hasCompactedRowsets, "Should have compacted rowsets starting from version 2") int segment_count = calc_segment_count(tablet) logger.info("TabletId: " + tablet_id + ", segment_count: " + segment_count) - check_nested_index_file(ip, port, tablet_id, activeRowsetCount, 3, "V2") + check_nested_index_file(ip, port, tablet_id, activeRowsetCount, 3, "V3") } int segmentsCount = 0 diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy index 61aadbcd0bb102..cb438df414234a 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy @@ -175,9 +175,9 @@ suite("test_mor_table_with_format_v2", "inverted_index_format_v2") { String port = backendId_to_backendHttpPort.get(backend_id) // after compaction, there are 1 rwoset in local mode and 2 rowsets in cloud mode. if (isCloudMode()) { - check_nested_index_file(ip, port, tablet_id, 2, 3, "V2") + check_nested_index_file(ip, port, tablet_id, 2, 3, "V3") } else { - check_nested_index_file(ip, port, tablet_id, 1, 3, "V2") + check_nested_index_file(ip, port, tablet_id, 1, 3, "V3") } } diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy index 7fa1a2f93d1fb7..7d99de66f94dd1 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy @@ -211,7 +211,7 @@ suite("test_mow_table_with_format_v2", "inverted_index_format_v2") { boolean hasCompactedRowsets = rowsets.any { it.contains("[2-") } assertTrue(hasBaseRowset, "Should have base rowset [0-1]") assertTrue(hasCompactedRowsets, "Should have compacted rowsets starting from version 2") - check_nested_index_file(ip, port, tablet_id, activeRowsetCount, 3, "V2") + check_nested_index_file(ip, port, tablet_id, activeRowsetCount, 3, "V3") } int segmentsCount = 0 diff --git a/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_default.groovy b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_default.groovy new file mode 100644 index 00000000000000..54d55aa73cb8a5 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_default.groovy @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_storage_format_default", "p0") { + def testTable = "httplogs_dup_default" + + def create_httplogs_dup_table = { tableName -> + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `@timestamp` int(11) NULL, + `clientip` varchar(20) NULL, + `request` string NULL, + `status` int(11) NULL, + `size` int(11) NULL, + INDEX size_idx (`size`) USING INVERTED COMMENT '', + INDEX status_idx (`status`) USING INVERTED COMMENT '', + INDEX clientip_idx (`clientip`) USING INVERTED COMMENT '', + INDEX request_idx (`request`) USING INVERTED PROPERTIES("support_phrase" = "true", "parser" = "english", "lower_case" = "true") COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`@timestamp`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`@timestamp`) BUCKETS 2 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true" + ); + """ + } + + def load_httplogs_data = { tableName, label, readFlag, formatFlag, fileName, ignoreFailure = false, + expectedSuccRows = -1 -> + streamLoad { + table "${tableName}" + + set 'label', label + "_" + UUID.randomUUID().toString() + set 'read_json_by_line', readFlag + set 'format', formatFlag + file fileName + time 10000 + if (expectedSuccRows >= 0) { + set 'max_filter_ratio', '1' + } + + check { result, exception, startTime, endTime -> + if (ignoreFailure && expectedSuccRows < 0) { return } + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + if (expectedSuccRows >= 0) { + assertEquals(json.NumberLoadedRows, expectedSuccRows) + } else { + assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + } + } + + try { + sql "DROP TABLE IF EXISTS ${testTable}" + create_httplogs_dup_table.call(testTable) + + def showCreate = sql """ SHOW CREATE TABLE ${testTable} """ + assertTrue(showCreate.size() > 0) + assertTrue(showCreate[0][1].contains("\"inverted_index_storage_format\" = \"V3\"")) + + load_httplogs_data.call(testTable, 'label', 'true', 'json', 'documents-1000.json') + sql "sync" + + qt_sql("select COUNT(*) from ${testTable} where request match 'images'") + } finally { + sql("DROP TABLE IF EXISTS ${testTable}") + } +} \ No newline at end of file diff --git a/regression-test/suites/variant_p0/predefine/test_multi_index_file.groovy b/regression-test/suites/variant_p0/predefine/test_multi_index_file.groovy index 9e1b73442029d9..61512328f4ee80 100644 --- a/regression-test/suites/variant_p0/predefine/test_multi_index_file.groovy +++ b/regression-test/suites/variant_p0/predefine/test_multi_index_file.groovy @@ -41,5 +41,5 @@ suite("test_variant_multi_index_file", "p0, nonConcurrent"){ String backend_id = tablets[0].BackendId String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) - check_nested_index_file(ip, port, tablet_id, 2, 4, "V2") + check_nested_index_file(ip, port, tablet_id, 2, 4, "V3") } \ No newline at end of file