From 6f1cc728b15c1e90ebd10d882fb72e1966860fdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Wed, 25 Jun 2025 01:41:07 +0800 Subject: [PATCH 01/14] [feature](index change)Support light index change for inverted index without parser --- .../doris/alter/SchemaChangeHandler.java | 95 ++++++-- .../doris/analysis/BuildIndexClause.java | 6 +- .../java/org/apache/doris/catalog/Index.java | 11 +- .../plans/commands/info/BuildIndexOp.java | 5 +- .../org/apache/doris/qe/SessionVariable.java | 17 ++ .../doris/alter/IndexChangeJobTest.java | 74 ++++-- ...test_ngram_bloomfilter_index_change.groovy | 226 +++++++++++------- 7 files changed, 301 insertions(+), 133 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index c9fca5724e2d23..2078001ee645fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1258,12 +1258,7 @@ private void checkAssignedTargetIndexName(String baseIndexName, String targetInd } private void createJob(String rawSql, long dbId, OlapTable olapTable, Map> indexSchemaMap, - Map propertyMap, List indexes, - boolean isBuildIndex) throws UserException { - if (isBuildIndex) { - // remove the index which is not the base index, only base index can be built index - indexSchemaMap.entrySet().removeIf(entry -> !entry.getKey().equals(olapTable.getBaseIndexId())); - } + Map propertyMap, List indexes) throws UserException { checkReplicaCount(olapTable); // process properties first @@ -1299,7 +1294,7 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map newSet = new HashSet<>(indexes); Set oriSet = new HashSet<>(olapTable.getIndexes()); - if (!newSet.equals(oriSet) || isBuildIndex) { + if (!newSet.equals(oriSet)) { hasIndexChange = true; } @@ -2092,9 +2087,21 @@ public int getAsInt() { } lightSchemaChange = false; + // Check if the index supports light index change and session variable is enabled + boolean enableLightAddIndex = true; + try { + ConnectContext context = ConnectContext.get(); + if (context != null && context.getSessionVariable() != null) { + enableLightAddIndex = context.getSessionVariable().isEnableLightAddIndex(); + } + } catch (Exception e) { + LOG.warn("Failed to get session variable enable_light_index_change, " + + "using default value: true", e); + } + // ngram_bf index can do light_schema_change in both local and cloud mode // inverted index can only do light_schema_change in local mode - if (index.isLightIndexChangeSupported()) { + if (index.isLightIndexChangeSupported() && enableLightAddIndex) { alterIndexes.add(index); isDropIndex = false; lightIndexChange = true; @@ -2103,7 +2110,7 @@ public int getAsInt() { BuildIndexClause buildIndexClause = (BuildIndexClause) alterClause; IndexDef indexDef = buildIndexClause.getIndexDef(); Index index = buildIndexClause.getIndex(); - if (Config.isCloudMode() && index.getIndexType() == IndexDef.IndexType.INVERTED) { + if (Config.isCloudMode()) { throw new DdlException("BUILD INDEX operation failed: No need to do it in cloud mode."); } @@ -2165,17 +2172,12 @@ public int getAsInt() { if (alterIndexes.isEmpty()) { throw new DdlException("Altered index is empty. please check your alter stmt."); } - IndexDef.IndexType indexType = alterIndexes.get(0).getIndexType(); if (Config.enable_light_index_change) { - if (indexType == IndexDef.IndexType.INVERTED) { - buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, - alterIndexes, indexOnPartitions, false); - } else { - createJob(rawSql, db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes, true); - } + buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, + alterIndexes, indexOnPartitions, false); } } else { - createJob(rawSql, db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes, false); + createJob(rawSql, db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes); } } finally { olapTable.writeUnlock(); @@ -2686,6 +2688,65 @@ public void cancelIndexJob(CancelBuildIndexCommand command) throws DdlException } } + private void cancelIndexJob(CancelAlterTableStmt cancelAlterTableStmt) throws DdlException { + String dbName = cancelAlterTableStmt.getDbName(); + String tableName = cancelAlterTableStmt.getTableName(); + Preconditions.checkState(!Strings.isNullOrEmpty(dbName)); + Preconditions.checkState(!Strings.isNullOrEmpty(tableName)); + + Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName); + + List jobList = new ArrayList<>(); + + Table olapTable = db.getTableOrDdlException(tableName, Table.TableType.OLAP); + olapTable.writeLock(); + try { + // find from index change jobs first + if (cancelAlterTableStmt.getAlterJobIdList() != null + && cancelAlterTableStmt.getAlterJobIdList().size() > 0) { + for (Long jobId : cancelAlterTableStmt.getAlterJobIdList()) { + IndexChangeJob job = indexChangeJobs.get(jobId); + if (job == null) { + continue; + } + jobList.add(job); + if (LOG.isDebugEnabled()) { + LOG.debug("add build index job {} on table {} for specific id", jobId, tableName); + } + } + } else { + for (IndexChangeJob job : indexChangeJobs.values()) { + if (!job.isDone() && job.getTableId() == olapTable.getId()) { + jobList.add(job); + if (LOG.isDebugEnabled()) { + LOG.debug("add build index job {} on table {} for all", job.getJobId(), tableName); + } + } + } + } + } finally { + olapTable.writeUnlock(); + } + + // alter job v2's cancel must be called outside the table lock + if (jobList.size() > 0) { + for (IndexChangeJob job : jobList) { + long jobId = job.getJobId(); + if (LOG.isDebugEnabled()) { + LOG.debug("cancel build index job {} on table {}", jobId, tableName); + } + if (!job.cancel("user cancelled")) { + LOG.warn("cancel build index job {} on table {} failed", jobId, tableName); + throw new DdlException("Job can not be cancelled. State: " + job.getJobState()); + } else { + LOG.info("cancel build index job {} on table {} success", jobId, tableName); + } + } + } else { + throw new DdlException("No job to cancel for Table[" + tableName + "]"); + } + } + /** * Returns true if the index already exists, there is no need to create the job to add the index. * Otherwise, return false, there is need to create a job to add the index. diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java index ba72155f7ec5fe..929835d9433433 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java @@ -116,10 +116,10 @@ public void analyze(Analyzer analyzer) throws AnalysisException { } IndexDef.IndexType indexType = existedIdx.getIndexType(); - if (!existedIdx.isLightIndexChangeSupported()) { - throw new AnalysisException(indexType.toString() + " index is not needed to build."); + if (indexType == IndexDef.IndexType.NGRAM_BF + || indexType == IndexDef.IndexType.BLOOMFILTER) { + throw new AnalysisException("ngram bloomfilter or bloomfilter index is not needed to build."); } - indexDef = new IndexDef(indexName, partitionNames, indexType, true); if (!table.isPartitionedTable()) { List specifiedPartitions = indexDef.getPartitionNames(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index fa4554951d3f88..0db441772d9168 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -170,12 +170,19 @@ public String getInvertedIndexParserStopwords() { } // Whether the index can be changed in light mode - // cloud mode only supports light change for ngram_bf index + // cloud mode supports light change for ngram_bf index and non-tokenized inverted index (parser="none") // local mode supports light change for both inverted index and ngram_bf index // the rest of the index types do not support light change public boolean isLightIndexChangeSupported() { if (Config.isCloudMode()) { - return indexType == IndexDef.IndexType.NGRAM_BF; + if (indexType == IndexDef.IndexType.NGRAM_BF) { + return true; + } else if (indexType == IndexDef.IndexType.INVERTED) { + // Only support non-tokenized inverted index (parser="none") in cloud mode + String parser = getInvertedIndexParser(); + return "none".equals(parser); + } + return false; } else { return indexType == IndexDef.IndexType.INVERTED || indexType == IndexDef.IndexType.NGRAM_BF; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java index 36439874760144..1038f23a3434d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java @@ -114,8 +114,9 @@ public void validate(ConnectContext ctx) throws UserException { } IndexDef.IndexType indexType = existedIdx.getIndexType(); - if (!existedIdx.isLightIndexChangeSupported()) { - throw new AnalysisException(indexType.toString() + " index is not needed to build."); + if (indexType == IndexDef.IndexType.NGRAM_BF + || indexType == IndexDef.IndexType.BLOOMFILTER) { + throw new AnalysisException(indexType + " index is not needed to build."); } indexDef = new IndexDefinition(indexName, partitionNamesInfo, indexType); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index dccfbd3f17f100..80f70920c4e11d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -746,6 +746,7 @@ public class SessionVariable implements Serializable, Writable { public static final String SQL_CONVERTOR_CONFIG = "sql_convertor_config"; public static final String PREFER_UDF_OVER_BUILTIN = "prefer_udf_over_builtin"; + public static final String ENABLE_LIGHT_ADD_INDEX = "enable_light_add_index"; /** * If set false, user couldn't submit analyze SQL and FE won't allocate any related resources. @@ -2663,6 +2664,15 @@ public boolean isEnableESParallelScroll() { return enableESParallelScroll; } + @VariableMgr.VarAttr(name = ENABLE_LIGHT_ADD_INDEX, fuzzy = true, + description = { + "是否启用ngram index的轻量级变更模式，开启时只变更元数据，关闭时进行数据转换重写", + "Whether to enable lightweight index change mode for ngram index, " + + "when enabled only metadata is changed, " + + "when disabled data conversion rewrite is performed" + }) + public boolean enableLightAddIndex = true; + // If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables, // not the default value set in the code. @SuppressWarnings("checkstyle:Indentation") @@ -4925,4 +4935,11 @@ public boolean showSplitProfileInfo() { return enableProfile() && getProfileLevel() > 1; } + public boolean isEnableLightAddIndex() { + return enableLightAddIndex; + } + + public void setEnableLightAddIndex(boolean enableLightAddIndex) { + this.enableLightAddIndex = enableLightAddIndex; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java index 697b101884824f..7468fc4cb43337 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java @@ -44,6 +44,7 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.UserException; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.thrift.TStatusCode; @@ -86,6 +87,7 @@ public class IndexChangeJobTest { private static DropIndexClause dropIndexClause; private static TableName tableName; private static String indexName; + private static ConnectContext ctx; @Rule public ExpectedException expectedEx = ExpectedException.none(); @@ -116,6 +118,15 @@ public Env getCurrentEnv() { } }; + // Initialize ConnectContext + ctx = new ConnectContext(); + new MockUp() { + @Mock + public ConnectContext get() { + return ctx; + } + }; + // set mow table property Map properties = Maps.newHashMap(); properties.put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE, "false"); @@ -587,25 +598,45 @@ public void testNgramBfBuildIndex() throws UserException { SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); ArrayList alterClauses = new ArrayList<>(); alterClauses.add(createIndexClause); + + // Test with enable_light_index_change = true (default) + ConnectContext context = ConnectContext.get(); + context.getSessionVariable().setEnableLightAddIndex(true); schemaChangeHandler.process(alterClauses, db, table); Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); Assert.assertEquals(1, indexChangeJobMap.size()); Assert.assertEquals(1, table.getIndexes().size()); Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); - long jobId = indexChangeJobMap.values().stream().findAny().get().jobId; + SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); - buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); - buildIndexClause.analyze(analyzer); - alterClauses.clear(); - alterClauses.add(buildIndexClause); + // Clean up for next test + table.setIndexes(Lists.newArrayList()); + indexChangeJobMap.clear(); + AgentTaskQueue.clearAllTasks(); - schemaChangeHandler.process(alterClauses, db, table); - Assert.assertEquals(2, indexChangeJobMap.size()); + // Test with enable_light_index_change = false + context.getSessionVariable().setEnableLightAddIndex(false); + String indexName2 = "ngram_bf_index2"; + IndexDef indexDef2 = new IndexDef(indexName2, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + Maps.newHashMap(), "ngram bf index2"); + + createIndexClause = new CreateIndexClause(tableName, indexDef2, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses2 = new ArrayList<>(); + alterClauses2.add(createIndexClause); + schemaChangeHandler.process(alterClauses2, db, table); + indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); - SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() - .filter(job -> job.jobId != jobId) + jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() .findFirst() .orElse(null); Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); @@ -630,6 +661,8 @@ public void testNgramBfBuildIndex() throws UserException { schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); } @Test @@ -651,25 +684,15 @@ public void testCancelNgramBfBuildIndex() throws UserException { SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); ArrayList alterClauses = new ArrayList<>(); alterClauses.add(createIndexClause); + + //cancel test can only with enable_light_index_change = false + ctx.getSessionVariable().setEnableLightAddIndex(false); schemaChangeHandler.process(alterClauses, db, table); Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); Assert.assertEquals(1, indexChangeJobMap.size()); - Assert.assertEquals(1, table.getIndexes().size()); - Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); - - long jobId = indexChangeJobMap.values().stream().findAny().get().jobId; - - buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); - buildIndexClause.analyze(analyzer); - alterClauses.clear(); - alterClauses.add(buildIndexClause); - - schemaChangeHandler.process(alterClauses, db, table); - Assert.assertEquals(2, indexChangeJobMap.size()); Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() - .filter(job -> job.jobId != jobId) .findFirst() .orElse(null); Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); @@ -685,5 +708,12 @@ public void testCancelNgramBfBuildIndex() throws UserException { schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + cancelAlterTableStmt = new CancelAlterTableStmt(ShowAlterStmt.AlterType.COLUMN, tableName); + cancelAlterTableStmt.analyze(analyzer); + schemaChangeHandler.cancel(cancelAlterTableStmt); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.CANCELLED, jobV2.getJobState()); } } diff --git a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy index 6fea7a68f92b8e..540ccb50983db7 100644 --- a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy +++ b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy @@ -54,13 +54,18 @@ suite("test_ngram_bloomfilter_index_change") { sql "sync" } - // Test setup - // 1. Create table - // 2. Insert test data - // 3. Add NGRAM Bloom Filter index - // 4. Build index - // 5. Insert more data - // 6. Drop index + // Test settings + sql "set enable_function_pushdown=true" + sql "set enable_profile=true" + sql "set profile_level=2" + + // Define test query + def query = "SELECT /*+SET_VAR(enable_function_pushdown = true, enable_profile = true, profile_level = 2)*/ * FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id" + // Test Case 1: Test with enable_light_add_index = true (default, lightweight mode) + logger.info("=== Test Case 1: enable_light_add_index = true ===") + // Set enable_light_add_index = true + sql "set enable_light_add_index = true" + // Create table sql "DROP TABLE IF EXISTS ${tableName}" sql """ CREATE TABLE ${tableName} ( @@ -86,25 +91,16 @@ suite("test_ngram_bloomfilter_index_change") { ); """ - // Insert first batch of data + // Insert test data insertTestData() - - // Test settings - sql "set enable_function_pushdown=true" - sql "set enable_profile=true" - sql "set profile_level=2" - // Verify data loaded correctly - qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + qt_select_light_mode_init "SELECT * FROM ${tableName} ORDER BY sale_id" - // Define test query - def query = "SELECT /*+SET_VAR(enable_function_pushdown = true, enable_profile = true, profile_level = 2)*/ * FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id" - - // Test 1: without NGRAM Bloom Filter index - profile("sql_select_like_without_ngram_index") { + // Test without NGRAM Bloom Filter index + profile("sql_select_like_without_ngram_index_light_mode") { run { - sql "/* sql_select_like_without_ngram_index */ ${query}" - sleep(1000) // sleep 1s wait for the profile collection to be completed + sql "/* sql_select_like_without_ngram_index_light_mode */ ${query}" + sleep(1000) } check { profileString, exception -> @@ -113,27 +109,17 @@ suite("test_ngram_bloomfilter_index_change") { } } - // Test 2: After adding NGRAM Bloom Filter index + // Add NGRAM Bloom Filter index (should be immediate in light mode) sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" - wait_for_latest_op_on_table_finish(tableName, timeout) - profile("sql_select_like_with_ngram_index_added") { - run { - sql "/* sql_select_like_with_ngram_index_added */ ${query}" - sleep(1000) - } - check { profileString, exception -> - log.info(profileString) - assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) - } - } + // In light mode, the index should be effective immediately, no need to wait for alter job + // But let's give it a moment to ensure metadata is updated + sleep(2000) - // Test 3: After building the index - sql "BUILD INDEX idx_ngram_customer_name ON ${tableName};" - wait_for_latest_op_on_table_finish(tableName, timeout) - profile("sql_select_like_with_ngram_index_built") { + // Test after adding NGRAM Bloom Filter index (should filter data immediately in light mode) + profile("sql_select_like_with_ngram_index_light_mode_added") { run { - sql "/* sql_select_like_with_ngram_index_built */ ${query}" + sql "/* sql_select_like_with_ngram_index_light_mode_added */ ${query}" sleep(1000) } @@ -142,16 +128,15 @@ suite("test_ngram_bloomfilter_index_change") { assertTrue(profileString.contains("RowsBloomFilterFiltered: 10")) } } - - // Insert second batch of data + // Insert more data after index added insertTestData() - // Verify data loaded correctly - qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + // Verify more data loaded correctly + qt_select_light_mode_more_data "SELECT * FROM ${tableName} ORDER BY sale_id" - // Test 4: Verify filtering with more data - profile("sql_select_like_with_ngram_index_more_data") { + // Test with more data (should still filter correctly) + profile("sql_select_like_with_ngram_index_light_mode_more_data") { run { - sql "/* sql_select_like_with_ngram_index_more_data */ ${query}" + sql "/* sql_select_like_with_ngram_index_light_mode_more_data */ ${query}" sleep(1000) } @@ -161,12 +146,14 @@ suite("test_ngram_bloomfilter_index_change") { } } - // Test 5: After dropping the index + // Drop index sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" - wait_for_latest_op_on_table_finish(tableName, timeout) - profile("sql_select_like_with_ngram_index_dropped") { + sleep(2000) + + // Test after dropping index + profile("sql_select_like_with_ngram_index_light_mode_dropped") { run { - sql "/* sql_select_like_with_ngram_index_dropped */ ${query}" + sql "/* sql_select_like_with_ngram_index_light_mode_dropped */ ${query}" sleep(1000) } @@ -176,13 +163,11 @@ suite("test_ngram_bloomfilter_index_change") { } } - // recreate table - // 1. Create table - // 2. Add NGRAM Bloom Filter index - // 3. Insert data - // 4. Insert more data - // 5. Build index - // 6. Drop index + // Test Case 2: Test with enable_light_add_index = false (schema change mode) + logger.info("=== Test Case 2: enable_light_add_index = false ===") + // Set enable_light_add_index = false + sql "set enable_light_add_index = false" + // Create new table sql "DROP TABLE IF EXISTS ${tableName}" sql """ CREATE TABLE ${tableName} ( @@ -207,21 +192,32 @@ suite("test_ngram_bloomfilter_index_change") { "disable_auto_compaction" = "false" ); """ + // Insert test data + insertTestData() + // Verify data loaded correctly + qt_select_schema_change_mode_init "SELECT * FROM ${tableName} ORDER BY sale_id" - // add ngram bf index - sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" - wait_for_latest_op_on_table_finish(tableName, timeout) + // Test without NGRAM Bloom Filter index + profile("sql_select_like_without_ngram_index_schema_change_mode") { + run { + sql "/* sql_select_like_without_ngram_index_schema_change_mode */ ${query}" + sleep(1000) + } - // insert data - insertTestData() + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } - // Verify data loaded correctly - qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + // Add NGRAM Bloom Filter index (will trigger schema change in this mode) + sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" + wait_for_latest_op_on_table_finish(tableName, timeout) - // Test 6: Verify filtering with index added - profile("sql_select_like_with_ngram_index_recreated") { + // Test after adding NGRAM Bloom Filter index (should filter existing data) + profile("sql_select_like_with_ngram_index_schema_change_mode_added") { run { - sql "/* sql_select_like_with_ngram_index_recreated */ ${query}" + sql "/* sql_select_like_with_ngram_index_schema_change_mode_added */ ${query}" sleep(1000) } @@ -231,16 +227,15 @@ suite("test_ngram_bloomfilter_index_change") { } } - // insert more data + // Insert more data after index is built insertTestData() - - // Verify data loaded correctly - qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + // Verify more data loaded correctly + qt_select_schema_change_mode_more_data "SELECT * FROM ${tableName} ORDER BY sale_id" - // Test 7: Verify filtering with more data - profile("sql_select_like_with_ngram_index_recreated_more_data") { + // Test with more data (should filter all data) + profile("sql_select_like_with_ngram_index_schema_change_mode_more_data") { run { - sql "/* sql_select_like_with_ngram_index_recreated_more_data */ ${query}" + sql "/* sql_select_like_with_ngram_index_schema_change_mode_more_data */ ${query}" sleep(1000) } @@ -250,37 +245,94 @@ suite("test_ngram_bloomfilter_index_change") { } } - // build index - sql "BUILD INDEX idx_ngram_customer_name ON ${tableName};" + // Drop index + sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" wait_for_latest_op_on_table_finish(tableName, timeout) - // Test 8: Verify filtering with index built - profile("sql_select_like_with_ngram_index_recreated_built") { + // Test after dropping index + profile("sql_select_like_with_ngram_index_schema_change_mode_dropped") { run { - sql "/* sql_select_like_with_ngram_index_recreated_built */ ${query}" + sql "/* sql_select_like_with_ngram_index_schema_change_mode_dropped */ ${query}" sleep(1000) } check { profileString, exception -> log.info(profileString) - assertTrue(profileString.contains("RowsBloomFilterFiltered: 20")) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) } } - // drop index - sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" - wait_for_latest_op_on_table_finish(tableName, timeout) + // Test Case 3: Test different scenarios for index lifecycle + logger.info("=== Test Case 3: Index lifecycle with light mode ===") + // Set back to light mode for additional tests + sql "set enable_light_add_index = true" + // Create table and add index before inserting data + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + `sale_id` int NULL, + `sale_date` datetime NULL, + `product_name` varchar(100) NULL, + `customer_name` varchar(100) NULL, + `amount` decimal(10,2) NULL, + `region` char(50) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`sale_id`) + PARTITION BY RANGE(`sale_date`) ( + PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01 00:00:00')), + PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01 00:00:00')), + PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00')) + ) + DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "false" + ); + """ + + // Add ngram bf index before data insertion + sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" + sleep(2000) + + // Insert data after index creation + insertTestData() + // Verify data loaded correctly + qt_select_lifecycle_after_data "SELECT * FROM ${tableName} ORDER BY sale_id" - // Test 9: Verify filtering with index dropped - profile("sql_select_like_with_ngram_index_recreated_dropped") { + // Test filtering with index added before data insertion + profile("sql_select_like_with_ngram_index_lifecycle_test") { run { - sql "/* sql_select_like_with_ngram_index_recreated_dropped */ ${query}" + sql "/* sql_select_like_with_ngram_index_lifecycle_test */ ${query}" sleep(1000) } check { profileString, exception -> log.info(profileString) - assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 10")) + } + } + + // Insert more data + insertTestData() + // Verify more data loaded correctly + qt_select_lifecycle_final "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test filtering with more data + profile("sql_select_like_with_ngram_index_lifecycle_final") { + run { + sql "/* sql_select_like_with_ngram_index_lifecycle_final */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 20")) } } + + // Final cleanup + sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" + sleep(2000) } \ No newline at end of file From b9ec45329bea454f774187fee7a75fc7707be569 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Wed, 25 Jun 2025 09:45:57 +0800 Subject: [PATCH 02/14] fix style --- .../src/main/java/org/apache/doris/qe/SessionVariable.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 80f70920c4e11d..584c76cd95d511 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -2664,12 +2664,10 @@ public boolean isEnableESParallelScroll() { return enableESParallelScroll; } - @VariableMgr.VarAttr(name = ENABLE_LIGHT_ADD_INDEX, fuzzy = true, - description = { + @VariableMgr.VarAttr(name = ENABLE_LIGHT_ADD_INDEX, fuzzy = true, description = { "是否启用ngram index的轻量级变更模式，开启时只变更元数据，关闭时进行数据转换重写", "Whether to enable lightweight index change mode for ngram index, " - + "when enabled only metadata is changed, " - + "when disabled data conversion rewrite is performed" + + "when enabled only metadata is changed, when disabled data conversion rewrite is performed" }) public boolean enableLightAddIndex = true; From 567f6d062cbe067758656ee72dc4dee709e6e3b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Wed, 25 Jun 2025 09:51:58 +0800 Subject: [PATCH 03/14] fix style --- .../doris/alter/SchemaChangeHandler.java | 59 ------------------- 1 file changed, 59 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 2078001ee645fb..166f3ceff3786c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -2688,65 +2688,6 @@ public void cancelIndexJob(CancelBuildIndexCommand command) throws DdlException } } - private void cancelIndexJob(CancelAlterTableStmt cancelAlterTableStmt) throws DdlException { - String dbName = cancelAlterTableStmt.getDbName(); - String tableName = cancelAlterTableStmt.getTableName(); - Preconditions.checkState(!Strings.isNullOrEmpty(dbName)); - Preconditions.checkState(!Strings.isNullOrEmpty(tableName)); - - Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName); - - List jobList = new ArrayList<>(); - - Table olapTable = db.getTableOrDdlException(tableName, Table.TableType.OLAP); - olapTable.writeLock(); - try { - // find from index change jobs first - if (cancelAlterTableStmt.getAlterJobIdList() != null - && cancelAlterTableStmt.getAlterJobIdList().size() > 0) { - for (Long jobId : cancelAlterTableStmt.getAlterJobIdList()) { - IndexChangeJob job = indexChangeJobs.get(jobId); - if (job == null) { - continue; - } - jobList.add(job); - if (LOG.isDebugEnabled()) { - LOG.debug("add build index job {} on table {} for specific id", jobId, tableName); - } - } - } else { - for (IndexChangeJob job : indexChangeJobs.values()) { - if (!job.isDone() && job.getTableId() == olapTable.getId()) { - jobList.add(job); - if (LOG.isDebugEnabled()) { - LOG.debug("add build index job {} on table {} for all", job.getJobId(), tableName); - } - } - } - } - } finally { - olapTable.writeUnlock(); - } - - // alter job v2's cancel must be called outside the table lock - if (jobList.size() > 0) { - for (IndexChangeJob job : jobList) { - long jobId = job.getJobId(); - if (LOG.isDebugEnabled()) { - LOG.debug("cancel build index job {} on table {}", jobId, tableName); - } - if (!job.cancel("user cancelled")) { - LOG.warn("cancel build index job {} on table {} failed", jobId, tableName); - throw new DdlException("Job can not be cancelled. State: " + job.getJobState()); - } else { - LOG.info("cancel build index job {} on table {} success", jobId, tableName); - } - } - } else { - throw new DdlException("No job to cancel for Table[" + tableName + "]"); - } - } - /** * Returns true if the index already exists, there is no need to create the job to add the index. * Otherwise, return false, there is need to create a job to add the index. From cbe486bc394809d51b8102c35c2f9c8ce67e8b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Wed, 25 Jun 2025 10:48:48 +0800 Subject: [PATCH 04/14] fix rebase --- .../org/apache/doris/alter/IndexChangeJobTest.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java index 7468fc4cb43337..caa3911f10ab8b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java @@ -44,6 +44,8 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.UserException; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.nereids.trees.plans.commands.CancelAlterTableCommand; +import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; import org.apache.doris.qe.ConnectContext; import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskQueue; @@ -709,9 +711,12 @@ public void testCancelNgramBfBuildIndex() throws UserException { Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); - cancelAlterTableStmt = new CancelAlterTableStmt(ShowAlterStmt.AlterType.COLUMN, tableName); - cancelAlterTableStmt.analyze(analyzer); - schemaChangeHandler.cancel(cancelAlterTableStmt); + TableNameInfo tableNameInfo = new TableNameInfo(db.getName(), table.getName()); + CancelAlterTableCommand cancelAlterTableCommand = new CancelAlterTableCommand( + tableNameInfo, + CancelAlterTableCommand.AlterType.COLUMN, + Lists.newArrayList()); + schemaChangeHandler.cancel(cancelAlterTableCommand); schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(AlterJobV2.JobState.CANCELLED, jobV2.getJobState()); From 300d2b5311beb8306affd03535ce7288d04067d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Wed, 25 Jun 2025 11:10:54 +0800 Subject: [PATCH 05/14] fix ut --- .../test/java/org/apache/doris/alter/IndexChangeJobTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java index caa3911f10ab8b..b189dedd01556c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java @@ -664,7 +664,7 @@ public void testNgramBfBuildIndex() throws UserException { schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); Assert.assertEquals(1, table.getIndexes().size()); - Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); + Assert.assertEquals("ngram_bf_index2", table.getIndexes().get(0).getIndexName()); } @Test From 1ab4256ac1a9731b25fd4fa456355710367ddca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Wed, 25 Jun 2025 14:48:51 +0800 Subject: [PATCH 06/14] fix ut --- .../apache/doris/alter/SchemaChangeHandlerTest.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java index 4b8f218acea094..f440136e5cb8d5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java @@ -849,14 +849,8 @@ public void testDupAddOrDropNgramBfIndex() throws Exception { waitAlterJobDone(alterJobs); String buildNgramBfIndexStmtStr = "BUILD INDEX idx_error_msg on test.sc_dup "; - AlterTableStmt buildNgramBfIndexStmt = (AlterTableStmt) parseAndAnalyzeStmt(buildNgramBfIndexStmtStr); - Env.getCurrentEnv().getAlterInstance().processAlterTable(buildNgramBfIndexStmt); - - jobSize++; - alterJobs = Env.getCurrentEnv().getSchemaChangeHandler().getAlterJobsV2(); - LOG.info("alterJobs:{}", alterJobs); - Assertions.assertEquals(jobSize, alterJobs.size()); - waitAlterJobDone(alterJobs); + Assertions.assertThrows(org.apache.doris.common.AnalysisException.class, + () -> parseAndAnalyzeStmt(buildNgramBfIndexStmtStr)); tbl.readLock(); try { From 89bd9b4e25df3474f6deffff8d98cf85cf8bb22b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Wed, 25 Jun 2025 18:27:58 +0800 Subject: [PATCH 07/14] fix regression --- .../test_ngram_bloomfilter_index_change.out | 42 +++++++++++++++++-- ...test_ngram_bloomfilter_index_change.groovy | 14 +------ 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out index 6f916a99c91705..49570b96d291f2 100644 --- a/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out +++ b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out @@ -1,5 +1,5 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !select -- +-- !select_light_mode_init -- 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South 1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East @@ -11,7 +11,7 @@ 1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North 1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South --- !select -- +-- !select_light_mode_more_data -- 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South @@ -33,7 +33,7 @@ 1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South 1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South --- !select -- +-- !select_schema_change_mode_init -- 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South 1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East @@ -45,7 +45,41 @@ 1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North 1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South --- !select -- +-- !select_schema_change_mode_more_data -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select_lifecycle_after_data -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select_lifecycle_final -- 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South diff --git a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy index 540ccb50983db7..ec48c477f4a080 100644 --- a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy +++ b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy @@ -116,18 +116,6 @@ suite("test_ngram_bloomfilter_index_change") { // But let's give it a moment to ensure metadata is updated sleep(2000) - // Test after adding NGRAM Bloom Filter index (should filter data immediately in light mode) - profile("sql_select_like_with_ngram_index_light_mode_added") { - run { - sql "/* sql_select_like_with_ngram_index_light_mode_added */ ${query}" - sleep(1000) - } - - check { profileString, exception -> - log.info(profileString) - assertTrue(profileString.contains("RowsBloomFilterFiltered: 10")) - } - } // Insert more data after index added insertTestData() // Verify more data loaded correctly @@ -142,7 +130,7 @@ suite("test_ngram_bloomfilter_index_change") { check { profileString, exception -> log.info(profileString) - assertTrue(profileString.contains("RowsBloomFilterFiltered: 20")) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 10")) } } From 961a03b81f89045bb7aba66e887e68962b150fe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Wed, 25 Jun 2025 23:04:29 +0800 Subject: [PATCH 08/14] update case --- .../index_p0/test_ngram_bloomfilter_index_change.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy index ec48c477f4a080..1b7d81291c6ac4 100644 --- a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy +++ b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy @@ -136,7 +136,7 @@ suite("test_ngram_bloomfilter_index_change") { // Drop index sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" - sleep(2000) + wait_for_latest_op_on_table_finish(tableName, timeout) // Test after dropping index profile("sql_select_like_with_ngram_index_light_mode_dropped") { @@ -282,7 +282,7 @@ suite("test_ngram_bloomfilter_index_change") { // Add ngram bf index before data insertion sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" - sleep(2000) + wait_for_latest_op_on_table_finish(tableName, timeout) // Insert data after index creation insertTestData() From 2fcbf0deaf413438483e2de0cff6c593cecc9e1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Wed, 25 Jun 2025 23:12:26 +0800 Subject: [PATCH 09/14] update case --- .../index_format_v2/test_add_build_index_with_format_v2.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy index dabb3534e32e20..e6207aabb565db 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy @@ -64,6 +64,7 @@ suite("test_add_build_index_with_format_v2", "inverted_index_format_v2"){ assertTrue(useTime <= OpTimeout, "wait_for_latest_build_index_on_partition_finish timeout") } + sql "set enable_light_add_index = false" sql "DROP TABLE IF EXISTS ${tableName}" sql """ From 3c6471bc71d1e47d43cfb42793142fbe7881f3ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Thu, 26 Jun 2025 00:49:00 +0800 Subject: [PATCH 10/14] update case --- .../test_add_build_index_with_format_v2.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy index e6207aabb565db..f79d5b5efdea9e 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy @@ -64,7 +64,10 @@ suite("test_add_build_index_with_format_v2", "inverted_index_format_v2"){ assertTrue(useTime <= OpTimeout, "wait_for_latest_build_index_on_partition_finish timeout") } - sql "set enable_light_add_index = false" + if (isCloudMode()) { + sql "set enable_light_add_index = false" + } + sql "DROP TABLE IF EXISTS ${tableName}" sql """ From 1faccf125d105b0db10f86cc1b8391099d1e1459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Thu, 26 Jun 2025 10:54:47 +0800 Subject: [PATCH 11/14] update --- .../doris/alter/SchemaChangeHandler.java | 10 ++--- .../java/org/apache/doris/catalog/Index.java | 38 +++++++++++-------- .../org/apache/doris/qe/SessionVariable.java | 20 +++++----- .../doris/alter/IndexChangeJobTest.java | 12 +++--- ...test_ngram_bloomfilter_index_change.groovy | 22 +++++------ ...test_add_build_index_with_format_v2.groovy | 4 -- 6 files changed, 55 insertions(+), 51 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 166f3ceff3786c..9de9e3c3a72b59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -2088,20 +2088,20 @@ public int getAsInt() { lightSchemaChange = false; // Check if the index supports light index change and session variable is enabled - boolean enableLightAddIndex = true; + boolean enableAddIndexForNewData = true; try { ConnectContext context = ConnectContext.get(); if (context != null && context.getSessionVariable() != null) { - enableLightAddIndex = context.getSessionVariable().isEnableLightAddIndex(); + enableAddIndexForNewData = context.getSessionVariable().isEnableAddIndexForNewData(); } } catch (Exception e) { - LOG.warn("Failed to get session variable enable_light_index_change, " - + "using default value: true", e); + LOG.warn("Failed to get session variable enable_add_index_for_new_data, " + + "using default value: false", e); } // ngram_bf index can do light_schema_change in both local and cloud mode // inverted index can only do light_schema_change in local mode - if (index.isLightIndexChangeSupported() && enableLightAddIndex) { + if (index.isLightAddIndexSupported(enableAddIndexForNewData)) { alterIndexes.add(index); isDropIndex = false; lightIndexChange = true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index 0db441772d9168..0793910da0b3c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -153,6 +153,10 @@ public String getInvertedIndexParser() { return InvertedIndexUtil.getInvertedIndexParser(properties); } + public boolean isInvertedIndexParserNone() { + return InvertedIndexUtil.INVERTED_INDEX_PARSER_NONE.equals(getInvertedIndexParser()); + } + public String getInvertedIndexParserMode() { return InvertedIndexUtil.getInvertedIndexParserMode(properties); } @@ -170,23 +174,27 @@ public String getInvertedIndexParserStopwords() { } // Whether the index can be changed in light mode - // cloud mode supports light change for ngram_bf index and non-tokenized inverted index (parser="none") - // local mode supports light change for both inverted index and ngram_bf index - // the rest of the index types do not support light change public boolean isLightIndexChangeSupported() { - if (Config.isCloudMode()) { - if (indexType == IndexDef.IndexType.NGRAM_BF) { - return true; - } else if (indexType == IndexDef.IndexType.INVERTED) { - // Only support non-tokenized inverted index (parser="none") in cloud mode - String parser = getInvertedIndexParser(); - return "none".equals(parser); - } - return false; - } else { - return indexType == IndexDef.IndexType.INVERTED - || indexType == IndexDef.IndexType.NGRAM_BF; + return indexType == IndexDef.IndexType.INVERTED; + } + + // Whether the index can be added in light mode + // cloud mode supports light add for ngram_bf index and non-tokenized inverted index (parser="none") + // local mode supports light add for both inverted index and ngram_bf index + // the rest of the index types do not support light add + public boolean isLightAddIndexSupported(boolean enableAddIndexForNewData) { + // Determine if the index supports light change based on the index type and configuration mode. + if (indexType == IndexDef.IndexType.NGRAM_BF) { + // For NGRAM_BF type, support is based solely on the enableAddIndexForNewData flag. + return enableAddIndexForNewData; + } else if (indexType == IndexDef.IndexType.INVERTED) { + // For INVERTED type: + // - In cloud mode (with enableAddIndexForNewData enabled), only support non-tokenized indexes (parser="none"). + // - In local mode or when add-index for new data is disabled, light add is always supported. + return (Config.isCloudMode() && enableAddIndexForNewData) ? isInvertedIndexParserNone() : true; } + // Other index types do not support light add. + return false; } public String getInvertedIndexCustomAnalyzer() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 584c76cd95d511..ffaecc2df3e9a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -746,7 +746,7 @@ public class SessionVariable implements Serializable, Writable { public static final String SQL_CONVERTOR_CONFIG = "sql_convertor_config"; public static final String PREFER_UDF_OVER_BUILTIN = "prefer_udf_over_builtin"; - public static final String ENABLE_LIGHT_ADD_INDEX = "enable_light_add_index"; + public static final String ENABLE_ADD_INDEX_FOR_NEW_DATA = "enable_add_index_for_new_data"; /** * If set false, user couldn't submit analyze SQL and FE won't allocate any related resources. @@ -2664,12 +2664,12 @@ public boolean isEnableESParallelScroll() { return enableESParallelScroll; } - @VariableMgr.VarAttr(name = ENABLE_LIGHT_ADD_INDEX, fuzzy = true, description = { - "是否启用ngram index的轻量级变更模式，开启时只变更元数据，关闭时进行数据转换重写", - "Whether to enable lightweight index change mode for ngram index, " - + "when enabled only metadata is changed, when disabled data conversion rewrite is performed" + @VariableMgr.VarAttr(name = ENABLE_ADD_INDEX_FOR_NEW_DATA, fuzzy = true, description = { + "是否启用仅对新数据生效的索引添加模式，开启时新建索引只对后续写入的数据生效，关闭时对全部数据重建索引", + "Whether to enable add index mode that only affects new data, " + + "when enabled new indexes only affect subsequently written data, when disabled rebuild indexes for all data" }) - public boolean enableLightAddIndex = true; + public boolean enableAddIndexForNewData = false; // If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables, // not the default value set in the code. @@ -4933,11 +4933,11 @@ public boolean showSplitProfileInfo() { return enableProfile() && getProfileLevel() > 1; } - public boolean isEnableLightAddIndex() { - return enableLightAddIndex; + public boolean isEnableAddIndexForNewData() { + return enableAddIndexForNewData; } - public void setEnableLightAddIndex(boolean enableLightAddIndex) { - this.enableLightAddIndex = enableLightAddIndex; + public void setEnableAddIndexForNewData(boolean enableAddIndexForNewData) { + this.enableAddIndexForNewData = enableAddIndexForNewData; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java index b189dedd01556c..b1eda304f227a8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java @@ -601,9 +601,9 @@ public void testNgramBfBuildIndex() throws UserException { ArrayList alterClauses = new ArrayList<>(); alterClauses.add(createIndexClause); - // Test with enable_light_index_change = true (default) + // Test with enable_add_index_for_new_data = true ConnectContext context = ConnectContext.get(); - context.getSessionVariable().setEnableLightAddIndex(true); + context.getSessionVariable().setEnableAddIndexForNewData(true); schemaChangeHandler.process(alterClauses, db, table); Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); Assert.assertEquals(1, indexChangeJobMap.size()); @@ -621,8 +621,8 @@ public void testNgramBfBuildIndex() throws UserException { indexChangeJobMap.clear(); AgentTaskQueue.clearAllTasks(); - // Test with enable_light_index_change = false - context.getSessionVariable().setEnableLightAddIndex(false); + // Test with enable_add_index_for_new_data = false + context.getSessionVariable().setEnableAddIndexForNewData(false); String indexName2 = "ngram_bf_index2"; IndexDef indexDef2 = new IndexDef(indexName2, false, Lists.newArrayList(table.getBaseSchema().get(3).getName()), @@ -687,8 +687,8 @@ public void testCancelNgramBfBuildIndex() throws UserException { ArrayList alterClauses = new ArrayList<>(); alterClauses.add(createIndexClause); - //cancel test can only with enable_light_index_change = false - ctx.getSessionVariable().setEnableLightAddIndex(false); + //cancel test can only with enable_add_index_for_new_data = false + ctx.getSessionVariable().setEnableAddIndexForNewData(false); schemaChangeHandler.process(alterClauses, db, table); Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); Assert.assertEquals(1, indexChangeJobMap.size()); diff --git a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy index 1b7d81291c6ac4..5f64c0ca3290a6 100644 --- a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy +++ b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy @@ -61,10 +61,10 @@ suite("test_ngram_bloomfilter_index_change") { // Define test query def query = "SELECT /*+SET_VAR(enable_function_pushdown = true, enable_profile = true, profile_level = 2)*/ * FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id" - // Test Case 1: Test with enable_light_add_index = true (default, lightweight mode) - logger.info("=== Test Case 1: enable_light_add_index = true ===") - // Set enable_light_add_index = true - sql "set enable_light_add_index = true" + // Test Case 1: Test with enable_add_index_for_new_data = true + logger.info("=== Test Case 1: enable_add_index_for_new_data = true ===") + // Set enable_add_index_for_new_data = true + sql "set enable_add_index_for_new_data = true" // Create table sql "DROP TABLE IF EXISTS ${tableName}" sql """ @@ -151,10 +151,10 @@ suite("test_ngram_bloomfilter_index_change") { } } - // Test Case 2: Test with enable_light_add_index = false (schema change mode) - logger.info("=== Test Case 2: enable_light_add_index = false ===") - // Set enable_light_add_index = false - sql "set enable_light_add_index = false" + // Test Case 2: Test with enable_add_index_for_new_data = false (schema change mode) + logger.info("=== Test Case 2: enable_add_index_for_new_data = false ===") + // Set enable_add_index_for_new_data = false + sql "set enable_add_index_for_new_data = false" // Create new table sql "DROP TABLE IF EXISTS ${tableName}" sql """ @@ -251,9 +251,9 @@ suite("test_ngram_bloomfilter_index_change") { } // Test Case 3: Test different scenarios for index lifecycle - logger.info("=== Test Case 3: Index lifecycle with light mode ===") - // Set back to light mode for additional tests - sql "set enable_light_add_index = true" + logger.info("=== Test Case 3: Index lifecycle with enable_add_index_for_new_data = true ===") + // Set enable_add_index_for_new_data = true + sql "set enable_add_index_for_new_data = true" // Create table and add index before inserting data sql "DROP TABLE IF EXISTS ${tableName}" sql """ diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy index f79d5b5efdea9e..dabb3534e32e20 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy @@ -64,10 +64,6 @@ suite("test_add_build_index_with_format_v2", "inverted_index_format_v2"){ assertTrue(useTime <= OpTimeout, "wait_for_latest_build_index_on_partition_finish timeout") } - if (isCloudMode()) { - sql "set enable_light_add_index = false" - } - sql "DROP TABLE IF EXISTS ${tableName}" sql """ From acc294f976022395171354bba3531b8a8d1e4a8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Thu, 26 Jun 2025 19:35:14 +0800 Subject: [PATCH 12/14] update --- fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java | 4 ++-- .../src/main/java/org/apache/doris/qe/SessionVariable.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index 0793910da0b3c3..676f6bc7644038 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -189,8 +189,8 @@ public boolean isLightAddIndexSupported(boolean enableAddIndexForNewData) { return enableAddIndexForNewData; } else if (indexType == IndexDef.IndexType.INVERTED) { // For INVERTED type: - // - In cloud mode (with enableAddIndexForNewData enabled), only support non-tokenized indexes (parser="none"). - // - In local mode or when add-index for new data is disabled, light add is always supported. + // - In cloud mode (with enableAddIndexForNewData enabled), only support non-tokenized indexes. + // - In local mode light add is always supported for inverted index. return (Config.isCloudMode() && enableAddIndexForNewData) ? isInvertedIndexParserNone() : true; } // Other index types do not support light add. diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index ffaecc2df3e9a0..be151e35c7e1ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -2667,7 +2667,8 @@ public boolean isEnableESParallelScroll() { @VariableMgr.VarAttr(name = ENABLE_ADD_INDEX_FOR_NEW_DATA, fuzzy = true, description = { "是否启用仅对新数据生效的索引添加模式，开启时新建索引只对后续写入的数据生效，关闭时对全部数据重建索引", "Whether to enable add index mode that only affects new data, " - + "when enabled new indexes only affect subsequently written data, when disabled rebuild indexes for all data" + + "when enabled new indexes only affect subsequently written data, " + + "when disabled rebuild indexes for all data" }) public boolean enableAddIndexForNewData = false; From 8d54e2c80649b4cf4bc74f91d24ddec86a9a99b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Thu, 26 Jun 2025 22:58:14 +0800 Subject: [PATCH 13/14] update --- .../java/org/apache/doris/catalog/Index.java | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index 676f6bc7644038..4916d0e48abf19 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -183,18 +183,16 @@ public boolean isLightIndexChangeSupported() { // local mode supports light add for both inverted index and ngram_bf index // the rest of the index types do not support light add public boolean isLightAddIndexSupported(boolean enableAddIndexForNewData) { - // Determine if the index supports light change based on the index type and configuration mode. - if (indexType == IndexDef.IndexType.NGRAM_BF) { - // For NGRAM_BF type, support is based solely on the enableAddIndexForNewData flag. - return enableAddIndexForNewData; - } else if (indexType == IndexDef.IndexType.INVERTED) { - // For INVERTED type: - // - In cloud mode (with enableAddIndexForNewData enabled), only support non-tokenized indexes. - // - In local mode light add is always supported for inverted index. - return (Config.isCloudMode() && enableAddIndexForNewData) ? isInvertedIndexParserNone() : true; + if (Config.isCloudMode()) { + if (indexType == IndexDef.IndexType.INVERTED) { + return isInvertedIndexParserNone() && enableAddIndexForNewData; + } else if (indexType == IndexDef.IndexType.NGRAM_BF) { + return enableAddIndexForNewData; + } + return false; } - // Other index types do not support light add. - return false; + return (indexType == IndexDef.IndexType.NGRAM_BF && enableAddIndexForNewData) + || (indexType == IndexDef.IndexType.INVERTED); } public String getInvertedIndexCustomAnalyzer() { From d911c9705910c80a11eb140c98f766b47e68925c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Sat, 28 Jun 2025 00:36:22 +0800 Subject: [PATCH 14/14] update --- .../datasource/CloudInternalCatalog.java | 8 + .../apache/doris/alter/CloudIndexTest.java | 664 ++++++++++++++++++ .../apache/doris/catalog/CatalogTestUtil.java | 42 +- .../org/apache/doris/catalog/FakeEditLog.java | 10 + ...test_ngram_bloomfilter_index_change.groovy | 9 +- 5 files changed, 716 insertions(+), 17 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index 15de6f132dada9..0353856d5e0ccc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -344,6 +344,14 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2); } else if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V3) { schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V3); + } else if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.DEFAULT) { + if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) { + schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V1); + } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V3")) { + schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V3); + } else { + schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2); + } } else { throw new DdlException("invalid inverted index storage format"); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java new file mode 100644 index 00000000000000..1836edc7bf6580 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java @@ -0,0 +1,664 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.alter; + +import org.apache.doris.analysis.AlterClause; +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.BuildIndexClause; +import org.apache.doris.analysis.CreateIndexClause; +import org.apache.doris.analysis.DataSortInfo; +import org.apache.doris.analysis.DropIndexClause; +import org.apache.doris.analysis.IndexDef; +import org.apache.doris.analysis.IndexDef.IndexType; +import org.apache.doris.analysis.ResourceTypeEnum; +import org.apache.doris.analysis.TableName; +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.CatalogTestUtil; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.EnvFactory; +import org.apache.doris.catalog.FakeEditLog; +import org.apache.doris.catalog.FakeEnv; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.OlapTable.OlapTableState; +import org.apache.doris.cloud.catalog.CloudEnv; +import org.apache.doris.cloud.catalog.CloudEnvFactory; +import org.apache.doris.cloud.datasource.CloudInternalCatalog; +import org.apache.doris.cloud.proto.Cloud; +import org.apache.doris.cloud.proto.Cloud.MetaServiceCode; +import org.apache.doris.cloud.rpc.MetaServiceProxy; +import org.apache.doris.cloud.system.CloudSystemInfoService; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.UserException; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.Auth; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.commands.CancelBuildIndexCommand; +import org.apache.doris.persist.EditLog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.resource.computegroup.ComputeGroup; +import org.apache.doris.resource.computegroup.ComputeGroupMgr; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.task.AgentTask; +import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; +import org.apache.doris.thrift.TSortType; +import org.apache.doris.thrift.TTaskType; +import org.apache.doris.utframe.MockedMetaServerFactory; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import mockit.Mock; +import mockit.MockUp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class CloudIndexTest { + private static final Logger LOG = LogManager.getLogger(CloudIndexTest.class); + + private static String fileName = "./CloudIndexTest"; + + private static FakeEditLog fakeEditLog; + private static FakeEnv fakeEnv; + private static Env masterEnv; + private static EditLog testEditLog; + private ConnectContext ctx; + + private static Analyzer analyzer; + private static Database db; + private static OlapTable olapTable; + private static CreateIndexClause createIndexClause; + private static BuildIndexClause buildIndexClause; + private static DropIndexClause dropIndexClause; + private static CancelBuildIndexCommand cancelBuildIndexCommand; + private static SchemaChangeHandler schemaChangeHandler; + + @Before + public void setUp() throws InstantiationException, IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException, SecurityException, UserException { + FeConstants.runningUnitTest = true; + // Setup for MetaServiceProxy mock + new MockUp(MetaServiceProxy.class) { + + @Mock + public Cloud.BeginTxnResponse beginTxn(Cloud.BeginTxnRequest request) { + Cloud.BeginTxnResponse.Builder beginTxnResponseBuilder = Cloud.BeginTxnResponse.newBuilder(); + beginTxnResponseBuilder.setTxnId(1000) + .setStatus( + Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK")); + return beginTxnResponseBuilder.build(); + } + + @Mock + public Cloud.CommitTxnResponse commitTxn(Cloud.CommitTxnRequest request) { + Cloud.TxnInfoPB.Builder txnInfoBuilder = Cloud.TxnInfoPB.newBuilder(); + txnInfoBuilder.setDbId(CatalogTestUtil.testDbId1); + txnInfoBuilder.addAllTableIds(Lists.newArrayList(olapTable.getId())); + txnInfoBuilder.setLabel("test_label"); + txnInfoBuilder.setListenerId(-1); + Cloud.CommitTxnResponse.Builder commitTxnResponseBuilder = Cloud.CommitTxnResponse.newBuilder(); + commitTxnResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .setTxnInfo(txnInfoBuilder.build()); + return commitTxnResponseBuilder.build(); + } + + @Mock + public Cloud.CheckTxnConflictResponse checkTxnConflict(Cloud.CheckTxnConflictRequest request) { + Cloud.CheckTxnConflictResponse.Builder checkTxnConflictResponseBuilder = + Cloud.CheckTxnConflictResponse.newBuilder(); + checkTxnConflictResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .setFinished(true); + return checkTxnConflictResponseBuilder.build(); + } + + @Mock + public Cloud.GetClusterResponse getCluster(Cloud.GetClusterRequest request) { + Cloud.GetClusterResponse.Builder getClusterResponseBuilder = Cloud.GetClusterResponse.newBuilder(); + Cloud.ClusterPB.Builder clusterBuilder = Cloud.ClusterPB.newBuilder(); + clusterBuilder.setClusterId("test_id").setClusterName("test_group"); + + Cloud.NodeInfoPB.Builder node1 = Cloud.NodeInfoPB.newBuilder(); + node1.setCloudUniqueId("test_cloud") + .setName("host1") + .setIp("host1") + .setHost("host1") + .setHeartbeatPort(123) + .setEditLogPort(125) + .setStatus(Cloud.NodeStatusPB.NODE_STATUS_RUNNING); + clusterBuilder.addNodes(node1.build()); + getClusterResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .addCluster(clusterBuilder.build()); + return getClusterResponseBuilder.build(); + } + + @Mock + public Cloud.CreateTabletsResponse createTablets(Cloud.CreateTabletsRequest request) { + Cloud.CreateTabletsResponse.Builder responseBuilder = Cloud.CreateTabletsResponse.newBuilder(); + responseBuilder.setStatus( + Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK")); + return responseBuilder.build(); + } + + @Mock + public Cloud.FinishTabletJobResponse finishTabletJob(Cloud.FinishTabletJobRequest request) { + Cloud.FinishTabletJobResponse.Builder responseBuilder = Cloud.FinishTabletJobResponse.newBuilder(); + responseBuilder.setStatus( + Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK")); + return responseBuilder.build(); + } + + @Mock + public Cloud.IndexResponse prepareIndex(Cloud.IndexRequest request) { + Cloud.IndexResponse.Builder builder = Cloud.IndexResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.IndexResponse commitIndex(Cloud.IndexRequest request) { + Cloud.IndexResponse.Builder builder = Cloud.IndexResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.IndexResponse dropIndex(Cloud.IndexRequest request) { + Cloud.IndexResponse.Builder builder = Cloud.IndexResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.CheckKVResponse checkKv(Cloud.CheckKVRequest request) { + Cloud.CheckKVResponse.Builder builder = Cloud.CheckKVResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.GetCurrentMaxTxnResponse getCurrentMaxTxnId(Cloud.GetCurrentMaxTxnRequest request) { + Cloud.GetCurrentMaxTxnResponse.Builder builder = Cloud.GetCurrentMaxTxnResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .setCurrentMaxTxnId(1000); + return builder.build(); + } + }; + + Config.cloud_unique_id = "test_cloud"; + Config.meta_service_endpoint = MockedMetaServerFactory.METASERVER_DEFAULT_IP + ":" + 20121; + + EnvFactory envFactory = EnvFactory.getInstance(); + masterEnv = envFactory.createEnv(false); + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + FakeEnv.setSystemInfo(cloudSystemInfo); + + fakeEditLog = new FakeEditLog(); + testEditLog = null; // Will be set by MockUp + FakeEnv.setEnv(masterEnv); + + ctx = new ConnectContext(); + ctx.setEnv(masterEnv); + ctx.setQualifiedUser("root"); + UserIdentity rootUser = new UserIdentity("root", "%"); + rootUser.setIsAnalyzed(); + ctx.setCurrentUserIdentity(rootUser); + ctx.setThreadLocalInfo(); + ctx.setCloudCluster("test_group"); + Assert.assertTrue(envFactory instanceof CloudEnvFactory); + Assert.assertTrue(masterEnv instanceof CloudEnv); + new MockUp() { + @Mock + public Env getCurrentEnv() { + return masterEnv; + } + + @Mock + public EditLog getEditLog() { + if (testEditLog == null) { + // Create a mock EditLog using a no-op approach + testEditLog = new EditLog("test") { + // Override to avoid initialization issues + }; + } + return testEditLog; + } + + @Mock + public ComputeGroupMgr getComputeGroupMgr() { + return new ComputeGroupMgr(Env.getCurrentSystemInfo()); + } + + @Mock + public SchemaChangeHandler getSchemaChangeHandler() { + // Create a new independent SchemaChangeHandler for each call + return schemaChangeHandler; + } + + @Mock + public AccessControllerManager getAccessManager() { + return new AccessControllerManager(masterEnv.getAuth()) { + @Override + public boolean checkTblPriv(ConnectContext ctx, String ctl, String db, String tbl, PrivPredicate wanted) { + return true; // Allow all access for test + } + + @Override + public boolean checkCloudPriv(UserIdentity user, String cluster, PrivPredicate wanted, ResourceTypeEnum resourceType) { + return true; // Allow all cloud privileges for test + } + }; + } + }; + + new MockUp() { + @Mock + public String getDefaultCloudCluster(String user) { + return "test_group"; // Return default cluster for test + } + + @Mock + public ComputeGroup getComputeGroup(String user) { + try { + return masterEnv.getComputeGroupMgr().getComputeGroupByName("test_group"); + } catch (Exception e) { + return masterEnv.getComputeGroupMgr().getAllBackendComputeGroup(); + } + } + }; + + // Mock cloud environment permissions + new MockUp() { + @Mock + public void checkCloudClusterPriv(String cluster) throws Exception { + // Always allow for tests + } + }; + + // Mock ConnectContext to avoid compute group permission check + new MockUp() { + @Mock + public String getCloudCluster() { + return "test_group"; + } + + @Mock + public UserIdentity getCurrentUserIdentity() { + UserIdentity rootUser = new UserIdentity("root", "%"); + rootUser.setIsAnalyzed(); + return rootUser; + } + }; + + analyzer = new Analyzer(masterEnv, ctx); + + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + // Mock addCloudCluster to avoid EditLog issues + new MockUp() { + @Mock + public void addCloudCluster(String clusterName, String clusterId) { + // Create backend manually for test + Backend backend = new Backend(10001L, "host1", 123); + backend.setAlive(true); + backend.setBePort(456); + backend.setHttpPort(789); + backend.setBrpcPort(321); + backend.setTagMap(Maps.newHashMap()); + backend.getTagMap().put("cloud_cluster_id", "test_id"); + backend.getTagMap().put("cloud_unique_id", "test_cloud"); + backend.getTagMap().put("cloud_cluster_name", "test_group"); + backend.getTagMap().put("cloud_cluster_status", "NORMAL"); + backend.getTagMap().put("location", "default"); + backend.getTagMap().put("cloud_cluster_private_endpoint", ""); + backend.getTagMap().put("cloud_cluster_public_endpoint", ""); + CloudSystemInfoService systemInfo = (CloudSystemInfoService) Env.getCurrentSystemInfo(); + systemInfo.addBackend(backend); + } + }; + ((CloudSystemInfoService) Env.getCurrentSystemInfo()).addCloudCluster("test_group", ""); + List backends = + ((CloudSystemInfoService) Env.getCurrentSystemInfo()).getBackendsByClusterName("test_group"); + Assert.assertEquals(1, backends.size()); + Assert.assertEquals("host1", backends.get(0).getHost()); + backends.get(0).setAlive(true); + ctx.setComputeGroup(masterEnv.getComputeGroupMgr().getAllBackendComputeGroup()); + + db = new Database(CatalogTestUtil.testDbId1, CatalogTestUtil.testDb1); + masterEnv.unprotectCreateDb(db); + + AgentTaskQueue.clearAllTasks(); + schemaChangeHandler = masterEnv.getSchemaChangeHandler(); + } + + @Test + public void testCreateNgramBfIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + table.setDataSortInfo(dataSortInfo); + String indexName = "ngram_bf_index"; + + // Add required properties for NGRAM_BF index + Map properties = Maps.newHashMap(); + properties.put("gram_size", "2"); + properties.put("bf_size", "256"); + + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + properties, "ngram bf index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + ctx.getSessionVariable().setEnableAddIndexForNewData(true); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); + Assert.assertEquals(OlapTableState.NORMAL, table.getState()); + + long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; + + // Finish the create index job first + SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2) indexChangeJobMap.get(createJobId); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, createJobV2.getJobState()); + } + + @Test + public void testNormalCreateNgramBfIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + table.setDataSortInfo(dataSortInfo); + String indexName = "ngram_bf_index"; + + // Add required properties for NGRAM_BF index + Map properties = Maps.newHashMap(); + properties.put("gram_size", "2"); + properties.put("bf_size", "256"); + + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + properties, "ngram bf index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + // Set session variable to false (default) + ctx.getSessionVariable().setEnableAddIndexForNewData(false); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + + long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; + + // Finish the create index job first + SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2) indexChangeJobMap.get(createJobId); + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, createJobV2.getJobState()); + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, createJobV2.getJobState()); + Assert.assertEquals(1, createJobV2.schemaChangeBatchTask.getTaskNum()); + + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, createJobV2.getJobState()); + Assert.assertEquals(OlapTableState.NORMAL, table.getState()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); + } + + @Test + public void testCreateInvertedIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + table.setDataSortInfo(dataSortInfo); + String indexName = "raw_inverted_index"; + // Explicitly set parser="none" for raw inverted index + Map properties = Maps.newHashMap(); + properties.put("parser", "none"); + + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + IndexType.INVERTED, + properties, "raw inverted index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + ctx.getSessionVariable().setEnableAddIndexForNewData(false); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + + long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + + // Finish the create index job first + SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2) indexChangeJobMap.get(createJobId); + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, createJobV2.getJobState()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, createJobV2.getJobState()); + Assert.assertEquals(1, createJobV2.schemaChangeBatchTask.getTaskNum()); + + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, createJobV2.getJobState()); + Assert.assertEquals(OlapTableState.NORMAL, table.getState()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("raw_inverted_index", table.getIndexes().get(0).getIndexName()); + } + + @Test + public void testCreateInvertedIndexWithLightweightMode() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + table.setDataSortInfo(dataSortInfo); + String indexName = "lightweight_raw_inverted_index"; + // Explicitly set parser="none" for raw inverted index + Map properties = Maps.newHashMap(); + properties.put("parser", "none"); + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + IndexType.INVERTED, + properties, "lightweight raw inverted index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + // Test with enable_add_index_for_new_data = true, should use lightweight mode + ctx.getSessionVariable().setEnableAddIndexForNewData(true); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + // Lightweight mode should not create any schema change jobs + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("lightweight_raw_inverted_index", table.getIndexes().get(0).getIndexName()); + Assert.assertEquals(OlapTableState.NORMAL, table.getState()); + // Verify the index properties + Assert.assertEquals("none", table.getIndexes().get(0).getProperties().get("parser")); + } + + @Test + public void testCreateTokenizedInvertedIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + table.setDataSortInfo(dataSortInfo); + + // Set inverted index file storage format to V2 for cloud mode + table.setInvertedIndexFileStorageFormat(TInvertedIndexFileStorageFormat.V2); + + String indexName = "tokenized_inverted_index"; + Map properties = Maps.newHashMap(); + properties.put("parser", "english"); + properties.put("support_phrase", "true"); + properties.put("lower_case", "true"); + + // Use VARCHAR column v1 (index 2) for string type support + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(2).getName()), + IndexType.INVERTED, + properties, "tokenized inverted index with english parser"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + + SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + // This should be a heavyweight schema change for tokenized index + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, jobV2.getJobState()); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); + + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("tokenized_inverted_index", table.getIndexes().get(0).getIndexName()); + + // Verify that the index has the correct properties + Assert.assertEquals("english", table.getIndexes().get(0).getProperties().get("parser")); + Assert.assertEquals("true", table.getIndexes().get(0).getProperties().get("support_phrase")); + Assert.assertEquals("true", table.getIndexes().get(0).getProperties().get("lower_case")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java index 590b6563e1106c..b16d3d15cf7e52 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java @@ -22,7 +22,8 @@ import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MaterializedIndex.IndexState; -import org.apache.doris.catalog.Replica.ReplicaState; +import org.apache.doris.cloud.catalog.CloudReplica; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.persist.EditLog; @@ -175,13 +176,26 @@ public static Database createSimpleDb(long dbId, long tableId, long partitionId, long version) { Env.getCurrentInvertedIndex().clear(); - // replica - Replica replica1 = new Replica(testReplicaId1, testBackendId1, version, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); - Replica replica2 = new Replica(testReplicaId2, testBackendId2, version, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); - Replica replica3 = new Replica(testReplicaId3, testBackendId3, version, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); + Replica replica1; + Replica replica2; + Replica replica3; + if (Config.isCloudMode()) { + // In cloud mode we must create CloudReplica instances to avoid ClassCastException + replica1 = new CloudReplica(testReplicaId1, testBackendId1, Replica.ReplicaState.NORMAL, version, + /*schemaHash*/ 0, dbId, tableId, partitionId, indexId, /*idx*/ 0); + replica2 = new CloudReplica(testReplicaId2, testBackendId2, Replica.ReplicaState.NORMAL, version, + 0, dbId, tableId, partitionId, indexId, 1); + replica3 = new CloudReplica(testReplicaId3, testBackendId3, Replica.ReplicaState.NORMAL, version, + 0, dbId, tableId, partitionId, indexId, 2); + } else { + replica1 = new Replica(testReplicaId1, testBackendId1, version, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + replica2 = new Replica(testReplicaId2, testBackendId2, version, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + replica3 = new Replica(testReplicaId3, testBackendId3, version, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + } + // tablet Tablet tablet = new Tablet(tabletId); @@ -244,10 +258,14 @@ public static Database createSimpleDb(long dbId, long tableId, long partitionId, } public static void createDupTable(Database db) { - - // replica - Replica replica = new Replica(testReplicaId4, testBackendId1, testStartVersion, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); + Replica replica; + if (Config.isCloudMode()) { + replica = new CloudReplica(testReplicaId4, testBackendId1, Replica.ReplicaState.NORMAL, testStartVersion, + 0, db.getId(), testTableId2, testPartitionId2, testIndexId2, 0); + } else { + replica = new Replica(testReplicaId4, testBackendId1, testStartVersion, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + } // tablet Tablet tablet = new Tablet(testTabletId2); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEditLog.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEditLog.java index 08800510a1a6ef..e2b113958366c8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEditLog.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEditLog.java @@ -100,6 +100,16 @@ public void logModifyDistributionType(TableInfo tableInfo) { } + @Mock + public void logAddBackend(Backend be) { + // do nothing for test + } + + @Mock + public int getNumEditStreams() { + return 1; // fake that we have streams + } + public TransactionState getTransaction(long transactionId) { return allTransactionState.get(transactionId); } diff --git a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy index 5f64c0ca3290a6..402f72ddc36065 100644 --- a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy +++ b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy @@ -63,8 +63,6 @@ suite("test_ngram_bloomfilter_index_change") { def query = "SELECT /*+SET_VAR(enable_function_pushdown = true, enable_profile = true, profile_level = 2)*/ * FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id" // Test Case 1: Test with enable_add_index_for_new_data = true logger.info("=== Test Case 1: enable_add_index_for_new_data = true ===") - // Set enable_add_index_for_new_data = true - sql "set enable_add_index_for_new_data = true" // Create table sql "DROP TABLE IF EXISTS ${tableName}" sql """ @@ -87,7 +85,7 @@ suite("test_ngram_bloomfilter_index_change") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false" + "disable_auto_compaction" = "true" ); """ @@ -108,6 +106,7 @@ suite("test_ngram_bloomfilter_index_change") { assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) } } + sql "set enable_add_index_for_new_data = true" // Add NGRAM Bloom Filter index (should be immediate in light mode) sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" @@ -177,7 +176,7 @@ suite("test_ngram_bloomfilter_index_change") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false" + "disable_auto_compaction" = "true" ); """ // Insert test data @@ -276,7 +275,7 @@ suite("test_ngram_bloomfilter_index_change") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false" + "disable_auto_compaction" = "true" ); """