diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index c9fca5724e2d23..9de9e3c3a72b59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1258,12 +1258,7 @@ private void checkAssignedTargetIndexName(String baseIndexName, String targetInd } private void createJob(String rawSql, long dbId, OlapTable olapTable, Map> indexSchemaMap, - Map propertyMap, List indexes, - boolean isBuildIndex) throws UserException { - if (isBuildIndex) { - // remove the index which is not the base index, only base index can be built index - indexSchemaMap.entrySet().removeIf(entry -> !entry.getKey().equals(olapTable.getBaseIndexId())); - } + Map propertyMap, List indexes) throws UserException { checkReplicaCount(olapTable); // process properties first @@ -1299,7 +1294,7 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map newSet = new HashSet<>(indexes); Set oriSet = new HashSet<>(olapTable.getIndexes()); - if (!newSet.equals(oriSet) || isBuildIndex) { + if (!newSet.equals(oriSet)) { hasIndexChange = true; } @@ -2092,9 +2087,21 @@ public int getAsInt() { } lightSchemaChange = false; + // Check if the index supports light index change and session variable is enabled + boolean enableAddIndexForNewData = true; + try { + ConnectContext context = ConnectContext.get(); + if (context != null && context.getSessionVariable() != null) { + enableAddIndexForNewData = context.getSessionVariable().isEnableAddIndexForNewData(); + } + } catch (Exception e) { + LOG.warn("Failed to get session variable enable_add_index_for_new_data, " + + "using default value: false", e); + } + // ngram_bf index can do light_schema_change in both local and cloud mode // inverted index can only do light_schema_change in local mode - if (index.isLightIndexChangeSupported()) { + if (index.isLightAddIndexSupported(enableAddIndexForNewData)) { alterIndexes.add(index); isDropIndex = false; lightIndexChange = true; @@ -2103,7 +2110,7 @@ public int getAsInt() { BuildIndexClause buildIndexClause = (BuildIndexClause) alterClause; IndexDef indexDef = buildIndexClause.getIndexDef(); Index index = buildIndexClause.getIndex(); - if (Config.isCloudMode() && index.getIndexType() == IndexDef.IndexType.INVERTED) { + if (Config.isCloudMode()) { throw new DdlException("BUILD INDEX operation failed: No need to do it in cloud mode."); } @@ -2165,17 +2172,12 @@ public int getAsInt() { if (alterIndexes.isEmpty()) { throw new DdlException("Altered index is empty. please check your alter stmt."); } - IndexDef.IndexType indexType = alterIndexes.get(0).getIndexType(); if (Config.enable_light_index_change) { - if (indexType == IndexDef.IndexType.INVERTED) { - buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, - alterIndexes, indexOnPartitions, false); - } else { - createJob(rawSql, db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes, true); - } + buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, + alterIndexes, indexOnPartitions, false); } } else { - createJob(rawSql, db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes, false); + createJob(rawSql, db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes); } } finally { olapTable.writeUnlock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java index ba72155f7ec5fe..929835d9433433 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java @@ -116,10 +116,10 @@ public void analyze(Analyzer analyzer) throws AnalysisException { } IndexDef.IndexType indexType = existedIdx.getIndexType(); - if (!existedIdx.isLightIndexChangeSupported()) { - throw new AnalysisException(indexType.toString() + " index is not needed to build."); + if (indexType == IndexDef.IndexType.NGRAM_BF + || indexType == IndexDef.IndexType.BLOOMFILTER) { + throw new AnalysisException("ngram bloomfilter or bloomfilter index is not needed to build."); } - indexDef = new IndexDef(indexName, partitionNames, indexType, true); if (!table.isPartitionedTable()) { List specifiedPartitions = indexDef.getPartitionNames(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index fa4554951d3f88..4916d0e48abf19 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -153,6 +153,10 @@ public String getInvertedIndexParser() { return InvertedIndexUtil.getInvertedIndexParser(properties); } + public boolean isInvertedIndexParserNone() { + return InvertedIndexUtil.INVERTED_INDEX_PARSER_NONE.equals(getInvertedIndexParser()); + } + public String getInvertedIndexParserMode() { return InvertedIndexUtil.getInvertedIndexParserMode(properties); } @@ -170,16 +174,25 @@ public String getInvertedIndexParserStopwords() { } // Whether the index can be changed in light mode - // cloud mode only supports light change for ngram_bf index - // local mode supports light change for both inverted index and ngram_bf index - // the rest of the index types do not support light change public boolean isLightIndexChangeSupported() { + return indexType == IndexDef.IndexType.INVERTED; + } + + // Whether the index can be added in light mode + // cloud mode supports light add for ngram_bf index and non-tokenized inverted index (parser="none") + // local mode supports light add for both inverted index and ngram_bf index + // the rest of the index types do not support light add + public boolean isLightAddIndexSupported(boolean enableAddIndexForNewData) { if (Config.isCloudMode()) { - return indexType == IndexDef.IndexType.NGRAM_BF; - } else { - return indexType == IndexDef.IndexType.INVERTED - || indexType == IndexDef.IndexType.NGRAM_BF; + if (indexType == IndexDef.IndexType.INVERTED) { + return isInvertedIndexParserNone() && enableAddIndexForNewData; + } else if (indexType == IndexDef.IndexType.NGRAM_BF) { + return enableAddIndexForNewData; + } + return false; } + return (indexType == IndexDef.IndexType.NGRAM_BF && enableAddIndexForNewData) + || (indexType == IndexDef.IndexType.INVERTED); } public String getInvertedIndexCustomAnalyzer() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index 15de6f132dada9..0353856d5e0ccc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -344,6 +344,14 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2); } else if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V3) { schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V3); + } else if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.DEFAULT) { + if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) { + schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V1); + } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V3")) { + schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V3); + } else { + schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2); + } } else { throw new DdlException("invalid inverted index storage format"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java index 36439874760144..1038f23a3434d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java @@ -114,8 +114,9 @@ public void validate(ConnectContext ctx) throws UserException { } IndexDef.IndexType indexType = existedIdx.getIndexType(); - if (!existedIdx.isLightIndexChangeSupported()) { - throw new AnalysisException(indexType.toString() + " index is not needed to build."); + if (indexType == IndexDef.IndexType.NGRAM_BF + || indexType == IndexDef.IndexType.BLOOMFILTER) { + throw new AnalysisException(indexType + " index is not needed to build."); } indexDef = new IndexDefinition(indexName, partitionNamesInfo, indexType); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index dccfbd3f17f100..be151e35c7e1ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -746,6 +746,7 @@ public class SessionVariable implements Serializable, Writable { public static final String SQL_CONVERTOR_CONFIG = "sql_convertor_config"; public static final String PREFER_UDF_OVER_BUILTIN = "prefer_udf_over_builtin"; + public static final String ENABLE_ADD_INDEX_FOR_NEW_DATA = "enable_add_index_for_new_data"; /** * If set false, user couldn't submit analyze SQL and FE won't allocate any related resources. @@ -2663,6 +2664,14 @@ public boolean isEnableESParallelScroll() { return enableESParallelScroll; } + @VariableMgr.VarAttr(name = ENABLE_ADD_INDEX_FOR_NEW_DATA, fuzzy = true, description = { + "是否启用仅对新数据生效的索引添加模式，开启时新建索引只对后续写入的数据生效，关闭时对全部数据重建索引", + "Whether to enable add index mode that only affects new data, " + + "when enabled new indexes only affect subsequently written data, " + + "when disabled rebuild indexes for all data" + }) + public boolean enableAddIndexForNewData = false; + // If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables, // not the default value set in the code. @SuppressWarnings("checkstyle:Indentation") @@ -4925,4 +4934,11 @@ public boolean showSplitProfileInfo() { return enableProfile() && getProfileLevel() > 1; } + public boolean isEnableAddIndexForNewData() { + return enableAddIndexForNewData; + } + + public void setEnableAddIndexForNewData(boolean enableAddIndexForNewData) { + this.enableAddIndexForNewData = enableAddIndexForNewData; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java new file mode 100644 index 00000000000000..1836edc7bf6580 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java @@ -0,0 +1,664 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.alter; + +import org.apache.doris.analysis.AlterClause; +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.BuildIndexClause; +import org.apache.doris.analysis.CreateIndexClause; +import org.apache.doris.analysis.DataSortInfo; +import org.apache.doris.analysis.DropIndexClause; +import org.apache.doris.analysis.IndexDef; +import org.apache.doris.analysis.IndexDef.IndexType; +import org.apache.doris.analysis.ResourceTypeEnum; +import org.apache.doris.analysis.TableName; +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.CatalogTestUtil; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.EnvFactory; +import org.apache.doris.catalog.FakeEditLog; +import org.apache.doris.catalog.FakeEnv; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.OlapTable.OlapTableState; +import org.apache.doris.cloud.catalog.CloudEnv; +import org.apache.doris.cloud.catalog.CloudEnvFactory; +import org.apache.doris.cloud.datasource.CloudInternalCatalog; +import org.apache.doris.cloud.proto.Cloud; +import org.apache.doris.cloud.proto.Cloud.MetaServiceCode; +import org.apache.doris.cloud.rpc.MetaServiceProxy; +import org.apache.doris.cloud.system.CloudSystemInfoService; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.UserException; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.Auth; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.commands.CancelBuildIndexCommand; +import org.apache.doris.persist.EditLog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.resource.computegroup.ComputeGroup; +import org.apache.doris.resource.computegroup.ComputeGroupMgr; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.task.AgentTask; +import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; +import org.apache.doris.thrift.TSortType; +import org.apache.doris.thrift.TTaskType; +import org.apache.doris.utframe.MockedMetaServerFactory; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import mockit.Mock; +import mockit.MockUp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class CloudIndexTest { + private static final Logger LOG = LogManager.getLogger(CloudIndexTest.class); + + private static String fileName = "./CloudIndexTest"; + + private static FakeEditLog fakeEditLog; + private static FakeEnv fakeEnv; + private static Env masterEnv; + private static EditLog testEditLog; + private ConnectContext ctx; + + private static Analyzer analyzer; + private static Database db; + private static OlapTable olapTable; + private static CreateIndexClause createIndexClause; + private static BuildIndexClause buildIndexClause; + private static DropIndexClause dropIndexClause; + private static CancelBuildIndexCommand cancelBuildIndexCommand; + private static SchemaChangeHandler schemaChangeHandler; + + @Before + public void setUp() throws InstantiationException, IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException, SecurityException, UserException { + FeConstants.runningUnitTest = true; + // Setup for MetaServiceProxy mock + new MockUp(MetaServiceProxy.class) { + + @Mock + public Cloud.BeginTxnResponse beginTxn(Cloud.BeginTxnRequest request) { + Cloud.BeginTxnResponse.Builder beginTxnResponseBuilder = Cloud.BeginTxnResponse.newBuilder(); + beginTxnResponseBuilder.setTxnId(1000) + .setStatus( + Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK")); + return beginTxnResponseBuilder.build(); + } + + @Mock + public Cloud.CommitTxnResponse commitTxn(Cloud.CommitTxnRequest request) { + Cloud.TxnInfoPB.Builder txnInfoBuilder = Cloud.TxnInfoPB.newBuilder(); + txnInfoBuilder.setDbId(CatalogTestUtil.testDbId1); + txnInfoBuilder.addAllTableIds(Lists.newArrayList(olapTable.getId())); + txnInfoBuilder.setLabel("test_label"); + txnInfoBuilder.setListenerId(-1); + Cloud.CommitTxnResponse.Builder commitTxnResponseBuilder = Cloud.CommitTxnResponse.newBuilder(); + commitTxnResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .setTxnInfo(txnInfoBuilder.build()); + return commitTxnResponseBuilder.build(); + } + + @Mock + public Cloud.CheckTxnConflictResponse checkTxnConflict(Cloud.CheckTxnConflictRequest request) { + Cloud.CheckTxnConflictResponse.Builder checkTxnConflictResponseBuilder = + Cloud.CheckTxnConflictResponse.newBuilder(); + checkTxnConflictResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .setFinished(true); + return checkTxnConflictResponseBuilder.build(); + } + + @Mock + public Cloud.GetClusterResponse getCluster(Cloud.GetClusterRequest request) { + Cloud.GetClusterResponse.Builder getClusterResponseBuilder = Cloud.GetClusterResponse.newBuilder(); + Cloud.ClusterPB.Builder clusterBuilder = Cloud.ClusterPB.newBuilder(); + clusterBuilder.setClusterId("test_id").setClusterName("test_group"); + + Cloud.NodeInfoPB.Builder node1 = Cloud.NodeInfoPB.newBuilder(); + node1.setCloudUniqueId("test_cloud") + .setName("host1") + .setIp("host1") + .setHost("host1") + .setHeartbeatPort(123) + .setEditLogPort(125) + .setStatus(Cloud.NodeStatusPB.NODE_STATUS_RUNNING); + clusterBuilder.addNodes(node1.build()); + getClusterResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .addCluster(clusterBuilder.build()); + return getClusterResponseBuilder.build(); + } + + @Mock + public Cloud.CreateTabletsResponse createTablets(Cloud.CreateTabletsRequest request) { + Cloud.CreateTabletsResponse.Builder responseBuilder = Cloud.CreateTabletsResponse.newBuilder(); + responseBuilder.setStatus( + Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK")); + return responseBuilder.build(); + } + + @Mock + public Cloud.FinishTabletJobResponse finishTabletJob(Cloud.FinishTabletJobRequest request) { + Cloud.FinishTabletJobResponse.Builder responseBuilder = Cloud.FinishTabletJobResponse.newBuilder(); + responseBuilder.setStatus( + Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK")); + return responseBuilder.build(); + } + + @Mock + public Cloud.IndexResponse prepareIndex(Cloud.IndexRequest request) { + Cloud.IndexResponse.Builder builder = Cloud.IndexResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.IndexResponse commitIndex(Cloud.IndexRequest request) { + Cloud.IndexResponse.Builder builder = Cloud.IndexResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.IndexResponse dropIndex(Cloud.IndexRequest request) { + Cloud.IndexResponse.Builder builder = Cloud.IndexResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.CheckKVResponse checkKv(Cloud.CheckKVRequest request) { + Cloud.CheckKVResponse.Builder builder = Cloud.CheckKVResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.GetCurrentMaxTxnResponse getCurrentMaxTxnId(Cloud.GetCurrentMaxTxnRequest request) { + Cloud.GetCurrentMaxTxnResponse.Builder builder = Cloud.GetCurrentMaxTxnResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .setCurrentMaxTxnId(1000); + return builder.build(); + } + }; + + Config.cloud_unique_id = "test_cloud"; + Config.meta_service_endpoint = MockedMetaServerFactory.METASERVER_DEFAULT_IP + ":" + 20121; + + EnvFactory envFactory = EnvFactory.getInstance(); + masterEnv = envFactory.createEnv(false); + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + FakeEnv.setSystemInfo(cloudSystemInfo); + + fakeEditLog = new FakeEditLog(); + testEditLog = null; // Will be set by MockUp + FakeEnv.setEnv(masterEnv); + + ctx = new ConnectContext(); + ctx.setEnv(masterEnv); + ctx.setQualifiedUser("root"); + UserIdentity rootUser = new UserIdentity("root", "%"); + rootUser.setIsAnalyzed(); + ctx.setCurrentUserIdentity(rootUser); + ctx.setThreadLocalInfo(); + ctx.setCloudCluster("test_group"); + Assert.assertTrue(envFactory instanceof CloudEnvFactory); + Assert.assertTrue(masterEnv instanceof CloudEnv); + new MockUp() { + @Mock + public Env getCurrentEnv() { + return masterEnv; + } + + @Mock + public EditLog getEditLog() { + if (testEditLog == null) { + // Create a mock EditLog using a no-op approach + testEditLog = new EditLog("test") { + // Override to avoid initialization issues + }; + } + return testEditLog; + } + + @Mock + public ComputeGroupMgr getComputeGroupMgr() { + return new ComputeGroupMgr(Env.getCurrentSystemInfo()); + } + + @Mock + public SchemaChangeHandler getSchemaChangeHandler() { + // Create a new independent SchemaChangeHandler for each call + return schemaChangeHandler; + } + + @Mock + public AccessControllerManager getAccessManager() { + return new AccessControllerManager(masterEnv.getAuth()) { + @Override + public boolean checkTblPriv(ConnectContext ctx, String ctl, String db, String tbl, PrivPredicate wanted) { + return true; // Allow all access for test + } + + @Override + public boolean checkCloudPriv(UserIdentity user, String cluster, PrivPredicate wanted, ResourceTypeEnum resourceType) { + return true; // Allow all cloud privileges for test + } + }; + } + }; + + new MockUp() { + @Mock + public String getDefaultCloudCluster(String user) { + return "test_group"; // Return default cluster for test + } + + @Mock + public ComputeGroup getComputeGroup(String user) { + try { + return masterEnv.getComputeGroupMgr().getComputeGroupByName("test_group"); + } catch (Exception e) { + return masterEnv.getComputeGroupMgr().getAllBackendComputeGroup(); + } + } + }; + + // Mock cloud environment permissions + new MockUp() { + @Mock + public void checkCloudClusterPriv(String cluster) throws Exception { + // Always allow for tests + } + }; + + // Mock ConnectContext to avoid compute group permission check + new MockUp() { + @Mock + public String getCloudCluster() { + return "test_group"; + } + + @Mock + public UserIdentity getCurrentUserIdentity() { + UserIdentity rootUser = new UserIdentity("root", "%"); + rootUser.setIsAnalyzed(); + return rootUser; + } + }; + + analyzer = new Analyzer(masterEnv, ctx); + + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + // Mock addCloudCluster to avoid EditLog issues + new MockUp() { + @Mock + public void addCloudCluster(String clusterName, String clusterId) { + // Create backend manually for test + Backend backend = new Backend(10001L, "host1", 123); + backend.setAlive(true); + backend.setBePort(456); + backend.setHttpPort(789); + backend.setBrpcPort(321); + backend.setTagMap(Maps.newHashMap()); + backend.getTagMap().put("cloud_cluster_id", "test_id"); + backend.getTagMap().put("cloud_unique_id", "test_cloud"); + backend.getTagMap().put("cloud_cluster_name", "test_group"); + backend.getTagMap().put("cloud_cluster_status", "NORMAL"); + backend.getTagMap().put("location", "default"); + backend.getTagMap().put("cloud_cluster_private_endpoint", ""); + backend.getTagMap().put("cloud_cluster_public_endpoint", ""); + CloudSystemInfoService systemInfo = (CloudSystemInfoService) Env.getCurrentSystemInfo(); + systemInfo.addBackend(backend); + } + }; + ((CloudSystemInfoService) Env.getCurrentSystemInfo()).addCloudCluster("test_group", ""); + List backends = + ((CloudSystemInfoService) Env.getCurrentSystemInfo()).getBackendsByClusterName("test_group"); + Assert.assertEquals(1, backends.size()); + Assert.assertEquals("host1", backends.get(0).getHost()); + backends.get(0).setAlive(true); + ctx.setComputeGroup(masterEnv.getComputeGroupMgr().getAllBackendComputeGroup()); + + db = new Database(CatalogTestUtil.testDbId1, CatalogTestUtil.testDb1); + masterEnv.unprotectCreateDb(db); + + AgentTaskQueue.clearAllTasks(); + schemaChangeHandler = masterEnv.getSchemaChangeHandler(); + } + + @Test + public void testCreateNgramBfIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + table.setDataSortInfo(dataSortInfo); + String indexName = "ngram_bf_index"; + + // Add required properties for NGRAM_BF index + Map properties = Maps.newHashMap(); + properties.put("gram_size", "2"); + properties.put("bf_size", "256"); + + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + properties, "ngram bf index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + ctx.getSessionVariable().setEnableAddIndexForNewData(true); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); + Assert.assertEquals(OlapTableState.NORMAL, table.getState()); + + long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; + + // Finish the create index job first + SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2) indexChangeJobMap.get(createJobId); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, createJobV2.getJobState()); + } + + @Test + public void testNormalCreateNgramBfIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + table.setDataSortInfo(dataSortInfo); + String indexName = "ngram_bf_index"; + + // Add required properties for NGRAM_BF index + Map properties = Maps.newHashMap(); + properties.put("gram_size", "2"); + properties.put("bf_size", "256"); + + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + properties, "ngram bf index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + // Set session variable to false (default) + ctx.getSessionVariable().setEnableAddIndexForNewData(false); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + + long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; + + // Finish the create index job first + SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2) indexChangeJobMap.get(createJobId); + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, createJobV2.getJobState()); + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, createJobV2.getJobState()); + Assert.assertEquals(1, createJobV2.schemaChangeBatchTask.getTaskNum()); + + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, createJobV2.getJobState()); + Assert.assertEquals(OlapTableState.NORMAL, table.getState()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); + } + + @Test + public void testCreateInvertedIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + table.setDataSortInfo(dataSortInfo); + String indexName = "raw_inverted_index"; + // Explicitly set parser="none" for raw inverted index + Map properties = Maps.newHashMap(); + properties.put("parser", "none"); + + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + IndexType.INVERTED, + properties, "raw inverted index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + ctx.getSessionVariable().setEnableAddIndexForNewData(false); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + + long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + + // Finish the create index job first + SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2) indexChangeJobMap.get(createJobId); + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, createJobV2.getJobState()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, createJobV2.getJobState()); + Assert.assertEquals(1, createJobV2.schemaChangeBatchTask.getTaskNum()); + + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, createJobV2.getJobState()); + Assert.assertEquals(OlapTableState.NORMAL, table.getState()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("raw_inverted_index", table.getIndexes().get(0).getIndexName()); + } + + @Test + public void testCreateInvertedIndexWithLightweightMode() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + table.setDataSortInfo(dataSortInfo); + String indexName = "lightweight_raw_inverted_index"; + // Explicitly set parser="none" for raw inverted index + Map properties = Maps.newHashMap(); + properties.put("parser", "none"); + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + IndexType.INVERTED, + properties, "lightweight raw inverted index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + // Test with enable_add_index_for_new_data = true, should use lightweight mode + ctx.getSessionVariable().setEnableAddIndexForNewData(true); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + // Lightweight mode should not create any schema change jobs + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("lightweight_raw_inverted_index", table.getIndexes().get(0).getIndexName()); + Assert.assertEquals(OlapTableState.NORMAL, table.getState()); + // Verify the index properties + Assert.assertEquals("none", table.getIndexes().get(0).getProperties().get("parser")); + } + + @Test + public void testCreateTokenizedInvertedIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + table.setDataSortInfo(dataSortInfo); + + // Set inverted index file storage format to V2 for cloud mode + table.setInvertedIndexFileStorageFormat(TInvertedIndexFileStorageFormat.V2); + + String indexName = "tokenized_inverted_index"; + Map properties = Maps.newHashMap(); + properties.put("parser", "english"); + properties.put("support_phrase", "true"); + properties.put("lower_case", "true"); + + // Use VARCHAR column v1 (index 2) for string type support + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(2).getName()), + IndexType.INVERTED, + properties, "tokenized inverted index with english parser"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + + SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + // This should be a heavyweight schema change for tokenized index + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, jobV2.getJobState()); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); + + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("tokenized_inverted_index", table.getIndexes().get(0).getIndexName()); + + // Verify that the index has the correct properties + Assert.assertEquals("english", table.getIndexes().get(0).getProperties().get("parser")); + Assert.assertEquals("true", table.getIndexes().get(0).getProperties().get("support_phrase")); + Assert.assertEquals("true", table.getIndexes().get(0).getProperties().get("lower_case")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java index 697b101884824f..b1eda304f227a8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java @@ -44,6 +44,9 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.UserException; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.nereids.trees.plans.commands.CancelAlterTableCommand; +import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.thrift.TStatusCode; @@ -86,6 +89,7 @@ public class IndexChangeJobTest { private static DropIndexClause dropIndexClause; private static TableName tableName; private static String indexName; + private static ConnectContext ctx; @Rule public ExpectedException expectedEx = ExpectedException.none(); @@ -116,6 +120,15 @@ public Env getCurrentEnv() { } }; + // Initialize ConnectContext + ctx = new ConnectContext(); + new MockUp() { + @Mock + public ConnectContext get() { + return ctx; + } + }; + // set mow table property Map properties = Maps.newHashMap(); properties.put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE, "false"); @@ -587,25 +600,45 @@ public void testNgramBfBuildIndex() throws UserException { SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); ArrayList alterClauses = new ArrayList<>(); alterClauses.add(createIndexClause); + + // Test with enable_add_index_for_new_data = true + ConnectContext context = ConnectContext.get(); + context.getSessionVariable().setEnableAddIndexForNewData(true); schemaChangeHandler.process(alterClauses, db, table); Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); Assert.assertEquals(1, indexChangeJobMap.size()); Assert.assertEquals(1, table.getIndexes().size()); Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); - long jobId = indexChangeJobMap.values().stream().findAny().get().jobId; + SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); - buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); - buildIndexClause.analyze(analyzer); - alterClauses.clear(); - alterClauses.add(buildIndexClause); + // Clean up for next test + table.setIndexes(Lists.newArrayList()); + indexChangeJobMap.clear(); + AgentTaskQueue.clearAllTasks(); - schemaChangeHandler.process(alterClauses, db, table); - Assert.assertEquals(2, indexChangeJobMap.size()); + // Test with enable_add_index_for_new_data = false + context.getSessionVariable().setEnableAddIndexForNewData(false); + String indexName2 = "ngram_bf_index2"; + IndexDef indexDef2 = new IndexDef(indexName2, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + Maps.newHashMap(), "ngram bf index2"); + + createIndexClause = new CreateIndexClause(tableName, indexDef2, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses2 = new ArrayList<>(); + alterClauses2.add(createIndexClause); + schemaChangeHandler.process(alterClauses2, db, table); + indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); - SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() - .filter(job -> job.jobId != jobId) + jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() .findFirst() .orElse(null); Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); @@ -630,6 +663,8 @@ public void testNgramBfBuildIndex() throws UserException { schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("ngram_bf_index2", table.getIndexes().get(0).getIndexName()); } @Test @@ -651,25 +686,15 @@ public void testCancelNgramBfBuildIndex() throws UserException { SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); ArrayList alterClauses = new ArrayList<>(); alterClauses.add(createIndexClause); + + //cancel test can only with enable_add_index_for_new_data = false + ctx.getSessionVariable().setEnableAddIndexForNewData(false); schemaChangeHandler.process(alterClauses, db, table); Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); Assert.assertEquals(1, indexChangeJobMap.size()); - Assert.assertEquals(1, table.getIndexes().size()); - Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); - - long jobId = indexChangeJobMap.values().stream().findAny().get().jobId; - - buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); - buildIndexClause.analyze(analyzer); - alterClauses.clear(); - alterClauses.add(buildIndexClause); - - schemaChangeHandler.process(alterClauses, db, table); - Assert.assertEquals(2, indexChangeJobMap.size()); Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() - .filter(job -> job.jobId != jobId) .findFirst() .orElse(null); Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); @@ -685,5 +710,15 @@ public void testCancelNgramBfBuildIndex() throws UserException { schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + TableNameInfo tableNameInfo = new TableNameInfo(db.getName(), table.getName()); + CancelAlterTableCommand cancelAlterTableCommand = new CancelAlterTableCommand( + tableNameInfo, + CancelAlterTableCommand.AlterType.COLUMN, + Lists.newArrayList()); + schemaChangeHandler.cancel(cancelAlterTableCommand); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.CANCELLED, jobV2.getJobState()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java index 4b8f218acea094..f440136e5cb8d5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java @@ -849,14 +849,8 @@ public void testDupAddOrDropNgramBfIndex() throws Exception { waitAlterJobDone(alterJobs); String buildNgramBfIndexStmtStr = "BUILD INDEX idx_error_msg on test.sc_dup "; - AlterTableStmt buildNgramBfIndexStmt = (AlterTableStmt) parseAndAnalyzeStmt(buildNgramBfIndexStmtStr); - Env.getCurrentEnv().getAlterInstance().processAlterTable(buildNgramBfIndexStmt); - - jobSize++; - alterJobs = Env.getCurrentEnv().getSchemaChangeHandler().getAlterJobsV2(); - LOG.info("alterJobs:{}", alterJobs); - Assertions.assertEquals(jobSize, alterJobs.size()); - waitAlterJobDone(alterJobs); + Assertions.assertThrows(org.apache.doris.common.AnalysisException.class, + () -> parseAndAnalyzeStmt(buildNgramBfIndexStmtStr)); tbl.readLock(); try { diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java index 590b6563e1106c..b16d3d15cf7e52 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java @@ -22,7 +22,8 @@ import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MaterializedIndex.IndexState; -import org.apache.doris.catalog.Replica.ReplicaState; +import org.apache.doris.cloud.catalog.CloudReplica; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.persist.EditLog; @@ -175,13 +176,26 @@ public static Database createSimpleDb(long dbId, long tableId, long partitionId, long version) { Env.getCurrentInvertedIndex().clear(); - // replica - Replica replica1 = new Replica(testReplicaId1, testBackendId1, version, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); - Replica replica2 = new Replica(testReplicaId2, testBackendId2, version, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); - Replica replica3 = new Replica(testReplicaId3, testBackendId3, version, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); + Replica replica1; + Replica replica2; + Replica replica3; + if (Config.isCloudMode()) { + // In cloud mode we must create CloudReplica instances to avoid ClassCastException + replica1 = new CloudReplica(testReplicaId1, testBackendId1, Replica.ReplicaState.NORMAL, version, + /*schemaHash*/ 0, dbId, tableId, partitionId, indexId, /*idx*/ 0); + replica2 = new CloudReplica(testReplicaId2, testBackendId2, Replica.ReplicaState.NORMAL, version, + 0, dbId, tableId, partitionId, indexId, 1); + replica3 = new CloudReplica(testReplicaId3, testBackendId3, Replica.ReplicaState.NORMAL, version, + 0, dbId, tableId, partitionId, indexId, 2); + } else { + replica1 = new Replica(testReplicaId1, testBackendId1, version, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + replica2 = new Replica(testReplicaId2, testBackendId2, version, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + replica3 = new Replica(testReplicaId3, testBackendId3, version, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + } + // tablet Tablet tablet = new Tablet(tabletId); @@ -244,10 +258,14 @@ public static Database createSimpleDb(long dbId, long tableId, long partitionId, } public static void createDupTable(Database db) { - - // replica - Replica replica = new Replica(testReplicaId4, testBackendId1, testStartVersion, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); + Replica replica; + if (Config.isCloudMode()) { + replica = new CloudReplica(testReplicaId4, testBackendId1, Replica.ReplicaState.NORMAL, testStartVersion, + 0, db.getId(), testTableId2, testPartitionId2, testIndexId2, 0); + } else { + replica = new Replica(testReplicaId4, testBackendId1, testStartVersion, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + } // tablet Tablet tablet = new Tablet(testTabletId2); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEditLog.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEditLog.java index 08800510a1a6ef..e2b113958366c8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEditLog.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEditLog.java @@ -100,6 +100,16 @@ public void logModifyDistributionType(TableInfo tableInfo) { } + @Mock + public void logAddBackend(Backend be) { + // do nothing for test + } + + @Mock + public int getNumEditStreams() { + return 1; // fake that we have streams + } + public TransactionState getTransaction(long transactionId) { return allTransactionState.get(transactionId); } diff --git a/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out index 6f916a99c91705..49570b96d291f2 100644 --- a/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out +++ b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out @@ -1,5 +1,5 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !select -- +-- !select_light_mode_init -- 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South 1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East @@ -11,7 +11,7 @@ 1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North 1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South --- !select -- +-- !select_light_mode_more_data -- 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South @@ -33,7 +33,7 @@ 1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South 1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South --- !select -- +-- !select_schema_change_mode_init -- 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South 1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East @@ -45,7 +45,41 @@ 1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North 1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South --- !select -- +-- !select_schema_change_mode_more_data -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select_lifecycle_after_data -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select_lifecycle_final -- 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1001 2023-10-06T15:00 Laptop John Smith 199.99 North 1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South diff --git a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy index 6fea7a68f92b8e..402f72ddc36065 100644 --- a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy +++ b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy @@ -54,13 +54,16 @@ suite("test_ngram_bloomfilter_index_change") { sql "sync" } - // Test setup - // 1. Create table - // 2. Insert test data - // 3. Add NGRAM Bloom Filter index - // 4. Build index - // 5. Insert more data - // 6. Drop index + // Test settings + sql "set enable_function_pushdown=true" + sql "set enable_profile=true" + sql "set profile_level=2" + + // Define test query + def query = "SELECT /*+SET_VAR(enable_function_pushdown = true, enable_profile = true, profile_level = 2)*/ * FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id" + // Test Case 1: Test with enable_add_index_for_new_data = true + logger.info("=== Test Case 1: enable_add_index_for_new_data = true ===") + // Create table sql "DROP TABLE IF EXISTS ${tableName}" sql """ CREATE TABLE ${tableName} ( @@ -82,29 +85,20 @@ suite("test_ngram_bloomfilter_index_change") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false" + "disable_auto_compaction" = "true" ); """ - // Insert first batch of data + // Insert test data insertTestData() - - // Test settings - sql "set enable_function_pushdown=true" - sql "set enable_profile=true" - sql "set profile_level=2" - // Verify data loaded correctly - qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + qt_select_light_mode_init "SELECT * FROM ${tableName} ORDER BY sale_id" - // Define test query - def query = "SELECT /*+SET_VAR(enable_function_pushdown = true, enable_profile = true, profile_level = 2)*/ * FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id" - - // Test 1: without NGRAM Bloom Filter index - profile("sql_select_like_without_ngram_index") { + // Test without NGRAM Bloom Filter index + profile("sql_select_like_without_ngram_index_light_mode") { run { - sql "/* sql_select_like_without_ngram_index */ ${query}" - sleep(1000) // sleep 1s wait for the profile collection to be completed + sql "/* sql_select_like_without_ngram_index_light_mode */ ${query}" + sleep(1000) } check { profileString, exception -> @@ -112,13 +106,41 @@ suite("test_ngram_bloomfilter_index_change") { assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) } } + sql "set enable_add_index_for_new_data = true" - // Test 2: After adding NGRAM Bloom Filter index + // Add NGRAM Bloom Filter index (should be immediate in light mode) sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" + + // In light mode, the index should be effective immediately, no need to wait for alter job + // But let's give it a moment to ensure metadata is updated + sleep(2000) + + // Insert more data after index added + insertTestData() + // Verify more data loaded correctly + qt_select_light_mode_more_data "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test with more data (should still filter correctly) + profile("sql_select_like_with_ngram_index_light_mode_more_data") { + run { + sql "/* sql_select_like_with_ngram_index_light_mode_more_data */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 10")) + } + } + + // Drop index + sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" wait_for_latest_op_on_table_finish(tableName, timeout) - profile("sql_select_like_with_ngram_index_added") { + + // Test after dropping index + profile("sql_select_like_with_ngram_index_light_mode_dropped") { run { - sql "/* sql_select_like_with_ngram_index_added */ ${query}" + sql "/* sql_select_like_with_ngram_index_light_mode_dropped */ ${query}" sleep(1000) } @@ -128,12 +150,61 @@ suite("test_ngram_bloomfilter_index_change") { } } - // Test 3: After building the index - sql "BUILD INDEX idx_ngram_customer_name ON ${tableName};" + // Test Case 2: Test with enable_add_index_for_new_data = false (schema change mode) + logger.info("=== Test Case 2: enable_add_index_for_new_data = false ===") + // Set enable_add_index_for_new_data = false + sql "set enable_add_index_for_new_data = false" + // Create new table + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + `sale_id` int NULL, + `sale_date` datetime NULL, + `product_name` varchar(100) NULL, + `customer_name` varchar(100) NULL, + `amount` decimal(10,2) NULL, + `region` char(50) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`sale_id`) + PARTITION BY RANGE(`sale_date`) ( + PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01 00:00:00')), + PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01 00:00:00')), + PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00')) + ) + DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "true" + ); + """ + // Insert test data + insertTestData() + // Verify data loaded correctly + qt_select_schema_change_mode_init "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test without NGRAM Bloom Filter index + profile("sql_select_like_without_ngram_index_schema_change_mode") { + run { + sql "/* sql_select_like_without_ngram_index_schema_change_mode */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } + + // Add NGRAM Bloom Filter index (will trigger schema change in this mode) + sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" wait_for_latest_op_on_table_finish(tableName, timeout) - profile("sql_select_like_with_ngram_index_built") { + + // Test after adding NGRAM Bloom Filter index (should filter existing data) + profile("sql_select_like_with_ngram_index_schema_change_mode_added") { run { - sql "/* sql_select_like_with_ngram_index_built */ ${query}" + sql "/* sql_select_like_with_ngram_index_schema_change_mode_added */ ${query}" sleep(1000) } @@ -143,15 +214,15 @@ suite("test_ngram_bloomfilter_index_change") { } } - // Insert second batch of data + // Insert more data after index is built insertTestData() - // Verify data loaded correctly - qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + // Verify more data loaded correctly + qt_select_schema_change_mode_more_data "SELECT * FROM ${tableName} ORDER BY sale_id" - // Test 4: Verify filtering with more data - profile("sql_select_like_with_ngram_index_more_data") { + // Test with more data (should filter all data) + profile("sql_select_like_with_ngram_index_schema_change_mode_more_data") { run { - sql "/* sql_select_like_with_ngram_index_more_data */ ${query}" + sql "/* sql_select_like_with_ngram_index_schema_change_mode_more_data */ ${query}" sleep(1000) } @@ -161,12 +232,14 @@ suite("test_ngram_bloomfilter_index_change") { } } - // Test 5: After dropping the index + // Drop index sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" wait_for_latest_op_on_table_finish(tableName, timeout) - profile("sql_select_like_with_ngram_index_dropped") { + + // Test after dropping index + profile("sql_select_like_with_ngram_index_schema_change_mode_dropped") { run { - sql "/* sql_select_like_with_ngram_index_dropped */ ${query}" + sql "/* sql_select_like_with_ngram_index_schema_change_mode_dropped */ ${query}" sleep(1000) } @@ -176,13 +249,11 @@ suite("test_ngram_bloomfilter_index_change") { } } - // recreate table - // 1. Create table - // 2. Add NGRAM Bloom Filter index - // 3. Insert data - // 4. Insert more data - // 5. Build index - // 6. Drop index + // Test Case 3: Test different scenarios for index lifecycle + logger.info("=== Test Case 3: Index lifecycle with enable_add_index_for_new_data = true ===") + // Set enable_add_index_for_new_data = true + sql "set enable_add_index_for_new_data = true" + // Create table and add index before inserting data sql "DROP TABLE IF EXISTS ${tableName}" sql """ CREATE TABLE ${tableName} ( @@ -204,24 +275,23 @@ suite("test_ngram_bloomfilter_index_change") { "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2", "light_schema_change" = "true", - "disable_auto_compaction" = "false" + "disable_auto_compaction" = "true" ); """ - // add ngram bf index + // Add ngram bf index before data insertion sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" wait_for_latest_op_on_table_finish(tableName, timeout) - // insert data + // Insert data after index creation insertTestData() - // Verify data loaded correctly - qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" + qt_select_lifecycle_after_data "SELECT * FROM ${tableName} ORDER BY sale_id" - // Test 6: Verify filtering with index added - profile("sql_select_like_with_ngram_index_recreated") { + // Test filtering with index added before data insertion + profile("sql_select_like_with_ngram_index_lifecycle_test") { run { - sql "/* sql_select_like_with_ngram_index_recreated */ ${query}" + sql "/* sql_select_like_with_ngram_index_lifecycle_test */ ${query}" sleep(1000) } @@ -231,33 +301,15 @@ suite("test_ngram_bloomfilter_index_change") { } } - // insert more data + // Insert more data insertTestData() - - // Verify data loaded correctly - qt_select "SELECT * FROM ${tableName} ORDER BY sale_id" - - // Test 7: Verify filtering with more data - profile("sql_select_like_with_ngram_index_recreated_more_data") { - run { - sql "/* sql_select_like_with_ngram_index_recreated_more_data */ ${query}" - sleep(1000) - } - - check { profileString, exception -> - log.info(profileString) - assertTrue(profileString.contains("RowsBloomFilterFiltered: 20")) - } - } - - // build index - sql "BUILD INDEX idx_ngram_customer_name ON ${tableName};" - wait_for_latest_op_on_table_finish(tableName, timeout) + // Verify more data loaded correctly + qt_select_lifecycle_final "SELECT * FROM ${tableName} ORDER BY sale_id" - // Test 8: Verify filtering with index built - profile("sql_select_like_with_ngram_index_recreated_built") { + // Test filtering with more data + profile("sql_select_like_with_ngram_index_lifecycle_final") { run { - sql "/* sql_select_like_with_ngram_index_recreated_built */ ${query}" + sql "/* sql_select_like_with_ngram_index_lifecycle_final */ ${query}" sleep(1000) } @@ -267,20 +319,7 @@ suite("test_ngram_bloomfilter_index_change") { } } - // drop index + // Final cleanup sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" - wait_for_latest_op_on_table_finish(tableName, timeout) - - // Test 9: Verify filtering with index dropped - profile("sql_select_like_with_ngram_index_recreated_dropped") { - run { - sql "/* sql_select_like_with_ngram_index_recreated_dropped */ ${query}" - sleep(1000) - } - - check { profileString, exception -> - log.info(profileString) - assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) - } - } + sleep(2000) } \ No newline at end of file