diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index d52ddf49430f7e..ca6268c5b03dab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -68,6 +68,7 @@ import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.PropertyAnalyzer.RewriteProperty; import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; import org.apache.doris.persist.AlterMTMV; import org.apache.doris.persist.AlterViewInfo; @@ -176,7 +177,8 @@ private boolean processAlterOlapTable(AlterTableStmt stmt, OlapTable olapTable, olapTable.checkNormalStateForAlter(); boolean needProcessOutsideTableLock = false; - String oldTableName = olapTable.getName(); + BaseTableInfo oldBaseTableInfo = new BaseTableInfo(olapTable); + Optional newBaseTableInfo = Optional.empty(); if (currentAlterOps.checkTableStoragePolicy(alterClauses)) { String tableStoragePolicy = olapTable.getStoragePolicy(); String currentStoragePolicy = currentAlterOps.getTableStoragePolicy(alterClauses); @@ -283,8 +285,14 @@ private boolean processAlterOlapTable(AlterTableStmt stmt, OlapTable olapTable, } } else if (currentAlterOps.hasRenameOp()) { processRename(db, olapTable, alterClauses); + newBaseTableInfo = Optional.of(new BaseTableInfo(olapTable)); } else if (currentAlterOps.hasReplaceTableOp()) { processReplaceTable(db, olapTable, alterClauses); + // after replace table, olapTable may still be old name, so need set it to new name + ReplaceTableClause clause = (ReplaceTableClause) alterClauses.get(0); + String newTblName = clause.getTblName(); + newBaseTableInfo = Optional.of(new BaseTableInfo(olapTable)); + newBaseTableInfo.get().setTableName(newTblName); } else if (currentAlterOps.contains(AlterOpType.MODIFY_TABLE_PROPERTY_SYNC)) { needProcessOutsideTableLock = true; } else if (currentAlterOps.contains(AlterOpType.MODIFY_DISTRIBUTION)) { @@ -302,7 +310,8 @@ private boolean processAlterOlapTable(AlterTableStmt stmt, OlapTable olapTable, throw new DdlException("Invalid alter operations: " + currentAlterOps); } if (needChangeMTMVState(alterClauses)) { - Env.getCurrentEnv().getMtmvService().alterTable(olapTable, oldTableName); + Env.getCurrentEnv().getMtmvService() + .alterTable(oldBaseTableInfo, newBaseTableInfo, currentAlterOps.hasReplaceTableOp()); } return needProcessOutsideTableLock; } @@ -661,11 +670,9 @@ private void replaceTableInternal(Database db, OlapTable origTable, OlapTable ne // drop origin table and new table db.unregisterTable(oldTblName); db.unregisterTable(newTblName); - // rename new table name to origin table name and add it to database newTbl.checkAndSetName(oldTblName, false); db.registerTable(newTbl); - if (swapTable) { // rename origin table name to new table name and add it to database origTable.checkAndSetName(newTblName, false); @@ -674,7 +681,12 @@ private void replaceTableInternal(Database db, OlapTable origTable, OlapTable ne // not swap, the origin table is not used anymore, need to drop all its tablets. Env.getCurrentEnv().onEraseOlapTable(origTable, isReplay); if (origTable.getType() == TableType.MATERIALIZED_VIEW) { - Env.getCurrentEnv().getMtmvService().deregisterMTMV((MTMV) origTable); + // Because the current dropMTMV will delete jobs related to materialized views, + // this method will maintain its own metadata for deleting jobs, + // so it cannot be called during playback + if (!isReplay) { + Env.getCurrentEnv().getMtmvService().dropMTMV((MTMV) origTable); + } } Env.getCurrentEnv().getAnalysisManager().removeTableStats(origTable.getId()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java index bb7b540a1312aa..a5d8d5e9665514 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java @@ -412,10 +412,7 @@ public Pair createTableWithLock( result = setIfNotExist; isTableExist = true; } else { - idToTable.put(table.getId(), table); - nameToTable.put(table.getName(), table); - lowerCaseToTableName.put(tableName.toLowerCase(), tableName); - + registerTable(table); if (!isReplay) { // Write edit log CreateTableInfo info = new CreateTableInfo(fullQualifiedName, table); @@ -423,8 +420,6 @@ public Pair createTableWithLock( } if (table.getType() == TableType.ELASTICSEARCH) { Env.getCurrentEnv().getEsRepository().registerTable((EsTable) table); - } else if (table.getType() == TableType.MATERIALIZED_VIEW) { - Env.getCurrentEnv().getMtmvService().registerMTMV((MTMV) table, id); } } return Pair.of(result, isTableExist); @@ -448,6 +443,9 @@ public boolean registerTable(TableIf table) { idToTable.put(olapTable.getId(), olapTable); nameToTable.put(olapTable.getName(), olapTable); lowerCaseToTableName.put(tableName.toLowerCase(), tableName); + if (olapTable instanceof MTMV) { + Env.getCurrentEnv().getMtmvService().registerMTMV((MTMV) olapTable, id); + } } olapTable.unmarkDropped(); return result; @@ -459,6 +457,9 @@ public void unregisterTable(String tableName) { } Table table = getTableNullable(tableName); if (table != null) { + if (table instanceof MTMV) { + Env.getCurrentEnv().getMtmvService().unregisterMTMV((MTMV) table); + } this.nameToTable.remove(tableName); this.idToTable.remove(table.getId()); this.lowerCaseToTableName.remove(tableName.toLowerCase()); @@ -638,14 +639,7 @@ private void readTables(DataInput in) throws IOException { int numTables = in.readInt(); for (int i = 0; i < numTables; ++i) { Table table = Table.read(in); - table.setQualifiedDbName(fullQualifiedName); - if (table instanceof MTMV) { - Env.getCurrentEnv().getMtmvService().registerMTMV((MTMV) table, id); - } - String tableName = table.getName(); - nameToTable.put(tableName, table); - idToTable.put(table.getId(), table); - lowerCaseToTableName.put(tableName.toLowerCase(), tableName); + registerTable(table); } } @@ -654,12 +648,7 @@ public void gsonPostProcess() throws IOException { Preconditions.checkState(nameToTable.getClass() == ConcurrentHashMap.class, "nameToTable should be ConcurrentMap"); nameToTable.forEach((tn, tb) -> { - tb.setQualifiedDbName(fullQualifiedName); - if (tb instanceof MTMV) { - Env.getCurrentEnv().getMtmvService().registerMTMV((MTMV) tb, id); - } - idToTable.put(tb.getId(), tb); - lowerCaseToTableName.put(tn.toLowerCase(), tn); + registerTable(tb); }); if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_105) { @@ -733,14 +722,7 @@ public void readFields(DataInput in) throws IOException { int numTables = in.readInt(); for (int i = 0; i < numTables; ++i) { Table table = Table.read(in); - table.setQualifiedDbName(fullQualifiedName); - if (table instanceof MTMV) { - Env.getCurrentEnv().getMtmvService().registerMTMV((MTMV) table, id); - } - String tableName = table.getName(); - nameToTable.put(tableName, table); - idToTable.put(table.getId(), table); - lowerCaseToTableName.put(tableName.toLowerCase(), tableName); + registerTable(table); } // read quota diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 1da6ffacf8ae1e..49598e17c26a84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -4832,6 +4832,14 @@ public void renameTable(Database db, Table table, String newTableName) throws Dd throw new DdlException("Table name[" + newTableName + "] is already used (in restoring)"); } + if (table.isManagedTable()) { + // If not checked first, execute db.unregisterTable first, + // and then check the name in setName, it cannot guarantee atomicity + ((OlapTable) table).checkAndSetName(newTableName, true); + } + + db.unregisterTable(oldTableName); + if (table.isManagedTable()) { // olap table should also check if any rollup has same name as "newTableName" ((OlapTable) table).checkAndSetName(newTableName, false); @@ -4839,9 +4847,7 @@ public void renameTable(Database db, Table table, String newTableName) throws Dd table.setName(newTableName); } - db.unregisterTable(oldTableName); db.registerTable(table); - TableInfo tableInfo = TableInfo.createForTableRename(db.getId(), table.getId(), oldTableName, newTableName); editLog.logTableRename(tableInfo); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index e41a845d969dba..3657915ae4fbc9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1046,9 +1046,6 @@ public boolean unprotectDropTable(Database db, Table table, boolean isForceDrop, // which make things easier. ((OlapTable) table).dropAllTempPartitions(); } - if (table.getType() == TableType.MATERIALIZED_VIEW) { - Env.getCurrentEnv().getMtmvService().deregisterMTMV((MTMV) table); - } Env.getCurrentEnv().getAnalysisManager().removeTableStats(table.getId()); db.unregisterTable(table.getName()); StopWatch watch = StopWatch.createStarted(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 0437a271008aa8..198ba21891832b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; @@ -44,6 +45,7 @@ import org.apache.doris.mtmv.MTMVPartitionUtil; import org.apache.doris.mtmv.MTMVPlanUtil; import org.apache.doris.mtmv.MTMVRefreshContext; +import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState; import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshMethod; import org.apache.doris.mtmv.MTMVRefreshPartitionSnapshot; import org.apache.doris.mtmv.MTMVRelation; @@ -51,6 +53,7 @@ import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.trees.plans.commands.UpdateMvByPartitionCommand; +import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; import org.apache.doris.qe.AuditLogHelper; import org.apache.doris.qe.ConnectContext; @@ -81,6 +84,7 @@ import java.util.Objects; import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; public class MTMVTask extends AbstractTask { private static final Logger LOG = LogManager.getLogger(MTMVTask.class); @@ -193,6 +197,11 @@ public void run() throws JobException { // lock table order by id to avoid deadlock MetaLockUtils.readLockTables(tableIfs); try { + // if mtmv is schema_change, check if column type has changed + // If it's not in the schema_change state, the column type definitely won't change. + if (MTMVState.SCHEMA_CHANGE.equals(mtmv.getStatus().getState())) { + checkColumnTypeIfChange(mtmv, ctx); + } if (mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE) { MTMVPartitionUtil.alignMvPartition(mtmv); } @@ -239,6 +248,39 @@ public void run() throws JobException { } } + private void checkColumnTypeIfChange(MTMV mtmv, ConnectContext ctx) throws JobException { + List currentColumnsDefinition = MTMVPlanUtil.generateColumnsBySql(mtmv.getQuerySql(), ctx, + mtmv.getMvPartitionInfo().getPartitionCol(), + mtmv.getDistributionColumnNames(), null, mtmv.getTableProperty().getProperties()); + List currentColumns = currentColumnsDefinition.stream() + .map(ColumnDefinition::translateToCatalogStyle) + .collect(Collectors.toList()); + List originalColumns = mtmv.getBaseSchema(true); + if (currentColumns.size() != originalColumns.size()) { + throw new JobException(String.format( + "column length not equals, please check whether columns of base table have changed, " + + "original length is: %s, current length is: %s", + originalColumns.size(), currentColumns.size())); + } + for (int i = 0; i < originalColumns.size(); i++) { + if (!isTypeLike(originalColumns.get(i).getType(), currentColumns.get(i).getType())) { + throw new JobException(String.format( + "column type not same, please check whether columns of base table have changed, " + + "column name is: %s, original type is: %s, current type is: %s", + originalColumns.get(i).getName(), originalColumns.get(i).getType().toSql(), + currentColumns.get(i).getType().toSql())); + } + } + } + + private boolean isTypeLike(Type type, Type typeOther) { + if (type.isStringType()) { + return typeOther.isStringType(); + } else { + return type.equals(typeOther); + } + } + private void executeWithRetry(Set execPartitionNames, Map tableWithPartKey) throws Exception { int retryCount = 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVHookService.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVHookService.java index e0edd06f8c9418..0f2b04147e78b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVHookService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVHookService.java @@ -29,6 +29,8 @@ import org.apache.doris.nereids.trees.plans.commands.info.ResumeMTMVInfo; import org.apache.doris.persist.AlterMTMV; +import java.util.Optional; + /** * Contains all operations that affect the mtmv */ @@ -63,7 +65,7 @@ public interface MTMVHookService { * * @param mtmv */ - void deregisterMTMV(MTMV mtmv); + void unregisterMTMV(MTMV mtmv); /** * triggered when alter mtmv, only once @@ -102,9 +104,11 @@ public interface MTMVHookService { /** * Triggered when baseTable is altered * - * @param table + * @param oldTableInfo info before alter + * @param newTableInfo info after alter + * @param isReplace */ - void alterTable(Table table, String oldTableName); + void alterTable(BaseTableInfo oldTableInfo, Optional newTableInfo, boolean isReplace); /** * Triggered when pause mtmv diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java index 7ea3d9dc05f9f1..752ff78fdafb70 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java @@ -52,6 +52,7 @@ import org.apache.logging.log4j.Logger; import java.util.List; +import java.util.Optional; /** * when do some operation, do something about job @@ -144,7 +145,7 @@ public void registerMTMV(MTMV mtmv, Long dbId) { } @Override - public void deregisterMTMV(MTMV mtmv) { + public void unregisterMTMV(MTMV mtmv) { } @@ -189,7 +190,7 @@ public void dropTable(Table table) { } @Override - public void alterTable(Table table, String oldTableName) { + public void alterTable(BaseTableInfo oldTableInfo, Optional newTableInfo, boolean isReplace) { } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index 8659ea4b287561..a03011e6e8a473 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -19,11 +19,15 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.common.FeNameFormat; +import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.mysql.privilege.Auth; import org.apache.doris.nereids.NereidsPlanner; @@ -33,14 +37,33 @@ import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; +import org.apache.doris.nereids.trees.plans.commands.info.SimpleColumnDefinition; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.types.AggStateType; +import org.apache.doris.nereids.types.CharType; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DecimalV2Type; +import org.apache.doris.nereids.types.NullType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TinyIntType; +import org.apache.doris.nereids.types.VarcharType; +import org.apache.doris.nereids.types.coercion.CharacterType; +import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.apache.commons.collections.CollectionUtils; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import javax.annotation.Nullable; @@ -148,4 +171,153 @@ public static Set getBaseTableFromQuery(String querySql, ConnectContext } } } + + /** + * Derive the MTMV columns based on the query statement. + * + * @param querySql + * @param ctx + * @param partitionCol partition column name of MTMV + * @param distributionColumnNames distribution column names of MTMV + * @param simpleColumnDefinitions Use custom column names if provided (non-empty); + * otherwise, auto-generate the column names. + * @param properties properties of MTMV, it determines whether row storage needs to be generated based on this. + * @return ColumnDefinitions of MTMV + */ + public static List generateColumnsBySql(String querySql, ConnectContext ctx, String partitionCol, + Set distributionColumnNames, List simpleColumnDefinitions, + Map properties) { + List statements; + try { + statements = new NereidsParser().parseSQL(querySql); + } catch (Exception e) { + throw new ParseException("Nereids parse failed. " + e.getMessage()); + } + StatementBase parsedStmt = statements.get(0); + LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); + StatementContext original = ctx.getStatementContext(); + try (StatementContext tempCtx = new StatementContext()) { + ctx.setStatementContext(tempCtx); + try { + NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); + Plan plan = planner.planWithLock(logicalPlan, PhysicalProperties.ANY, ExplainLevel.ANALYZED_PLAN); + return generateColumns(plan, ctx, partitionCol, distributionColumnNames, simpleColumnDefinitions, + properties); + } finally { + ctx.setStatementContext(original); + } + } + } + + /** + * Derive the MTMV columns based on the analyzed plan. + * + * @param plan should be analyzed plan + * @param ctx + * @param partitionCol partition column name of MTMV + * @param distributionColumnNames distribution column names of MTMV + * @param simpleColumnDefinitions Use custom column names if provided (non-empty); + * otherwise, auto-generate the column names. + * @param properties properties of MTMV, it determines whether row storage needs to be generated based on this. + * @return ColumnDefinitions of MTMV + */ + public static List generateColumns(Plan plan, ConnectContext ctx, String partitionCol, + Set distributionColumnNames, List simpleColumnDefinitions, + Map properties) { + List columns = Lists.newArrayList(); + List slots = plan.getOutput(); + if (slots.isEmpty()) { + throw new org.apache.doris.nereids.exceptions.AnalysisException("table should contain at least one column"); + } + if (!CollectionUtils.isEmpty(simpleColumnDefinitions) && simpleColumnDefinitions.size() != slots.size()) { + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "simpleColumnDefinitions size is not equal to the query's"); + } + Set colNames = Sets.newHashSet(); + for (int i = 0; i < slots.size(); i++) { + String colName = CollectionUtils.isEmpty(simpleColumnDefinitions) ? slots.get(i).getName() + : simpleColumnDefinitions.get(i).getName(); + try { + FeNameFormat.checkColumnName(colName); + } catch (org.apache.doris.common.AnalysisException e) { + throw new org.apache.doris.nereids.exceptions.AnalysisException(e.getMessage(), e); + } + if (colNames.contains(colName)) { + throw new org.apache.doris.nereids.exceptions.AnalysisException("repeat cols:" + colName); + } else { + colNames.add(colName); + } + DataType dataType = getDataType(slots.get(i), i, ctx, partitionCol, distributionColumnNames); + // If datatype is AggStateType, AggregateType should be generic, or column definition check will fail + columns.add(new ColumnDefinition( + colName, + dataType, + false, + slots.get(i).getDataType() instanceof AggStateType ? AggregateType.GENERIC : null, + slots.get(i).nullable(), + Optional.empty(), + CollectionUtils.isEmpty(simpleColumnDefinitions) ? null + : simpleColumnDefinitions.get(i).getComment())); + } + // add a hidden column as row store + if (properties != null) { + try { + boolean storeRowColumn = + PropertyAnalyzer.analyzeStoreRowColumn(Maps.newHashMap(properties)); + if (storeRowColumn) { + columns.add(ColumnDefinition.newRowStoreColumnDefinition(null)); + } + } catch (Exception e) { + throw new org.apache.doris.nereids.exceptions.AnalysisException(e.getMessage(), e.getCause()); + } + } + return columns; + } + + /** + * generate DataType by Slot + * @param s + * @param i + * @param ctx + * @param partitionCol + * @param distributionColumnNames + * @return + */ + public static DataType getDataType(Slot s, int i, ConnectContext ctx, String partitionCol, + Set distributionColumnNames) { + DataType dataType = s.getDataType().conversion(); + // first column can not be TEXT, should transfer to varchar + if (i == 0 && dataType.isStringType()) { + dataType = VarcharType.createVarcharType(ScalarType.MAX_VARCHAR_LENGTH); + } else { + dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, + NullType.class, TinyIntType.INSTANCE); + dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, + DecimalV2Type.class, DecimalV2Type.SYSTEM_DEFAULT); + if (s.isColumnFromTable()) { + // check if external table + if ((!((SlotReference) s).getTable().isPresent() + || !((SlotReference) s).getTable().get().isManagedTable())) { + if (s.getName().equals(partitionCol) || distributionColumnNames.contains(s.getName())) { + // String type can not be used in partition/distributed column + // so we replace it to varchar + dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, + CharacterType.class, VarcharType.MAX_VARCHAR_TYPE); + } else { + dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, + CharacterType.class, StringType.INSTANCE); + } + } + } else { + if (ctx.getSessionVariable().useMaxLengthOfVarcharInCtas) { + // The calculation of columns that are not from the original table will become VARCHAR(65533) + dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, + VarcharType.class, VarcharType.MAX_VARCHAR_TYPE); + dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, + CharType.class, VarcharType.MAX_VARCHAR_TYPE); + } + } + } + return dataType; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java index 125b8ce38deecd..7858aa952777c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java @@ -46,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.function.BiPredicate; @@ -213,7 +214,7 @@ public void registerMTMV(MTMV mtmv, Long dbId) { * @param mtmv */ @Override - public void deregisterMTMV(MTMV mtmv) { + public void unregisterMTMV(MTMV mtmv) { removeMTMV(new BaseTableInfo(mtmv)); } @@ -255,17 +256,15 @@ public void dropTable(Table table) { /** * update mtmv status to `SCHEMA_CHANGE` * - * @param table + * @param isReplace */ @Override - public void alterTable(Table table, String oldTableName) { - BaseTableInfo baseTableInfo = new BaseTableInfo(table); - baseTableInfo.setTableName(oldTableName); - if (table instanceof MTMV) { - removeMTMV(baseTableInfo); - refreshMTMVCache(((MTMV) table).getRelation(), new BaseTableInfo(table)); + public void alterTable(BaseTableInfo oldTableInfo, Optional newTableInfo, boolean isReplace) { + // when replace, need deal two table + if (isReplace) { + processBaseTableChange(newTableInfo.get(), "The base table has been updated:"); } - processBaseTableChange(baseTableInfo, "The base table has been updated:"); + processBaseTableChange(oldTableInfo, "The base table has been updated:"); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java index 2a9b1618df803d..2ee42c0aa2c833 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java @@ -45,6 +45,7 @@ import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; public class MTMVService implements EventListener { @@ -76,23 +77,23 @@ public void deregisterHook(String name) { } public void registerMTMV(MTMV mtmv, Long dbId) { - Objects.requireNonNull(mtmv); + Objects.requireNonNull(mtmv, "mtmv can not be null"); LOG.info("registerMTMV: " + mtmv.getName()); for (MTMVHookService mtmvHookService : hooks.values()) { mtmvHookService.registerMTMV(mtmv, dbId); } } - public void deregisterMTMV(MTMV mtmv) { - Objects.requireNonNull(mtmv); + public void unregisterMTMV(MTMV mtmv) { + Objects.requireNonNull(mtmv, "mtmv can not be null"); LOG.info("deregisterMTMV: " + mtmv.getName()); for (MTMVHookService mtmvHookService : hooks.values()) { - mtmvHookService.deregisterMTMV(mtmv); + mtmvHookService.unregisterMTMV(mtmv); } } public void createMTMV(MTMV mtmv) throws DdlException, AnalysisException { - Objects.requireNonNull(mtmv); + Objects.requireNonNull(mtmv, "mtmv can not be null"); LOG.info("createMTMV: " + mtmv.getName()); for (MTMVHookService mtmvHookService : hooks.values()) { mtmvHookService.createMTMV(mtmv); @@ -100,7 +101,7 @@ public void createMTMV(MTMV mtmv) throws DdlException, AnalysisException { } public void dropMTMV(MTMV mtmv) throws DdlException { - Objects.requireNonNull(mtmv); + Objects.requireNonNull(mtmv, "mtmv can not be null"); LOG.info("dropMTMV: " + mtmv.getName()); for (MTMVHookService mtmvHookService : hooks.values()) { mtmvHookService.dropMTMV(mtmv); @@ -108,8 +109,8 @@ public void dropMTMV(MTMV mtmv) throws DdlException { } public void alterMTMV(MTMV mtmv, AlterMTMV alterMTMV) throws DdlException { - Objects.requireNonNull(mtmv); - Objects.requireNonNull(alterMTMV); + Objects.requireNonNull(mtmv, "mtmv can not be null"); + Objects.requireNonNull(alterMTMV, "alterMTMV can not be null"); LOG.info("alterMTMV, mtmvName: {}, AlterMTMV: {}", mtmv.getName(), alterMTMV); for (MTMVHookService mtmvHookService : hooks.values()) { mtmvHookService.alterMTMV(mtmv, alterMTMV); @@ -117,7 +118,7 @@ public void alterMTMV(MTMV mtmv, AlterMTMV alterMTMV) throws DdlException { } public void refreshMTMV(RefreshMTMVInfo info) throws DdlException, MetaNotFoundException, JobException { - Objects.requireNonNull(info); + Objects.requireNonNull(info, "info can not be null"); LOG.info("refreshMTMV, RefreshMTMVInfo: {}", info); for (MTMVHookService mtmvHookService : hooks.values()) { mtmvHookService.refreshMTMV(info); @@ -125,24 +126,26 @@ public void refreshMTMV(RefreshMTMVInfo info) throws DdlException, MetaNotFoundE } public void dropTable(Table table) { - Objects.requireNonNull(table); + Objects.requireNonNull(table, "table can not be null"); LOG.info("dropTable, tableName: {}", table.getName()); for (MTMVHookService mtmvHookService : hooks.values()) { mtmvHookService.dropTable(table); } } - public void alterTable(Table table, String oldTableName) { - Objects.requireNonNull(table); - LOG.info("alterTable, tableName: {}", table.getName()); + public void alterTable(BaseTableInfo oldTableInfo, Optional newTableInfo, boolean isReplace) { + Objects.requireNonNull(oldTableInfo, "oldTableInfo can not be null"); + Objects.requireNonNull(newTableInfo, "newTableInfo can not be null"); + LOG.info("alterTable, oldTableInfo: {}, newTableInfo: {}, isReplace: {}", oldTableInfo, newTableInfo, + isReplace); for (MTMVHookService mtmvHookService : hooks.values()) { - mtmvHookService.alterTable(table, oldTableName); + mtmvHookService.alterTable(oldTableInfo, newTableInfo, isReplace); } } public void refreshComplete(MTMV mtmv, MTMVRelation cache, MTMVTask task) { - Objects.requireNonNull(mtmv); - Objects.requireNonNull(task); + Objects.requireNonNull(mtmv, "mtmv can not be null"); + Objects.requireNonNull(task, "task can not be null"); LOG.info("refreshComplete: " + mtmv.getName()); for (MTMVHookService mtmvHookService : hooks.values()) { mtmvHookService.refreshComplete(mtmv, cache, task); @@ -150,7 +153,7 @@ public void refreshComplete(MTMV mtmv, MTMVRelation cache, MTMVTask task) { } public void pauseMTMV(PauseMTMVInfo info) throws DdlException, MetaNotFoundException, JobException { - Objects.requireNonNull(info); + Objects.requireNonNull(info, "info can not be null"); LOG.info("pauseMTMV, PauseMTMVInfo: {}", info); for (MTMVHookService mtmvHookService : hooks.values()) { mtmvHookService.pauseMTMV(info); @@ -158,7 +161,7 @@ public void pauseMTMV(PauseMTMVInfo info) throws DdlException, MetaNotFoundExcep } public void resumeMTMV(ResumeMTMVInfo info) throws MetaNotFoundException, DdlException, JobException { - Objects.requireNonNull(info); + Objects.requireNonNull(info, "info can not be null"); LOG.info("resumeMTMV, ResumeMTMVInfo: {}", info); for (MTMVHookService mtmvHookService : hooks.values()) { mtmvHookService.resumeMTMV(info); @@ -166,7 +169,7 @@ public void resumeMTMV(ResumeMTMVInfo info) throws MetaNotFoundException, DdlExc } public void cancelMTMVTask(CancelMTMVTaskInfo info) throws MetaNotFoundException, DdlException, JobException { - Objects.requireNonNull(info); + Objects.requireNonNull(info, "info can not be null"); LOG.info("cancelMTMVTask, CancelMTMVTaskInfo: {}", info); for (MTMVHookService mtmvHookService : hooks.values()) { mtmvHookService.cancelMTMVTask(info); @@ -175,7 +178,7 @@ public void cancelMTMVTask(CancelMTMVTaskInfo info) throws MetaNotFoundException @Override public void processEvent(Event event) throws EventException { - Objects.requireNonNull(event); + Objects.requireNonNull(event, "event can not be null"); if (!(event instanceof TableEvent)) { return; } @@ -199,7 +202,7 @@ public void processEvent(Event event) throws EventException { try { // check if mtmv should trigger by event MTMV mtmv = (MTMV) MTMVUtil.getTable(baseTableInfo); - if (canRefresh(mtmv, table)) { + if (shouldRefreshOnBaseTableDataChange(mtmv, table)) { jobManager.onCommit(mtmv); } } catch (Exception e) { @@ -208,7 +211,7 @@ public void processEvent(Event event) throws EventException { } } - private boolean canRefresh(MTMV mtmv, TableIf table) { + private boolean shouldRefreshOnBaseTableDataChange(MTMV mtmv, TableIf table) { TableName tableName = null; try { tableName = new TableName(table); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterMTMVRenameInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterMTMVRenameInfo.java index 066342c3b2c706..4e34c50321a48a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterMTMVRenameInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterMTMVRenameInfo.java @@ -22,10 +22,12 @@ import org.apache.doris.catalog.Table; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeNameFormat; +import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.qe.ConnectContext; import java.util.Objects; +import java.util.Optional; /** * rename @@ -60,7 +62,9 @@ public void analyze(ConnectContext ctx) throws AnalysisException { public void run() throws DdlException { Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(mvName.getDb()); Table table = db.getTableOrDdlException(mvName.getTbl()); + BaseTableInfo oldTableInfo = new BaseTableInfo(table); Env.getCurrentEnv().renameTable(db, table, newName); - Env.getCurrentEnv().getMtmvService().alterTable(table, mvName.getTbl()); + BaseTableInfo newTableInfo = new BaseTableInfo(table); + Env.getCurrentEnv().getMtmvService().alterTable(oldTableInfo, Optional.of(newTableInfo), false); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterMTMVReplaceInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterMTMVReplaceInfo.java index 6dd0907db62063..a02af4ebeddae7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterMTMVReplaceInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterMTMVReplaceInfo.java @@ -25,12 +25,14 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.UserException; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.qe.ConnectContext; import java.util.Map; import java.util.Objects; +import java.util.Optional; /** * replace @@ -90,12 +92,7 @@ public void run() throws UserException { MTMV mtmv = (MTMV) db.getTableOrDdlException(mvName.getTbl(), TableType.MATERIALIZED_VIEW); MTMV newMtmv = (MTMV) db.getTableOrDdlException(newName, TableType.MATERIALIZED_VIEW); Env.getCurrentEnv().getAlterInstance().processReplaceTable(db, mtmv, newName, swapTable); - Env.getCurrentEnv().getMtmvService().alterTable(newMtmv, mvName.getTbl()); - if (swapTable) { - Env.getCurrentEnv().getMtmvService().alterTable(mtmv, newName); - } else { - Env.getCurrentEnv().getMtmvService().dropMTMV(mtmv); - Env.getCurrentEnv().getMtmvService().dropTable(mtmv); - } + Env.getCurrentEnv().getMtmvService() + .alterTable(new BaseTableInfo(mtmv), Optional.of(new BaseTableInfo(newMtmv)), true); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index 170e6c0e0a1ebe..f66e544aca9b39 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -24,12 +24,10 @@ import org.apache.doris.analysis.PartitionDesc; import org.apache.doris.analysis.RangePartitionDesc; import org.apache.doris.analysis.TableName; -import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.PartitionType; -import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; import org.apache.doris.catalog.View; @@ -60,8 +58,6 @@ import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.commands.info.BaseViewInfo.AnalyzerForCreateView; @@ -69,16 +65,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalSink; import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; -import org.apache.doris.nereids.types.AggStateType; -import org.apache.doris.nereids.types.CharType; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.nereids.types.DecimalV2Type; -import org.apache.doris.nereids.types.NullType; -import org.apache.doris.nereids.types.StringType; -import org.apache.doris.nereids.types.TinyIntType; -import org.apache.doris.nereids.types.VarcharType; -import org.apache.doris.nereids.types.coercion.CharacterType; -import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; @@ -116,7 +103,7 @@ public class CreateMTMVInfo { private final LogicalPlan logicalQuery; private String querySql; private final MTMVRefreshInfo refreshInfo; - private final List columns = Lists.newArrayList(); + private List columns = Lists.newArrayList(); private final List simpleColumnDefinitions; private final MTMVPartitionDefinition mvPartitionDefinition; private PartitionDesc partitionDesc; @@ -284,7 +271,10 @@ public void analyzeQuery(ConnectContext ctx, Map mvProperties) { getRelation(Sets.newHashSet(statementContext.getTables().values()), ctx); this.mvPartitionInfo = mvPartitionDefinition.analyzeAndTransferToMTMVPartitionInfo(planner); this.partitionDesc = generatePartitionDesc(ctx); - getColumns(plan, ctx, mvPartitionInfo.getPartitionCol(), distribution); + columns = MTMVPlanUtil.generateColumns(plan, ctx, mvPartitionInfo.getPartitionCol(), + (distribution == null || CollectionUtils.isEmpty(distribution.getCols())) ? Sets.newHashSet() + : Sets.newHashSet(distribution.getCols()), + simpleColumnDefinitions, properties); analyzeKeys(); } } @@ -402,90 +392,6 @@ private void analyzeExpressions(Plan plan, Map mvProperties) { } } - private void getColumns(Plan plan, ConnectContext ctx, String partitionCol, DistributionDescriptor distribution) { - List slots = plan.getOutput(); - if (slots.isEmpty()) { - throw new AnalysisException("table should contain at least one column"); - } - if (!CollectionUtils.isEmpty(simpleColumnDefinitions) && simpleColumnDefinitions.size() != slots.size()) { - throw new AnalysisException("simpleColumnDefinitions size is not equal to the query's"); - } - Set colNames = Sets.newHashSet(); - for (int i = 0; i < slots.size(); i++) { - String colName = CollectionUtils.isEmpty(simpleColumnDefinitions) ? slots.get(i).getName() - : simpleColumnDefinitions.get(i).getName(); - try { - FeNameFormat.checkColumnName(colName); - } catch (org.apache.doris.common.AnalysisException e) { - throw new AnalysisException(e.getMessage(), e); - } - if (colNames.contains(colName)) { - throw new AnalysisException("repeat cols:" + colName); - } else { - colNames.add(colName); - } - DataType dataType = getDataType(slots.get(i), i, ctx, partitionCol, distribution); - // If datatype is AggStateType, AggregateType should be generic, or column definition check will fail - columns.add(new ColumnDefinition( - colName, - dataType, - false, - slots.get(i).getDataType() instanceof AggStateType ? AggregateType.GENERIC : null, - slots.get(i).nullable(), - Optional.empty(), - CollectionUtils.isEmpty(simpleColumnDefinitions) ? null - : simpleColumnDefinitions.get(i).getComment())); - } - // add a hidden column as row store - if (properties != null) { - try { - boolean storeRowColumn = - PropertyAnalyzer.analyzeStoreRowColumn(Maps.newHashMap(properties)); - if (storeRowColumn) { - columns.add(ColumnDefinition.newRowStoreColumnDefinition(null)); - } - } catch (Exception e) { - throw new AnalysisException(e.getMessage(), e.getCause()); - } - } - } - - private DataType getDataType(Slot s, int i, ConnectContext ctx, String partitionCol, - DistributionDescriptor distribution) { - DataType dataType = s.getDataType().conversion(); - if (i == 0 && dataType.isStringType()) { - dataType = VarcharType.createVarcharType(ScalarType.MAX_VARCHAR_LENGTH); - } else { - dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, - NullType.class, TinyIntType.INSTANCE); - dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, - DecimalV2Type.class, DecimalV2Type.SYSTEM_DEFAULT); - if (s.isColumnFromTable()) { - if ((!((SlotReference) s).getTable().isPresent() - || !((SlotReference) s).getTable().get().isManagedTable())) { - if (s.getName().equals(partitionCol) || (distribution != null && distribution.inDistributionColumns( - s.getName()))) { - // String type can not be used in partition/distributed column - // so we replace it to varchar - dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, - CharacterType.class, VarcharType.MAX_VARCHAR_TYPE); - } else { - dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, - CharacterType.class, StringType.INSTANCE); - } - } - } else { - if (ctx.getSessionVariable().useMaxLengthOfVarcharInCtas) { - dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, - VarcharType.class, VarcharType.MAX_VARCHAR_TYPE); - dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, - CharType.class, VarcharType.MAX_VARCHAR_TYPE); - } - } - } - return dataType; - } - /** * translate to catalog CreateMultiTableMaterializedViewStmt */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DistributionDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DistributionDescriptor.java index 3e4d007ff5b913..26985dc80a4631 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DistributionDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DistributionDescriptor.java @@ -115,4 +115,8 @@ public DistributionDesc translateToCatalogStyle() { public boolean inDistributionColumns(String columnName) { return cols != null && cols.contains(columnName); } + + public List getCols() { + return cols; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java new file mode 100644 index 00000000000000..110f457cd00cb0 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java @@ -0,0 +1,282 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.Config; +import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.nereids.sqltest.SqlTestBase; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; +import org.apache.doris.nereids.trees.plans.commands.info.SimpleColumnDefinition; +import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DecimalV2Type; +import org.apache.doris.nereids.types.NullType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TinyIntType; +import org.apache.doris.nereids.types.VarcharType; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import mockit.Expectations; +import mockit.Mocked; +import org.junit.Assert; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public class MTMVPlanUtilTest extends SqlTestBase { + + @Test + public void testGenerateColumnsBySql() throws Exception { + createTables( + "CREATE TABLE IF NOT EXISTS MTMVPlanUtilTestT1 (\n" + + " id varchar(10),\n" + + " score String\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "AUTO PARTITION BY LIST(`id`)\n" + + "(\n" + + ")\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 1\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")\n" + ); + + String querySql = "select * from T1"; + List actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null, + Sets.newHashSet(), Lists.newArrayList(), + Maps.newHashMap()); + List expect = Lists.newArrayList(new ColumnDefinition("id", BigIntType.INSTANCE, true), + new ColumnDefinition("score", BigIntType.INSTANCE, true)); + checkRes(expect, actual); + + Map properties = Maps.newHashMap(); + properties.put(PropertyAnalyzer.PROPERTIES_STORE_ROW_COLUMN, "true"); + actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null, + Sets.newHashSet(), Lists.newArrayList(), properties); + expect = Lists.newArrayList(new ColumnDefinition("id", BigIntType.INSTANCE, true), + new ColumnDefinition("score", BigIntType.INSTANCE, true), + new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false)); + checkRes(expect, actual); + + querySql = "select T1.id from T1 inner join T2 on T1.id = T2.id"; + actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null, + Sets.newHashSet(), Lists.newArrayList(), + Maps.newHashMap()); + expect = Lists.newArrayList(new ColumnDefinition("id", BigIntType.INSTANCE, true)); + checkRes(expect, actual); + + querySql = "select id,sum(score) from T1 group by id"; + actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null, + Sets.newHashSet(), Lists.newArrayList(), + Maps.newHashMap()); + expect = Lists.newArrayList(new ColumnDefinition("id", BigIntType.INSTANCE, true), + new ColumnDefinition("__sum_1", BigIntType.INSTANCE, true)); + checkRes(expect, actual); + + querySql = "select id,sum(score) from T1 group by id"; + actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null, + Sets.newHashSet(), Lists.newArrayList(new SimpleColumnDefinition("id", null), + new SimpleColumnDefinition("sum_score", null)), + Maps.newHashMap()); + expect = Lists.newArrayList(new ColumnDefinition("id", BigIntType.INSTANCE, true), + new ColumnDefinition("sum_score", BigIntType.INSTANCE, true)); + checkRes(expect, actual); + + querySql = "select * from MTMVPlanUtilTestT1"; + actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null, + Sets.newHashSet(), Lists.newArrayList(), + Maps.newHashMap()); + expect = Lists.newArrayList(new ColumnDefinition("id", new VarcharType(10), true), + new ColumnDefinition("score", StringType.INSTANCE, true)); + checkRes(expect, actual); + + querySql = "select score from MTMVPlanUtilTestT1"; + actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null, + Sets.newHashSet(), Lists.newArrayList(), + Maps.newHashMap()); + expect = Lists.newArrayList( + new ColumnDefinition("score", VarcharType.MAX_VARCHAR_TYPE, true)); + checkRes(expect, actual); + } + + private void checkRes(List expect, List actual) { + Assert.assertEquals(expect.size(), actual.size()); + for (int i = 0; i < expect.size(); i++) { + Assert.assertEquals(expect.get(i).getName(), actual.get(i).getName()); + Assert.assertEquals(expect.get(i).getType(), actual.get(i).getType()); + } + } + + @Test + public void testGetDataType(@Mocked SlotReference slot, @Mocked TableIf slotTable) { + new Expectations() { + { + slot.getDataType(); + minTimes = 0; + result = StringType.INSTANCE; + + slot.isColumnFromTable(); + minTimes = 0; + result = true; + + slot.getTable(); + minTimes = 0; + result = Optional.empty(); + + slot.getName(); + minTimes = 0; + result = "slot_name"; + } + }; + // test i=0 + DataType dataType = MTMVPlanUtil.getDataType(slot, 0, connectContext, "pcol", Sets.newHashSet("dcol")); + Assert.assertEquals(VarcharType.MAX_VARCHAR_TYPE, dataType); + + // test isColumnFromTable and is not managed table + dataType = MTMVPlanUtil.getDataType(slot, 1, connectContext, "pcol", Sets.newHashSet("dcol")); + Assert.assertEquals(StringType.INSTANCE, dataType); + + // test is partitionCol + dataType = MTMVPlanUtil.getDataType(slot, 1, connectContext, "slot_name", Sets.newHashSet("dcol")); + Assert.assertEquals(VarcharType.MAX_VARCHAR_TYPE, dataType); + + // test is partitdistribution Col + dataType = MTMVPlanUtil.getDataType(slot, 1, connectContext, "pcol", Sets.newHashSet("slot_name")); + Assert.assertEquals(VarcharType.MAX_VARCHAR_TYPE, dataType); + // test managed table + new Expectations() { + { + slot.getTable(); + minTimes = 0; + result = Optional.of(slotTable); + + slotTable.isManagedTable(); + minTimes = 0; + result = true; + } + }; + + dataType = MTMVPlanUtil.getDataType(slot, 1, connectContext, "pcol", Sets.newHashSet("slot_name")); + Assert.assertEquals(StringType.INSTANCE, dataType); + + // test is not column table + boolean originalUseMaxLengthOfVarcharInCtas = connectContext.getSessionVariable().useMaxLengthOfVarcharInCtas; + new Expectations() { + { + slot.getDataType(); + minTimes = 0; + result = new VarcharType(10); + + slot.isColumnFromTable(); + minTimes = 0; + result = false; + } + }; + connectContext.getSessionVariable().useMaxLengthOfVarcharInCtas = true; + dataType = MTMVPlanUtil.getDataType(slot, 1, connectContext, "pcol", Sets.newHashSet("slot_name")); + Assert.assertEquals(VarcharType.MAX_VARCHAR_TYPE, dataType); + + connectContext.getSessionVariable().useMaxLengthOfVarcharInCtas = false; + dataType = MTMVPlanUtil.getDataType(slot, 1, connectContext, "pcol", Sets.newHashSet("slot_name")); + Assert.assertEquals(new VarcharType(10), dataType); + + connectContext.getSessionVariable().useMaxLengthOfVarcharInCtas = originalUseMaxLengthOfVarcharInCtas; + + // test null type + new Expectations() { + { + slot.getDataType(); + minTimes = 0; + result = NullType.INSTANCE; + } + }; + dataType = MTMVPlanUtil.getDataType(slot, 1, connectContext, "pcol", Sets.newHashSet("slot_name")); + Assert.assertEquals(TinyIntType.INSTANCE, dataType); + + // test decimal type + new Expectations() { + { + slot.getDataType(); + minTimes = 0; + result = DecimalV2Type.createDecimalV2Type(1, 1); + } + }; + boolean originalEnableDecimalConversion = Config.enable_decimal_conversion; + Config.enable_decimal_conversion = false; + dataType = MTMVPlanUtil.getDataType(slot, 1, connectContext, "pcol", Sets.newHashSet("slot_name")); + Assert.assertEquals(DecimalV2Type.SYSTEM_DEFAULT, dataType); + + Config.enable_decimal_conversion = originalEnableDecimalConversion; + } + + @Test + public void testGenerateColumns(@Mocked SlotReference slot, @Mocked Plan plan) { + new Expectations() { + { + plan.getOutput(); + minTimes = 0; + result = Lists.newArrayList(); + } + }; + // test slots is empty + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, () -> + MTMVPlanUtil.generateColumns(plan, connectContext, null, null, null, null)); + + new Expectations() { + { + plan.getOutput(); + minTimes = 0; + result = Lists.newArrayList(slot); + } + }; + + // test size of slots and SimpleColumnDefinitions is different + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, () -> + MTMVPlanUtil.generateColumns(plan, connectContext, null, null, + Lists.newArrayList(new SimpleColumnDefinition("col1", "c1"), + new SimpleColumnDefinition("col2", "c2")), null)); + + // test name format + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, () -> + MTMVPlanUtil.generateColumns(plan, connectContext, null, null, + Lists.newArrayList(new SimpleColumnDefinition("", "c1")), null)); + + new Expectations() { + { + plan.getOutput(); + minTimes = 0; + result = Lists.newArrayList(slot, slot); + } + }; + // test repeat col name + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, () -> + MTMVPlanUtil.generateColumns(plan, connectContext, null, null, + Lists.newArrayList(new SimpleColumnDefinition("col1", "c1"), + new SimpleColumnDefinition("col1", "c2")), null)); + } +} diff --git a/regression-test/data/mtmv_p0/test_base_add_col_multi_level_mtmv.out b/regression-test/data/mtmv_p0/test_base_add_col_multi_level_mtmv.out new file mode 100644 index 00000000000000..8d04722a3d69f7 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_base_add_col_multi_level_mtmv.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !add_col_t1_mv1 -- +test_base_add_col_multi_level_mtmv_mv1 NORMAL SUCCESS + +-- !add_col_t1_mv2 -- +test_base_add_col_multi_level_mtmv_mv2 NORMAL SUCCESS + +-- !add_col_t1_mv3 -- +test_base_add_col_multi_level_mtmv_mv3 NORMAL SUCCESS + +-- !add_col_t1_mv4 -- +test_base_add_col_multi_level_mtmv_mv4 NORMAL SUCCESS + diff --git a/regression-test/data/mtmv_p0/test_base_alter_col_comment_multi_level_mtmv.out b/regression-test/data/mtmv_p0/test_base_alter_col_comment_multi_level_mtmv.out new file mode 100644 index 00000000000000..1715233c26c786 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_base_alter_col_comment_multi_level_mtmv.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !add_col_t1_mv1 -- +test_base_alter_col_comment_multi_level_mtmv_mv1 NORMAL SUCCESS + +-- !add_col_t1_mv2 -- +test_base_alter_col_comment_multi_level_mtmv_mv2 NORMAL SUCCESS + +-- !add_col_t1_mv3 -- +test_base_alter_col_comment_multi_level_mtmv_mv3 NORMAL SUCCESS + +-- !add_col_t1_mv4 -- +test_base_alter_col_comment_multi_level_mtmv_mv4 NORMAL SUCCESS + diff --git a/regression-test/data/mtmv_p0/test_base_alter_col_type_multi_level_mtmv.out b/regression-test/data/mtmv_p0/test_base_alter_col_type_multi_level_mtmv.out new file mode 100644 index 00000000000000..10c71d60f7cdf4 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_base_alter_col_type_multi_level_mtmv.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !add_col_t1_mv1 -- +test_base_alter_col_type_multi_level_mtmv_mv1 SCHEMA_CHANGE SUCCESS + +-- !add_col_t1_mv2 -- +test_base_alter_col_type_multi_level_mtmv_mv2 NORMAL SUCCESS + +-- !add_col_t1_mv3 -- +test_base_alter_col_type_multi_level_mtmv_mv3 SCHEMA_CHANGE SUCCESS + +-- !add_col_t1_mv4 -- +test_base_alter_col_type_multi_level_mtmv_mv4 SCHEMA_CHANGE SUCCESS + diff --git a/regression-test/data/mtmv_p0/test_base_comment_multi_level_mtmv.out b/regression-test/data/mtmv_p0/test_base_comment_multi_level_mtmv.out new file mode 100644 index 00000000000000..dec544860a3e64 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_base_comment_multi_level_mtmv.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !comment_t1_mv1 -- +test_base_comment_multi_level_mtmv_mv1 NORMAL SUCCESS + +-- !comment_t1_mv2 -- +test_base_comment_multi_level_mtmv_mv2 NORMAL SUCCESS + +-- !comment_t1_mv3 -- +test_base_comment_multi_level_mtmv_mv3 NORMAL SUCCESS + +-- !comment_t1_mv4 -- +test_base_comment_multi_level_mtmv_mv4 NORMAL SUCCESS + diff --git a/regression-test/data/mtmv_p0/test_base_drop_col_multi_level_mtmv.out b/regression-test/data/mtmv_p0/test_base_drop_col_multi_level_mtmv.out new file mode 100644 index 00000000000000..c9a16bf56f62a1 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_base_drop_col_multi_level_mtmv.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !add_col_t1_mv1 -- +test_base_drop_col_multi_level_mtmv_mv1 SCHEMA_CHANGE SUCCESS + +-- !add_col_t1_mv2 -- +test_base_drop_col_multi_level_mtmv_mv2 NORMAL SUCCESS + +-- !add_col_t1_mv3 -- +test_base_drop_col_multi_level_mtmv_mv3 SCHEMA_CHANGE SUCCESS + +-- !add_col_t1_mv4 -- +test_base_drop_col_multi_level_mtmv_mv4 SCHEMA_CHANGE SUCCESS + diff --git a/regression-test/data/mtmv_p0/test_base_drop_multi_level_mtmv.out b/regression-test/data/mtmv_p0/test_base_drop_multi_level_mtmv.out new file mode 100644 index 00000000000000..f04ea1d8b30f64 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_base_drop_multi_level_mtmv.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !drop_t1_mv1 -- +test_base_drop_multi_level_mtmv_mv1 SCHEMA_CHANGE SUCCESS + +-- !drop_t1_mv2 -- +test_base_drop_multi_level_mtmv_mv2 NORMAL SUCCESS + +-- !drop_t1_mv3 -- +test_base_drop_multi_level_mtmv_mv3 SCHEMA_CHANGE SUCCESS + +-- !drop_t1_mv4 -- +test_base_drop_multi_level_mtmv_mv4 SCHEMA_CHANGE SUCCESS + diff --git a/regression-test/data/mtmv_p0/test_base_recreate_on_commit_mtmv.out b/regression-test/data/mtmv_p0/test_base_recreate_on_commit_mtmv.out new file mode 100644 index 00000000000000..c40f161d60670b --- /dev/null +++ b/regression-test/data/mtmv_p0/test_base_recreate_on_commit_mtmv.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_init -- +1 1 + +-- !drop -- +test_base_recreate_on_commit_mtmv_mv SCHEMA_CHANGE SUCCESS + +-- !recreate -- +test_base_recreate_on_commit_mtmv_mv NORMAL SUCCESS + +-- !select_recreate -- +2 2 + diff --git a/regression-test/data/mtmv_p0/test_base_rename_col_multi_level_mtmv.out b/regression-test/data/mtmv_p0/test_base_rename_col_multi_level_mtmv.out new file mode 100644 index 00000000000000..4976d963cd6336 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_base_rename_col_multi_level_mtmv.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !add_col_t1_mv1 -- +test_base_rename_col_multi_level_mtmv_mv1 SCHEMA_CHANGE SUCCESS + +-- !add_col_t1_mv2 -- +test_base_rename_col_multi_level_mtmv_mv2 NORMAL SUCCESS + +-- !add_col_t1_mv3 -- +test_base_rename_col_multi_level_mtmv_mv3 SCHEMA_CHANGE SUCCESS + +-- !add_col_t1_mv4 -- +test_base_rename_col_multi_level_mtmv_mv4 SCHEMA_CHANGE SUCCESS + diff --git a/regression-test/data/mtmv_p0/test_base_rename_multi_level_mtmv.out b/regression-test/data/mtmv_p0/test_base_rename_multi_level_mtmv.out new file mode 100644 index 00000000000000..8bbccdee4ce5b6 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_base_rename_multi_level_mtmv.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !drop_t1_mv1 -- +test_base_rename_multi_level_mtmv_mv1 SCHEMA_CHANGE SUCCESS + +-- !drop_t1_mv2 -- +test_base_rename_multi_level_mtmv_mv2 NORMAL SUCCESS + +-- !drop_t1_mv3 -- +test_base_rename_multi_level_mtmv_mv3 SCHEMA_CHANGE SUCCESS + +-- !drop_t1_mv4 -- +test_base_rename_multi_level_mtmv_mv4 SCHEMA_CHANGE SUCCESS + diff --git a/regression-test/data/mtmv_p0/test_base_rename_mv_multi_level_mtmv.out b/regression-test/data/mtmv_p0/test_base_rename_mv_multi_level_mtmv.out new file mode 100644 index 00000000000000..a275b2ec9dbaff --- /dev/null +++ b/regression-test/data/mtmv_p0/test_base_rename_mv_multi_level_mtmv.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !rename_mv_mv1 -- +test_base_rename_mv_multi_level_mtmv_mv1_rename NORMAL SUCCESS + +-- !rename_mv_mv2 -- +test_base_rename_mv_multi_level_mtmv_mv2 NORMAL SUCCESS + +-- !rename_mv_mv3 -- +test_base_rename_mv_multi_level_mtmv_mv3 NORMAL SUCCESS + +-- !rename_mv_mv4 -- +test_base_rename_mv_multi_level_mtmv_mv4 SCHEMA_CHANGE SUCCESS + diff --git a/regression-test/data/mtmv_p0/test_base_rename_on_commit_mtmv.out b/regression-test/data/mtmv_p0/test_base_rename_on_commit_mtmv.out new file mode 100644 index 00000000000000..dddffa8c2f17b9 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_base_rename_on_commit_mtmv.out @@ -0,0 +1,25 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_init -- +1 1 + +-- !rename -- +test_base_rename_on_commit_mtmv_mv SCHEMA_CHANGE SUCCESS + +-- !rename -- +test_base_rename_on_commit_mtmv_mv SCHEMA_CHANGE SUCCESS + +-- !select_rename -- +1 1 + +-- !recreate_auto -- +test_base_rename_on_commit_mtmv_mv NORMAL SUCCESS + +-- !select_recreate_auto -- +10 10 + +-- !before_trigger -- +2 + +-- !after_trigger -- +2 + diff --git a/regression-test/data/mtmv_p0/test_base_rename_p_mv_multi_level_mtmv.out b/regression-test/data/mtmv_p0/test_base_rename_p_mv_multi_level_mtmv.out new file mode 100644 index 00000000000000..2dfc8b7f831966 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_base_rename_p_mv_multi_level_mtmv.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !rename_mv_mv1 -- +test_base_rename_p_mv_multi_level_mtmv_mv1_rename NORMAL SUCCESS + +-- !rename_mv_mv2 -- +test_base_rename_p_mv_multi_level_mtmv_mv2 NORMAL SUCCESS + +-- !rename_mv_mv3 -- +test_base_rename_p_mv_multi_level_mtmv_mv3 NORMAL SUCCESS + +-- !rename_mv_mv4 -- +test_base_rename_p_mv_multi_level_mtmv_mv4 SCHEMA_CHANGE SUCCESS + diff --git a/regression-test/data/mtmv_p0/test_base_replace_multi_level_mtmv.out b/regression-test/data/mtmv_p0/test_base_replace_multi_level_mtmv.out new file mode 100644 index 00000000000000..147506ca708a41 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_base_replace_multi_level_mtmv.out @@ -0,0 +1,37 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !replace_t1_mv1 -- +test_base_replace_multi_level_mtmv_mv1 SCHEMA_CHANGE SUCCESS + +-- !replace_t1_mv2 -- +test_base_replace_multi_level_mtmv_mv2 SCHEMA_CHANGE SUCCESS + +-- !replace_t1_mv3 -- +test_base_replace_multi_level_mtmv_mv3 SCHEMA_CHANGE SUCCESS + +-- !replace_t1_mv4 -- +test_base_replace_multi_level_mtmv_mv4 SCHEMA_CHANGE SUCCESS + +-- !refresh_t1_mv1 -- +test_base_replace_multi_level_mtmv_mv1 NORMAL SUCCESS + +-- !refresh_t1_mv2 -- +test_base_replace_multi_level_mtmv_mv2 NORMAL SUCCESS + +-- !refresh_t1_mv3 -- +test_base_replace_multi_level_mtmv_mv3 NORMAL SUCCESS + +-- !refresh_t1_mv4 -- +test_base_replace_multi_level_mtmv_mv4 NORMAL SUCCESS + +-- !replace_t1_mv1 -- +test_base_replace_multi_level_mtmv_mv1 SCHEMA_CHANGE SUCCESS + +-- !replace_t1_mv2 -- +test_base_replace_multi_level_mtmv_mv2 SCHEMA_CHANGE SUCCESS + +-- !replace_t1_mv3 -- +test_base_replace_multi_level_mtmv_mv3 SCHEMA_CHANGE SUCCESS + +-- !replace_t1_mv4 -- +test_base_replace_multi_level_mtmv_mv4 SCHEMA_CHANGE SUCCESS + diff --git a/regression-test/data/mtmv_p0/test_base_replace_mv_multi_level_mtmv.out b/regression-test/data/mtmv_p0/test_base_replace_mv_multi_level_mtmv.out new file mode 100644 index 00000000000000..1fdd40d9270ced --- /dev/null +++ b/regression-test/data/mtmv_p0/test_base_replace_mv_multi_level_mtmv.out @@ -0,0 +1,30 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !replace_true_mv_mv1 -- +test_base_replace_mv_multi_level_mtmv_mv1 NORMAL SUCCESS + +-- !replace_true_mv_mv11 -- +test_base_replace_mv_multi_level_mtmv_mv11 NORMAL SUCCESS + +-- !replace_true_mv_mv2 -- +test_base_replace_mv_multi_level_mtmv_mv2 NORMAL SUCCESS + +-- !replace_true_mv_mv3 -- +test_base_replace_mv_multi_level_mtmv_mv3 NORMAL SUCCESS + +-- !replace_true_mv_mv4 -- +test_base_replace_mv_multi_level_mtmv_mv4 SCHEMA_CHANGE SUCCESS + +-- !replace_false_mv_mv1 -- +test_base_replace_mv_multi_level_mtmv_mv1 NORMAL SUCCESS + +-- !replace_false_mv_mv11 -- + +-- !replace_false_mv_mv2 -- +test_base_replace_mv_multi_level_mtmv_mv2 NORMAL SUCCESS + +-- !replace_false_mv_mv3 -- +test_base_replace_mv_multi_level_mtmv_mv3 NORMAL SUCCESS + +-- !replace_false_mv_mv4 -- +test_base_replace_mv_multi_level_mtmv_mv4 SCHEMA_CHANGE SUCCESS + diff --git a/regression-test/data/mtmv_p0/test_base_replace_on_commit_mtmv.out b/regression-test/data/mtmv_p0/test_base_replace_on_commit_mtmv.out new file mode 100644 index 00000000000000..e5bb84bf37884e --- /dev/null +++ b/regression-test/data/mtmv_p0/test_base_replace_on_commit_mtmv.out @@ -0,0 +1,20 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_init -- +1 1 + +-- !replace -- +test_base_replace_on_commit_mtmv_mv SCHEMA_CHANGE SUCCESS + +-- !replace -- +test_base_replace_on_commit_mtmv_mv NORMAL SUCCESS + +-- !select_replace -- +10 10 +2 2 + +-- !before_trigger -- +2 + +-- !after_trigger -- +2 + diff --git a/regression-test/data/mtmv_p0/test_column_type_mtmv.out b/regression-test/data/mtmv_p0/test_column_type_mtmv.out new file mode 100644 index 00000000000000..442df2b235f737 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_column_type_mtmv.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !normal -- +test_column_type_mtmv_mv NORMAL SUCCESS + +-- !after_schema_change -- +test_column_type_mtmv_mv SCHEMA_CHANGE SUCCESS + +-- !after_refresh -- +test_column_type_mtmv_mv SCHEMA_CHANGE FAIL + diff --git a/regression-test/suites/mtmv_p0/test_base_add_col_multi_level_mtmv.groovy b/regression-test/suites/mtmv_p0/test_base_add_col_multi_level_mtmv.groovy new file mode 100644 index 00000000000000..1d079661968b49 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_base_add_col_multi_level_mtmv.groovy @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_base_add_col_multi_level_mtmv","mtmv") { + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_base_add_col_multi_level_mtmv" + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName1 = "${suiteName}_mv1" + String mvName2 = "${suiteName}_mv2" + String mvName3 = "${suiteName}_mv3" + String mvName4 = "${suiteName}_mv4" + String querySql = "SELECT t1.k1,t1.k2,t2.k2 as k3 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k1;"; + sql """set enable_materialized_view_nest_rewrite = true;""" + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName1};""" + sql """drop materialized view if exists ${mvName2};""" + sql """drop materialized view if exists ${mvName3};""" + sql """drop materialized view if exists ${mvName4};""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 varchar(32) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + CREATE TABLE ${tableName2} + ( + k1 INT, + k2 varchar(33) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + INSERT INTO ${tableName1} VALUES(1,"a"); + """ + sql """ + INSERT INTO ${tableName2} VALUES(1,"b"); + """ + sql """ + CREATE MATERIALIZED VIEW ${mvName1} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName1}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName1} auto + """ + waitingMTMVTaskFinishedByMvName(mvName1) + + sql """ + CREATE MATERIALIZED VIEW ${mvName2} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName2}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName2} auto + """ + waitingMTMVTaskFinishedByMvName(mvName2) + + sql """ + CREATE MATERIALIZED VIEW ${mvName3} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k2 as k3 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k1; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName3} auto + """ + waitingMTMVTaskFinishedByMvName(mvName3) + + sql """ + CREATE MATERIALIZED VIEW ${mvName4} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k2 as k3 from ${mvName1} t1 join ${mvName2} t2 on t1.k1=t2.k1; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName4} auto + """ + waitingMTMVTaskFinishedByMvName(mvName4) + + // add column + sql """ + alter table ${tableName1} add COLUMN new_col INT AFTER k2; + """ + assertEquals("FINISHED", getAlterColumnFinalState("${tableName1}")) + order_qt_add_col_t1_mv1 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName1}'" + order_qt_add_col_t1_mv2 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName2}'" + order_qt_add_col_t1_mv3 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName3}'" + order_qt_add_col_t1_mv4 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName4}'" + + mv_rewrite_success_without_check_chosen(querySql, mvName1) + mv_rewrite_success_without_check_chosen(querySql, mvName2) + mv_rewrite_success_without_check_chosen(querySql, mvName3) + mv_rewrite_success_without_check_chosen(querySql, mvName4) +} diff --git a/regression-test/suites/mtmv_p0/test_base_alter_col_comment_multi_level_mtmv.groovy b/regression-test/suites/mtmv_p0/test_base_alter_col_comment_multi_level_mtmv.groovy new file mode 100644 index 00000000000000..6bb2eba17adb41 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_base_alter_col_comment_multi_level_mtmv.groovy @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_base_alter_col_comment_multi_level_mtmv","mtmv") { + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_base_alter_col_comment_multi_level_mtmv" + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName1 = "${suiteName}_mv1" + String mvName2 = "${suiteName}_mv2" + String mvName3 = "${suiteName}_mv3" + String mvName4 = "${suiteName}_mv4" + String querySql = "SELECT t1.k1,t1.k2,t2.k2 as k3 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k1;"; + sql """set enable_materialized_view_nest_rewrite = true;""" + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName1};""" + sql """drop materialized view if exists ${mvName2};""" + sql """drop materialized view if exists ${mvName3};""" + sql """drop materialized view if exists ${mvName4};""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 varchar(32), + k3 varchar(32) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + CREATE TABLE ${tableName2} + ( + k1 INT, + k2 varchar(33) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + INSERT INTO ${tableName1} VALUES(1,"a","a2"); + """ + sql """ + INSERT INTO ${tableName2} VALUES(1,"b"); + """ + sql """ + CREATE MATERIALIZED VIEW ${mvName1} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName1}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName1} auto + """ + waitingMTMVTaskFinishedByMvName(mvName1) + + sql """ + CREATE MATERIALIZED VIEW ${mvName2} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName2}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName2} auto + """ + waitingMTMVTaskFinishedByMvName(mvName2) + + sql """ + CREATE MATERIALIZED VIEW ${mvName3} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k2 as k3 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k1; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName3} auto + """ + waitingMTMVTaskFinishedByMvName(mvName3) + + sql """ + CREATE MATERIALIZED VIEW ${mvName4} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k2 as k3 from ${mvName1} t1 join ${mvName2} t2 on t1.k1=t2.k1; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName4} auto + """ + waitingMTMVTaskFinishedByMvName(mvName4) + + // alter column comment + sql """ + alter table ${tableName1} modify COLUMN k3 COMMENT 'new comment'; + """ + order_qt_add_col_t1_mv1 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName1}'" + order_qt_add_col_t1_mv2 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName2}'" + order_qt_add_col_t1_mv3 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName3}'" + order_qt_add_col_t1_mv4 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName4}'" + + mv_rewrite_success_without_check_chosen(querySql, mvName1) + mv_rewrite_success_without_check_chosen(querySql, mvName2) + mv_rewrite_success_without_check_chosen(querySql, mvName3) + mv_rewrite_success_without_check_chosen(querySql, mvName4) +} diff --git a/regression-test/suites/mtmv_p0/test_base_alter_col_type_multi_level_mtmv.groovy b/regression-test/suites/mtmv_p0/test_base_alter_col_type_multi_level_mtmv.groovy new file mode 100644 index 00000000000000..83abd73dde56bc --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_base_alter_col_type_multi_level_mtmv.groovy @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_base_alter_col_type_multi_level_mtmv","mtmv") { + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_base_alter_col_type_multi_level_mtmv" + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName1 = "${suiteName}_mv1" + String mvName2 = "${suiteName}_mv2" + String mvName3 = "${suiteName}_mv3" + String mvName4 = "${suiteName}_mv4" + String querySql = "SELECT t1.k1,t1.k2,t2.k2 as k3 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k1;"; + sql """set enable_materialized_view_nest_rewrite = true;""" + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName1};""" + sql """drop materialized view if exists ${mvName2};""" + sql """drop materialized view if exists ${mvName3};""" + sql """drop materialized view if exists ${mvName4};""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 varchar(32), + k3 varchar(32) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + CREATE TABLE ${tableName2} + ( + k1 INT, + k2 varchar(33) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + INSERT INTO ${tableName1} VALUES(1,"a","a2"); + """ + sql """ + INSERT INTO ${tableName2} VALUES(1,"b"); + """ + sql """ + CREATE MATERIALIZED VIEW ${mvName1} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName1}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName1} auto + """ + waitingMTMVTaskFinishedByMvName(mvName1) + + sql """ + CREATE MATERIALIZED VIEW ${mvName2} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName2}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName2} auto + """ + waitingMTMVTaskFinishedByMvName(mvName2) + + sql """ + CREATE MATERIALIZED VIEW ${mvName3} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k2 as k3 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k1; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName3} auto + """ + waitingMTMVTaskFinishedByMvName(mvName3) + + sql """ + CREATE MATERIALIZED VIEW ${mvName4} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k2 as k3 from ${mvName1} t1 join ${mvName2} t2 on t1.k1=t2.k1; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName4} auto + """ + waitingMTMVTaskFinishedByMvName(mvName4) + + // rename column + sql """ + alter table ${tableName1} modify COLUMN k3 VARCHAR(100); + """ + order_qt_add_col_t1_mv1 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName1}'" + order_qt_add_col_t1_mv2 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName2}'" + order_qt_add_col_t1_mv3 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName3}'" + order_qt_add_col_t1_mv4 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName4}'" + + mv_rewrite_success_without_check_chosen(querySql, mvName2) + mv_not_part_in(querySql, mvName1) + mv_not_part_in(querySql, mvName3) + mv_not_part_in(querySql, mvName4) +} diff --git a/regression-test/suites/mtmv_p0/test_base_comment_multi_level_mtmv.groovy b/regression-test/suites/mtmv_p0/test_base_comment_multi_level_mtmv.groovy new file mode 100644 index 00000000000000..5636c7dbcb4fa9 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_base_comment_multi_level_mtmv.groovy @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_base_comment_multi_level_mtmv","mtmv") { + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_base_comment_multi_level_mtmv" + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName1 = "${suiteName}_mv1" + String mvName2 = "${suiteName}_mv2" + String mvName3 = "${suiteName}_mv3" + String mvName4 = "${suiteName}_mv4" + String querySql = "SELECT t1.k1,t1.k2,t2.k2 as k3 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k1;"; + sql """set enable_materialized_view_nest_rewrite = true;""" + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName1};""" + sql """drop materialized view if exists ${mvName2};""" + sql """drop materialized view if exists ${mvName3};""" + sql """drop materialized view if exists ${mvName4};""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 varchar(32) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + CREATE TABLE ${tableName2} + ( + k1 INT, + k2 varchar(33) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + INSERT INTO ${tableName1} VALUES(1,"a"); + """ + sql """ + INSERT INTO ${tableName2} VALUES(1,"b"); + """ + sql """ + CREATE MATERIALIZED VIEW ${mvName1} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName1}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName1} auto + """ + waitingMTMVTaskFinishedByMvName(mvName1) + + sql """ + CREATE MATERIALIZED VIEW ${mvName2} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName2}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName2} auto + """ + waitingMTMVTaskFinishedByMvName(mvName2) + + sql """ + CREATE MATERIALIZED VIEW ${mvName3} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k2 as k3 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k1; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName3} auto + """ + waitingMTMVTaskFinishedByMvName(mvName3) + + sql """ + CREATE MATERIALIZED VIEW ${mvName4} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k2 as k3 from ${mvName1} t1 join ${mvName2} t2 on t1.k1=t2.k1; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName4} auto + """ + waitingMTMVTaskFinishedByMvName(mvName4) + + // alter comment + sql """ + alter table ${tableName1} MODIFY COMMENT "new table comment"; + """ + order_qt_comment_t1_mv1 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName1}'" + order_qt_comment_t1_mv2 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName2}'" + order_qt_comment_t1_mv3 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName3}'" + order_qt_comment_t1_mv4 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName4}'" + + mv_rewrite_success_without_check_chosen(querySql, mvName1) + mv_rewrite_success_without_check_chosen(querySql, mvName2) + mv_rewrite_success_without_check_chosen(querySql, mvName3) + mv_rewrite_success_without_check_chosen(querySql, mvName4) +} diff --git a/regression-test/suites/mtmv_p0/test_base_drop_col_multi_level_mtmv.groovy b/regression-test/suites/mtmv_p0/test_base_drop_col_multi_level_mtmv.groovy new file mode 100644 index 00000000000000..3f0b77e6073416 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_base_drop_col_multi_level_mtmv.groovy @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_base_drop_col_multi_level_mtmv","mtmv") { + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_base_drop_col_multi_level_mtmv" + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName1 = "${suiteName}_mv1" + String mvName2 = "${suiteName}_mv2" + String mvName3 = "${suiteName}_mv3" + String mvName4 = "${suiteName}_mv4" + String querySql = "SELECT t1.k1,t1.k2,t2.k2 as k3 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k1;"; + sql """set enable_materialized_view_nest_rewrite = true;""" + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName1};""" + sql """drop materialized view if exists ${mvName2};""" + sql """drop materialized view if exists ${mvName3};""" + sql """drop materialized view if exists ${mvName4};""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 varchar(32), + k3 varchar(32) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + CREATE TABLE ${tableName2} + ( + k1 INT, + k2 varchar(33) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + INSERT INTO ${tableName1} VALUES(1,"a","a2"); + """ + sql """ + INSERT INTO ${tableName2} VALUES(1,"b"); + """ + sql """ + CREATE MATERIALIZED VIEW ${mvName1} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName1}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName1} auto + """ + waitingMTMVTaskFinishedByMvName(mvName1) + + sql """ + CREATE MATERIALIZED VIEW ${mvName2} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName2}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName2} auto + """ + waitingMTMVTaskFinishedByMvName(mvName2) + + sql """ + CREATE MATERIALIZED VIEW ${mvName3} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k2 as k3 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k1; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName3} auto + """ + waitingMTMVTaskFinishedByMvName(mvName3) + + sql """ + CREATE MATERIALIZED VIEW ${mvName4} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k2 as k3 from ${mvName1} t1 join ${mvName2} t2 on t1.k1=t2.k1; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName4} auto + """ + waitingMTMVTaskFinishedByMvName(mvName4) + + // drop column + sql """ + alter table ${tableName1} drop COLUMN k3; + """ + assertEquals("FINISHED", getAlterColumnFinalState("${tableName1}")) + order_qt_add_col_t1_mv1 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName1}'" + order_qt_add_col_t1_mv2 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName2}'" + order_qt_add_col_t1_mv3 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName3}'" + order_qt_add_col_t1_mv4 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName4}'" + + mv_rewrite_success_without_check_chosen(querySql, mvName2) + mv_not_part_in(querySql, mvName1) + mv_not_part_in(querySql, mvName3) + mv_not_part_in(querySql, mvName4) +} diff --git a/regression-test/suites/mtmv_p0/test_base_drop_multi_level_mtmv.groovy b/regression-test/suites/mtmv_p0/test_base_drop_multi_level_mtmv.groovy new file mode 100644 index 00000000000000..e81f1799ec50ed --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_base_drop_multi_level_mtmv.groovy @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_base_drop_multi_level_mtmv","mtmv") { + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_base_drop_multi_level_mtmv" + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName1 = "${suiteName}_mv1" + String mvName2 = "${suiteName}_mv2" + String mvName3 = "${suiteName}_mv3" + String mvName4 = "${suiteName}_mv4" + String querySql = "SELECT t1.k1,t1.k2,t2.k4 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k3;"; + sql """set enable_materialized_view_nest_rewrite = true;""" + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName1};""" + sql """drop materialized view if exists ${mvName2};""" + sql """drop materialized view if exists ${mvName3};""" + sql """drop materialized view if exists ${mvName4};""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 varchar(32) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + CREATE TABLE ${tableName2} + ( + k3 INT, + k4 varchar(32) + ) + DISTRIBUTED BY HASH(k3) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + INSERT INTO ${tableName1} VALUES(1,"a"); + """ + sql """ + INSERT INTO ${tableName2} VALUES(1,"b"); + """ + sql """ + CREATE MATERIALIZED VIEW ${mvName1} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName1}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName1} auto + """ + waitingMTMVTaskFinishedByMvName(mvName1) + + sql """ + CREATE MATERIALIZED VIEW ${mvName2} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k3) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName2}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName2} auto + """ + waitingMTMVTaskFinishedByMvName(mvName2) + + sql """ + CREATE MATERIALIZED VIEW ${mvName3} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k4 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k3; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName3} auto + """ + waitingMTMVTaskFinishedByMvName(mvName3) + + sql """ + CREATE MATERIALIZED VIEW ${mvName4} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k4 from ${mvName1} t1 join ${mvName2} t2 on t1.k1=t2.k3; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName4} auto + """ + waitingMTMVTaskFinishedByMvName(mvName4) + + // drop t1 + sql """drop table if exists `${tableName1}`""" + order_qt_drop_t1_mv1 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName1}'" + order_qt_drop_t1_mv2 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName2}'" + order_qt_drop_t1_mv3 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName3}'" + order_qt_drop_t1_mv4 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName4}'" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 varchar(32) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + INSERT INTO ${tableName1} VALUES(1,"a"); + """ + + mv_rewrite_success_without_check_chosen(querySql, mvName2) + mv_not_part_in(querySql, mvName1) + mv_not_part_in(querySql, mvName3) + mv_not_part_in(querySql, mvName4) +} diff --git a/regression-test/suites/mtmv_p0/test_base_recreate_on_commit_mtmv.groovy b/regression-test/suites/mtmv_p0/test_base_recreate_on_commit_mtmv.groovy new file mode 100644 index 00000000000000..c2011952a52730 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_base_recreate_on_commit_mtmv.groovy @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_base_recreate_on_commit_mtmv","mtmv") { + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_base_recreate_on_commit_mtmv" + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName = "${suiteName}_mv" + + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 INT + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + INSERT INTO ${tableName1} VALUES(1,1); + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON COMMIT + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName1}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} auto + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_select_init "select * from ${mvName}" + + // drop and recreate + sql """drop table if exists `${tableName1}`""" + order_qt_drop "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName}'" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 INT + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + INSERT INTO ${tableName1} VALUES(2,2); + """ + + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_recreate "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName}'" + order_qt_select_recreate "select * from ${mvName}" +} diff --git a/regression-test/suites/mtmv_p0/test_base_rename_col_multi_level_mtmv.groovy b/regression-test/suites/mtmv_p0/test_base_rename_col_multi_level_mtmv.groovy new file mode 100644 index 00000000000000..2e803efa5d2389 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_base_rename_col_multi_level_mtmv.groovy @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_base_rename_col_multi_level_mtmv","mtmv") { + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_base_rename_col_multi_level_mtmv" + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName1 = "${suiteName}_mv1" + String mvName2 = "${suiteName}_mv2" + String mvName3 = "${suiteName}_mv3" + String mvName4 = "${suiteName}_mv4" + String querySql = "SELECT t1.k1,t1.k2,t2.k2 as k3 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k1;"; + sql """set enable_materialized_view_nest_rewrite = true;""" + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName1};""" + sql """drop materialized view if exists ${mvName2};""" + sql """drop materialized view if exists ${mvName3};""" + sql """drop materialized view if exists ${mvName4};""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 varchar(32), + k3 varchar(32) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + CREATE TABLE ${tableName2} + ( + k1 INT, + k2 varchar(33) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + INSERT INTO ${tableName1} VALUES(1,"a","a2"); + """ + sql """ + INSERT INTO ${tableName2} VALUES(1,"b"); + """ + sql """ + CREATE MATERIALIZED VIEW ${mvName1} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName1}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName1} auto + """ + waitingMTMVTaskFinishedByMvName(mvName1) + + sql """ + CREATE MATERIALIZED VIEW ${mvName2} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName2}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName2} auto + """ + waitingMTMVTaskFinishedByMvName(mvName2) + + sql """ + CREATE MATERIALIZED VIEW ${mvName3} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k2 as k3 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k1; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName3} auto + """ + waitingMTMVTaskFinishedByMvName(mvName3) + + sql """ + CREATE MATERIALIZED VIEW ${mvName4} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k2 as k3 from ${mvName1} t1 join ${mvName2} t2 on t1.k1=t2.k1; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName4} auto + """ + waitingMTMVTaskFinishedByMvName(mvName4) + + // rename column + sql """ + alter table ${tableName1} rename COLUMN k3 k4; + """ + order_qt_add_col_t1_mv1 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName1}'" + order_qt_add_col_t1_mv2 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName2}'" + order_qt_add_col_t1_mv3 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName3}'" + order_qt_add_col_t1_mv4 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName4}'" + + mv_rewrite_success_without_check_chosen(querySql, mvName2) + mv_not_part_in(querySql, mvName1) + mv_not_part_in(querySql, mvName3) + mv_not_part_in(querySql, mvName4) +} diff --git a/regression-test/suites/mtmv_p0/test_base_rename_multi_level_mtmv.groovy b/regression-test/suites/mtmv_p0/test_base_rename_multi_level_mtmv.groovy new file mode 100644 index 00000000000000..0f967ec096aec1 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_base_rename_multi_level_mtmv.groovy @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_base_rename_multi_level_mtmv","mtmv") { + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_base_rename_multi_level_mtmv" + String tableName1 = "${suiteName}_table1" + String tableName1Rename = "${suiteName}_table1_rename" + String tableName2 = "${suiteName}_table2" + String mvName1 = "${suiteName}_mv1" + String mvName2 = "${suiteName}_mv2" + String mvName3 = "${suiteName}_mv3" + String mvName4 = "${suiteName}_mv4" + String querySql = "SELECT t1.k1,t1.k2,t2.k4 from ${tableName1Rename} t1 join ${tableName2} t2 on t1.k1=t2.k3;"; + sql """set enable_materialized_view_nest_rewrite = true;""" + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName1Rename}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName1};""" + sql """drop materialized view if exists ${mvName2};""" + sql """drop materialized view if exists ${mvName3};""" + sql """drop materialized view if exists ${mvName4};""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 varchar(32) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + CREATE TABLE ${tableName2} + ( + k3 INT, + k4 varchar(32) + ) + DISTRIBUTED BY HASH(k3) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + INSERT INTO ${tableName1} VALUES(1,"a"); + """ + sql """ + INSERT INTO ${tableName2} VALUES(1,"b"); + """ + sql """ + CREATE MATERIALIZED VIEW ${mvName1} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName1}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName1} auto + """ + waitingMTMVTaskFinishedByMvName(mvName1) + + sql """ + CREATE MATERIALIZED VIEW ${mvName2} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k3) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName2}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName2} auto + """ + waitingMTMVTaskFinishedByMvName(mvName2) + + sql """ + CREATE MATERIALIZED VIEW ${mvName3} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k4 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k3; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName3} auto + """ + waitingMTMVTaskFinishedByMvName(mvName3) + + sql """ + CREATE MATERIALIZED VIEW ${mvName4} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k4 from ${mvName1} t1 join ${mvName2} t2 on t1.k1=t2.k3; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName4} auto + """ + waitingMTMVTaskFinishedByMvName(mvName4) + + // rename t1 + sql """ + ALTER TABLE ${tableName1} rename ${tableName1Rename}; + """ + order_qt_drop_t1_mv1 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName1}'" + order_qt_drop_t1_mv2 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName2}'" + order_qt_drop_t1_mv3 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName3}'" + order_qt_drop_t1_mv4 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName4}'" + + mv_rewrite_success_without_check_chosen(querySql, mvName2) + mv_not_part_in(querySql, mvName1) + mv_not_part_in(querySql, mvName3) + mv_not_part_in(querySql, mvName4) +} diff --git a/regression-test/suites/mtmv_p0/test_base_rename_mv_multi_level_mtmv.groovy b/regression-test/suites/mtmv_p0/test_base_rename_mv_multi_level_mtmv.groovy new file mode 100644 index 00000000000000..db95b0e412c5f2 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_base_rename_mv_multi_level_mtmv.groovy @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_base_rename_mv_multi_level_mtmv","mtmv") { + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_base_rename_mv_multi_level_mtmv" + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName1 = "${suiteName}_mv1" + String mvName1Rename = "${suiteName}_mv1_rename" + String mvName2 = "${suiteName}_mv2" + String mvName3 = "${suiteName}_mv3" + String mvName4 = "${suiteName}_mv4" + String querySql = "SELECT t1.k1,t1.k2,t2.k4 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k3;"; + sql """set enable_materialized_view_nest_rewrite = true;""" + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName1};""" + sql """drop materialized view if exists ${mvName1Rename};""" + sql """drop materialized view if exists ${mvName2};""" + sql """drop materialized view if exists ${mvName3};""" + sql """drop materialized view if exists ${mvName4};""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 varchar(32) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + CREATE TABLE ${tableName2} + ( + k3 INT, + k4 varchar(32) + ) + DISTRIBUTED BY HASH(k3) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + INSERT INTO ${tableName1} VALUES(1,"a"); + """ + sql """ + INSERT INTO ${tableName2} VALUES(1,"b"); + """ + sql """ + CREATE MATERIALIZED VIEW ${mvName1} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName1}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName1} auto + """ + waitingMTMVTaskFinishedByMvName(mvName1) + + sql """ + CREATE MATERIALIZED VIEW ${mvName2} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k3) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName2}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName2} auto + """ + waitingMTMVTaskFinishedByMvName(mvName2) + + sql """ + CREATE MATERIALIZED VIEW ${mvName3} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k4 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k3; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName3} auto + """ + waitingMTMVTaskFinishedByMvName(mvName3) + + sql """ + CREATE MATERIALIZED VIEW ${mvName4} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k4 from ${mvName1} t1 join ${mvName2} t2 on t1.k1=t2.k3; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName4} auto + """ + waitingMTMVTaskFinishedByMvName(mvName4) + + // rename mv1 + sql """ + ALTER MATERIALIZED VIEW ${mvName1} rename ${mvName1Rename}; + """ + order_qt_rename_mv_mv1 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName1Rename}'" + order_qt_rename_mv_mv2 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName2}'" + order_qt_rename_mv_mv3 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName3}'" + order_qt_rename_mv_mv4 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName4}'" + // rename table will rename default partition name, so will change to async + mv_not_part_in(querySql, mvName1Rename) + mv_rewrite_success_without_check_chosen(querySql, mvName2) + mv_rewrite_success_without_check_chosen(querySql, mvName3) + mv_not_part_in(querySql, mvName4) + + // after refresh,should can rewrite + sql """ + REFRESH MATERIALIZED VIEW ${mvName1Rename} auto + """ + waitingMTMVTaskFinishedByMvName(mvName1Rename) + mv_rewrite_success_without_check_chosen(querySql, mvName1Rename) +} diff --git a/regression-test/suites/mtmv_p0/test_base_rename_on_commit_mtmv.groovy b/regression-test/suites/mtmv_p0/test_base_rename_on_commit_mtmv.groovy new file mode 100644 index 00000000000000..02aca18c0f33e4 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_base_rename_on_commit_mtmv.groovy @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_base_rename_on_commit_mtmv","mtmv") { + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_base_rename_on_commit_mtmv" + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName = "${suiteName}_mv" + + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 INT + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + INSERT INTO ${tableName1} VALUES(1,1); + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON COMMIT + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName1}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} auto + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_select_init "select * from ${mvName}" + + // rename and recreate + sql """ + ALTER TABLE ${tableName1} rename ${tableName2}; + """ + order_qt_rename "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName}'" + + sql """ + INSERT INTO ${tableName2} VALUES(2,2); + """ + + // after rename, should not refresh auto + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_rename "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName}'" + order_qt_select_rename "select * from ${mvName}" + + // create t1 + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 INT + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + INSERT INTO ${tableName1} VALUES(10,10); + """ + + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_recreate_auto "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName}'" + order_qt_select_recreate_auto "select * from ${mvName}" + + // t2 should not trigger refresh + order_qt_before_trigger "select count(*) from tasks('type'='mv') where MvName='${mvName}'" + sql """ + INSERT INTO ${tableName2} VALUES(4,4); + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_after_trigger "select count(*) from tasks('type'='mv') where MvName='${mvName}'" +} diff --git a/regression-test/suites/mtmv_p0/test_base_rename_p_mv_multi_level_mtmv.groovy b/regression-test/suites/mtmv_p0/test_base_rename_p_mv_multi_level_mtmv.groovy new file mode 100644 index 00000000000000..e7ba67650f99cf --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_base_rename_p_mv_multi_level_mtmv.groovy @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_base_rename_p_mv_multi_level_mtmv","mtmv") { + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_base_rename_p_mv_multi_level_mtmv" + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName1 = "${suiteName}_mv1" + String mvName1Rename = "${suiteName}_mv1_rename" + String mvName2 = "${suiteName}_mv2" + String mvName3 = "${suiteName}_mv3" + String mvName4 = "${suiteName}_mv4" + String querySql = "SELECT t1.k1,t1.k2,t2.k4 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k3;"; + sql """set enable_materialized_view_nest_rewrite = true;""" + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName1};""" + sql """drop materialized view if exists ${mvName1Rename};""" + sql """drop materialized view if exists ${mvName2};""" + sql """drop materialized view if exists ${mvName3};""" + sql """drop materialized view if exists ${mvName4};""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 varchar(32) + ) + PARTITION BY LIST(`k1`) + ( + PARTITION `p1` VALUES IN ('1'), + PARTITION `p2` VALUES IN ('2'), + PARTITION `p3` VALUES IN ('3') + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + CREATE TABLE ${tableName2} + ( + k3 INT, + k4 varchar(32) + ) + PARTITION BY LIST(`k3`) + ( + PARTITION `p1` VALUES IN ('1'), + PARTITION `p2` VALUES IN ('2'), + PARTITION `p3` VALUES IN ('3') + ) + DISTRIBUTED BY HASH(k3) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + INSERT INTO ${tableName1} VALUES(1,"a"); + """ + sql """ + INSERT INTO ${tableName2} VALUES(1,"b"); + """ + sql """ + CREATE MATERIALIZED VIEW ${mvName1} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(`k1`) + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName1}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName1} auto + """ + waitingMTMVTaskFinishedByMvName(mvName1) + + sql """ + CREATE MATERIALIZED VIEW ${mvName2} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k3) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName2}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName2} auto + """ + waitingMTMVTaskFinishedByMvName(mvName2) + + sql """ + CREATE MATERIALIZED VIEW ${mvName3} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k4 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k3; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName3} auto + """ + waitingMTMVTaskFinishedByMvName(mvName3) + + sql """ + CREATE MATERIALIZED VIEW ${mvName4} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k4 from ${mvName1} t1 join ${mvName2} t2 on t1.k1=t2.k3; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName4} auto + """ + waitingMTMVTaskFinishedByMvName(mvName4) + + // rename mv1 + sql """ + ALTER MATERIALIZED VIEW ${mvName1} rename ${mvName1Rename}; + """ + order_qt_rename_mv_mv1 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName1Rename}'" + order_qt_rename_mv_mv2 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName2}'" + order_qt_rename_mv_mv3 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName3}'" + order_qt_rename_mv_mv4 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName4}'" + mv_rewrite_success_without_check_chosen(querySql, mvName1Rename) + mv_rewrite_success_without_check_chosen(querySql, mvName2) + mv_rewrite_success_without_check_chosen(querySql, mvName3) + mv_not_part_in(querySql, mvName4) + mv_not_part_in(querySql, mvName1) +} diff --git a/regression-test/suites/mtmv_p0/test_base_replace_multi_level_mtmv.groovy b/regression-test/suites/mtmv_p0/test_base_replace_multi_level_mtmv.groovy new file mode 100644 index 00000000000000..d572a818a9b108 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_base_replace_multi_level_mtmv.groovy @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_base_replace_multi_level_mtmv","mtmv") { + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_base_replace_multi_level_mtmv" + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName1 = "${suiteName}_mv1" + String mvName2 = "${suiteName}_mv2" + String mvName3 = "${suiteName}_mv3" + String mvName4 = "${suiteName}_mv4" + String querySql = "SELECT t1.k1,t1.k2,t2.k2 as k3 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k1;"; + sql """set enable_materialized_view_nest_rewrite = true;""" + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName1};""" + sql """drop materialized view if exists ${mvName2};""" + sql """drop materialized view if exists ${mvName3};""" + sql """drop materialized view if exists ${mvName4};""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 varchar(32) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + CREATE TABLE ${tableName2} + ( + k1 INT, + k2 varchar(33) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + INSERT INTO ${tableName1} VALUES(1,"a"); + """ + sql """ + INSERT INTO ${tableName2} VALUES(1,"b"); + """ + sql """ + CREATE MATERIALIZED VIEW ${mvName1} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName1}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName1} auto + """ + waitingMTMVTaskFinishedByMvName(mvName1) + + sql """ + CREATE MATERIALIZED VIEW ${mvName2} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName2}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName2} auto + """ + waitingMTMVTaskFinishedByMvName(mvName2) + + sql """ + CREATE MATERIALIZED VIEW ${mvName3} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k2 as k3 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k1; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName3} auto + """ + waitingMTMVTaskFinishedByMvName(mvName3) + + sql """ + CREATE MATERIALIZED VIEW ${mvName4} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k2 as k3 from ${mvName1} t1 join ${mvName2} t2 on t1.k1=t2.k1; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName4} auto + """ + waitingMTMVTaskFinishedByMvName(mvName4) + + // replace t1 + sql """ + ALTER TABLE ${tableName1} REPLACE WITH TABLE ${tableName2} PROPERTIES('swap' = 'true'); + """ + order_qt_replace_t1_mv1 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName1}'" + order_qt_replace_t1_mv2 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName2}'" + order_qt_replace_t1_mv3 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName3}'" + order_qt_replace_t1_mv4 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName4}'" + + mv_not_part_in(querySql, mvName2) + mv_not_part_in(querySql, mvName1) + mv_not_part_in(querySql, mvName3) + mv_not_part_in(querySql, mvName4) + + sql """ + REFRESH MATERIALIZED VIEW ${mvName1} auto + """ + waitingMTMVTaskFinishedByMvName(mvName1) + + sql """ + REFRESH MATERIALIZED VIEW ${mvName2} auto + """ + waitingMTMVTaskFinishedByMvName(mvName2) + + sql """ + REFRESH MATERIALIZED VIEW ${mvName3} auto + """ + waitingMTMVTaskFinishedByMvName(mvName3) + + sql """ + REFRESH MATERIALIZED VIEW ${mvName4} auto + """ + waitingMTMVTaskFinishedByMvName(mvName4) + + order_qt_refresh_t1_mv1 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName1}'" + order_qt_refresh_t1_mv2 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName2}'" + order_qt_refresh_t1_mv3 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName3}'" + order_qt_refresh_t1_mv4 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName4}'" + + + mv_rewrite_success_without_check_chosen(querySql, mvName1) + mv_rewrite_success_without_check_chosen(querySql, mvName2) + mv_rewrite_success_without_check_chosen(querySql, mvName3) + mv_rewrite_success_without_check_chosen(querySql, mvName4) + + // replace t1 swap false + sql """ + ALTER TABLE ${tableName1} REPLACE WITH TABLE ${tableName2} PROPERTIES('swap' = 'false'); + """ + order_qt_replace_t1_mv1 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName1}'" + order_qt_replace_t1_mv2 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName2}'" + order_qt_replace_t1_mv3 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName3}'" + order_qt_replace_t1_mv4 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName4}'" + +} diff --git a/regression-test/suites/mtmv_p0/test_base_replace_mv_multi_level_mtmv.groovy b/regression-test/suites/mtmv_p0/test_base_replace_mv_multi_level_mtmv.groovy new file mode 100644 index 00000000000000..410f28afd02343 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_base_replace_mv_multi_level_mtmv.groovy @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_base_replace_mv_multi_level_mtmv","mtmv") { + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_base_replace_mv_multi_level_mtmv" + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName1 = "${suiteName}_mv1" + String mvName11 = "${suiteName}_mv11" + String mvName2 = "${suiteName}_mv2" + String mvName3 = "${suiteName}_mv3" + String mvName4 = "${suiteName}_mv4" + String querySql = "SELECT t1.k1,t1.k2,t2.k4 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k3;"; + sql """set enable_materialized_view_nest_rewrite = true;""" + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName1};""" + sql """drop materialized view if exists ${mvName11};""" + sql """drop materialized view if exists ${mvName2};""" + sql """drop materialized view if exists ${mvName3};""" + sql """drop materialized view if exists ${mvName4};""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 varchar(32) + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + CREATE TABLE ${tableName2} + ( + k3 INT, + k4 varchar(32) + ) + DISTRIBUTED BY HASH(k3) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + INSERT INTO ${tableName1} VALUES(1,"a"); + """ + sql """ + INSERT INTO ${tableName2} VALUES(1,"b"); + """ + sql """ + CREATE MATERIALIZED VIEW ${mvName1} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName1}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName1} auto + """ + waitingMTMVTaskFinishedByMvName(mvName1) + + sql """ + CREATE MATERIALIZED VIEW ${mvName11} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName1}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName11} auto + """ + waitingMTMVTaskFinishedByMvName(mvName11) + + sql """ + CREATE MATERIALIZED VIEW ${mvName2} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k3) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName2}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName2} auto + """ + waitingMTMVTaskFinishedByMvName(mvName2) + + sql """ + CREATE MATERIALIZED VIEW ${mvName3} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k4 from ${tableName1} t1 join ${tableName2} t2 on t1.k1=t2.k3; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName3} auto + """ + waitingMTMVTaskFinishedByMvName(mvName3) + + sql """ + CREATE MATERIALIZED VIEW ${mvName4} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT t1.k1,t1.k2,t2.k4 from ${mvName1} t1 join ${mvName2} t2 on t1.k1=t2.k3; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName4} auto + """ + waitingMTMVTaskFinishedByMvName(mvName4) + + // replace mv1 + sql """ + ALTER MATERIALIZED VIEW ${mvName1} REPLACE WITH MATERIALIZED VIEW ${mvName11} PROPERTIES('swap' = 'true');; + """ + order_qt_replace_true_mv_mv1 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName1}'" + order_qt_replace_true_mv_mv11 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName11}'" + order_qt_replace_true_mv_mv2 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName2}'" + order_qt_replace_true_mv_mv3 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName3}'" + order_qt_replace_true_mv_mv4 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName4}'" + // replace table will rename default partition name, so will change to async + mv_not_part_in(querySql, mvName1) + mv_not_part_in(querySql, mvName11) + mv_rewrite_success_without_check_chosen(querySql, mvName2) + mv_rewrite_success_without_check_chosen(querySql, mvName3) + mv_not_part_in(querySql, mvName4) + + // after refresh,should can rewrite + sql """ + REFRESH MATERIALIZED VIEW ${mvName1} auto + """ + waitingMTMVTaskFinishedByMvName(mvName1) + mv_rewrite_success_without_check_chosen(querySql, mvName1) + + // after refresh,should can rewrite + sql """ + REFRESH MATERIALIZED VIEW ${mvName11} auto + """ + waitingMTMVTaskFinishedByMvName(mvName11) + mv_rewrite_success_without_check_chosen(querySql, mvName11) + + // replace mv1 + sql """ + ALTER MATERIALIZED VIEW ${mvName1} REPLACE WITH MATERIALIZED VIEW ${mvName11} PROPERTIES('swap' = 'false');; + """ + order_qt_replace_false_mv_mv1 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName1}'" + order_qt_replace_false_mv_mv11 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName11}'" + order_qt_replace_false_mv_mv2 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName2}'" + order_qt_replace_false_mv_mv3 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName3}'" + order_qt_replace_false_mv_mv4 "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName4}'" + // replace table will rename default partition name, so will change to async + mv_not_part_in(querySql, mvName1) + mv_rewrite_success_without_check_chosen(querySql, mvName2) + mv_rewrite_success_without_check_chosen(querySql, mvName3) + mv_not_part_in(querySql, mvName4) + + // after refresh,should can rewrite + sql """ + REFRESH MATERIALIZED VIEW ${mvName1} auto + """ + waitingMTMVTaskFinishedByMvName(mvName1) + mv_rewrite_success_without_check_chosen(querySql, mvName1) +} diff --git a/regression-test/suites/mtmv_p0/test_base_replace_on_commit_mtmv.groovy b/regression-test/suites/mtmv_p0/test_base_replace_on_commit_mtmv.groovy new file mode 100644 index 00000000000000..8e8c3f599b2482 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_base_replace_on_commit_mtmv.groovy @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_base_replace_on_commit_mtmv","mtmv") { + String dbName = context.config.getDbNameByFile(context.file) + String suiteName = "test_base_replace_on_commit_mtmv" + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName = "${suiteName}_mv" + + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 INT + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + INSERT INTO ${tableName1} VALUES(1,1); + """ + + sql """ + CREATE TABLE ${tableName2} + ( + k1 INT, + k2 INT + ) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + INSERT INTO ${tableName2} VALUES(10,10); + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON COMMIT + DISTRIBUTED BY hash(k1) BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName1}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} auto + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_select_init "select * from ${mvName}" + + // replace + sql """ + ALTER TABLE ${tableName1} REPLACE WITH TABLE ${tableName2} PROPERTIES('swap' = 'true'); + """ + order_qt_replace "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName}'" + + sql """ + INSERT INTO ${tableName1} VALUES(2,2); + """ + + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_replace "select Name,State,RefreshState from mv_infos('database'='${dbName}') where Name='${mvName}'" + order_qt_select_replace "select * from ${mvName}" + + // t2 should not trigger refresh + order_qt_before_trigger "select count(*) from tasks('type'='mv') where MvName='${mvName}'" + sql """ + INSERT INTO ${tableName2} VALUES(20,20); + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_after_trigger "select count(*) from tasks('type'='mv') where MvName='${mvName}'" +} diff --git a/regression-test/suites/mtmv_p0/test_column_type_mtmv.groovy b/regression-test/suites/mtmv_p0/test_column_type_mtmv.groovy new file mode 100644 index 00000000000000..fdf03ad19da692 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_column_type_mtmv.groovy @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_column_type_mtmv","mtmv") { + String suiteName = "test_column_type_mtmv" + String tableName = "${suiteName}_table" + String mvName = "${suiteName}_mv" + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE ${tableName} ( + `user_id` int NOT NULL, + class_id int, + `age` int NULL + ) ENGINE=OLAP + DUPLICATE KEY(`user_id`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`user_id`) BUCKETS 2 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql """insert into ${tableName} values(1,1,1);""" + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') AS + SELECT user_id,class_id,sum(age) from ${tableName} group by user_id,class_id; + """ + + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_normal "select Name,State,RefreshState from mv_infos('database'='regression_test_mtmv_p0') where Name='${mvName}'" + sql """alter table ${tableName} modify column class_id bigint;""" + assertEquals("FINISHED", getAlterColumnFinalState("${tableName}")) + order_qt_after_schema_change "select Name,State,RefreshState from mv_infos('database'='regression_test_mtmv_p0') where Name='${mvName}'" + sql """ + REFRESH MATERIALIZED VIEW ${mvName} auto; + """ + def jobName = getJobName("regression_test_mtmv_p0", mvName); + waitingMTMVTaskFinishedNotNeedSuccess(jobName) + // can not be rewrite + order_qt_after_refresh "select Name,State,RefreshState from mv_infos('database'='regression_test_mtmv_p0') where Name='${mvName}'" +} diff --git a/regression-test/suites/mtmv_p0/test_recreate_table_mtmv.groovy b/regression-test/suites/mtmv_p0/test_recreate_table_mtmv.groovy index e9f7f339a0c6a7..ffae7398e3ab0c 100644 --- a/regression-test/suites/mtmv_p0/test_recreate_table_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_recreate_table_mtmv.groovy @@ -81,6 +81,9 @@ suite("test_recreate_table_mtmv","mtmv") { """ waitingMTMVTaskFinishedByMvName(mvName1); order_qt_mv1_recreate "SELECT * FROM ${mvName1}" + sql """ + REFRESH MATERIALIZED VIEW ${mvName2} auto + """ waitingMTMVTaskFinishedByMvName(mvName2); order_qt_mv2_recreate "SELECT * FROM ${mvName2}"