diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index e3333f325858e3..d23c7534f8664e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -95,9 +95,9 @@ private static Set getBaseTables(Plan plan) { TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( com.google.common.collect.Sets - .newHashSet(TableType.values())); + .newHashSet(TableType.values()), true); plan.accept(TableCollector.INSTANCE, collectorContext); - List collectedTables = collectorContext.getCollectedTables(); + Set collectedTables = collectorContext.getCollectedTables(); return transferTableIfToInfo(collectedTables); } @@ -105,7 +105,7 @@ private static Set getBaseViews(Plan plan) { return Sets.newHashSet(); } - private static Set transferTableIfToInfo(List tables) { + private static Set transferTableIfToInfo(Set tables) { Set result = com.google.common.collect.Sets.newHashSet(); for (TableIf table : tables) { result.add(new BaseTableInfo(table)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index 8046639d1bac9d..914b213361348a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -66,9 +66,9 @@ public void initMaterializationContext(CascadesContext cascadesContext) { return; } Plan rewritePlan = cascadesContext.getRewritePlan(); - TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet()); + TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), true); rewritePlan.accept(TableCollector.INSTANCE, collectorContext); - List collectedTables = collectorContext.getCollectedTables(); + Set collectedTables = collectorContext.getCollectedTables(); if (collectedTables.isEmpty()) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index d7c502a9eb84c6..7e7f58db44db22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -320,9 +320,9 @@ private PartitionDesc generatePartitionDesc(MTMVRelatedTableIf relatedTable, Con private void analyzeBaseTables(Plan plan) { TableCollectorContext collectorContext = - new TableCollector.TableCollectorContext(Sets.newHashSet(TableType.MATERIALIZED_VIEW)); + new TableCollector.TableCollectorContext(Sets.newHashSet(TableType.MATERIALIZED_VIEW), true); plan.accept(TableCollector.INSTANCE, collectorContext); - List collectedTables = collectorContext.getCollectedTables(); + Set collectedTables = collectorContext.getCollectedTables(); if (!CollectionUtils.isEmpty(collectedTables)) { throw new AnalysisException("can not contain MATERIALIZED_VIEW"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTestScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTestScan.java index 7d326ade3d603b..94cae4b955d798 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTestScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTestScan.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import java.util.List; @@ -68,4 +69,9 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr Optional logicalProperties, List children) { return new LogicalTestScan(relationId, table, qualifier, groupExpression, logicalProperties); } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitLogicalTestScan(this, context); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java index adfe4991665216..5eaf0b7213648d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundTVFRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan; @@ -31,6 +32,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan; import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalTestScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalEsScan; @@ -54,8 +57,16 @@ public interface RelationVisitor { R visitLogicalRelation(LogicalRelation logicalRelation, C context); + default R visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, C context) { + return visitLogicalRelation(catalogRelation, context); + } + R visitPhysicalRelation(PhysicalRelation physicalRelation, C context); + default R visitPhysicalCatalogRelation(PhysicalCatalogRelation catalogRelation, C context) { + return visitPhysicalRelation(catalogRelation, context); + } + // ******************************* // unbound relations // ******************************* @@ -81,28 +92,28 @@ default R visitLogicalEmptyRelation(LogicalEmptyRelation emptyRelation, C contex } default R visitLogicalEsScan(LogicalEsScan esScan, C context) { - return visitLogicalRelation(esScan, context); + return visitLogicalCatalogRelation(esScan, context); } default R visitLogicalFileScan(LogicalFileScan fileScan, C context) { - return visitLogicalRelation(fileScan, context); + return visitLogicalCatalogRelation(fileScan, context); } default R visitLogicalJdbcScan(LogicalJdbcScan jdbcScan, C context) { - return visitLogicalRelation(jdbcScan, context); + return visitLogicalCatalogRelation(jdbcScan, context); } default R visitLogicalOdbcScan(LogicalOdbcScan odbcScan, C context) { - return visitLogicalRelation(odbcScan, context); + return visitLogicalCatalogRelation(odbcScan, context); } default R visitLogicalOlapScan(LogicalOlapScan olapScan, C context) { - return visitLogicalRelation(olapScan, context); + return visitLogicalCatalogRelation(olapScan, context); } default R visitLogicalDeferMaterializeOlapScan( LogicalDeferMaterializeOlapScan deferMaterializeOlapScan, C context) { - return visitLogicalRelation(deferMaterializeOlapScan, context); + return visitLogicalCatalogRelation(deferMaterializeOlapScan, context); } default R visitLogicalOneRowRelation(LogicalOneRowRelation oneRowRelation, C context) { @@ -110,13 +121,17 @@ default R visitLogicalOneRowRelation(LogicalOneRowRelation oneRowRelation, C con } default R visitLogicalSchemaScan(LogicalSchemaScan schemaScan, C context) { - return visitLogicalRelation(schemaScan, context); + return visitLogicalCatalogRelation(schemaScan, context); } default R visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, C context) { return visitLogicalRelation(tvfRelation, context); } + default R visitLogicalTestScan(LogicalTestScan testScan, C context) { + return visitLogicalCatalogRelation(testScan, context); + } + // ******************************* // physical relations // ******************************* @@ -126,28 +141,28 @@ default R visitPhysicalEmptyRelation(PhysicalEmptyRelation emptyRelation, C cont } default R visitPhysicalEsScan(PhysicalEsScan esScan, C context) { - return visitPhysicalRelation(esScan, context); + return visitPhysicalCatalogRelation(esScan, context); } default R visitPhysicalFileScan(PhysicalFileScan fileScan, C context) { - return visitPhysicalRelation(fileScan, context); + return visitPhysicalCatalogRelation(fileScan, context); } default R visitPhysicalJdbcScan(PhysicalJdbcScan jdbcScan, C context) { - return visitPhysicalRelation(jdbcScan, context); + return visitPhysicalCatalogRelation(jdbcScan, context); } default R visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan, C context) { - return visitPhysicalRelation(odbcScan, context); + return visitPhysicalCatalogRelation(odbcScan, context); } default R visitPhysicalOlapScan(PhysicalOlapScan olapScan, C context) { - return visitPhysicalRelation(olapScan, context); + return visitPhysicalCatalogRelation(olapScan, context); } default R visitPhysicalDeferMaterializeOlapScan( PhysicalDeferMaterializeOlapScan deferMaterializeOlapScan, C context) { - return visitPhysicalRelation(deferMaterializeOlapScan, context); + return visitPhysicalCatalogRelation(deferMaterializeOlapScan, context); } default R visitPhysicalOneRowRelation(PhysicalOneRowRelation oneRowRelation, C context) { @@ -155,7 +170,7 @@ default R visitPhysicalOneRowRelation(PhysicalOneRowRelation oneRowRelation, C c } default R visitPhysicalSchemaScan(PhysicalSchemaScan schemaScan, C context) { - return visitPhysicalRelation(schemaScan, context); + return visitPhysicalCatalogRelation(schemaScan, context); } default R visitPhysicalTVFRelation(PhysicalTVFRelation tvfRelation, C context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java index a3c874f6370444..3736ab60b0f6e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java @@ -17,33 +17,73 @@ package org.apache.doris.nereids.trees.plans.visitor; +import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.mtmv.MTMVCache; +import org.apache.doris.mtmv.MTMVPlanUtil; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext; -import java.util.ArrayList; -import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.HashSet; import java.util.Set; /** * Collect the table in plan * Note: will not get table if table is eliminated by EmptyRelation in rewrite. + * View expand is in RBO, if call this method with the plan after RBO, this will get base tables in view, or will not. + * Materialized view is extended or not can be controlled by the field expand */ -public class TableCollector extends DefaultPlanVisitor { +public class TableCollector extends DefaultPlanVisitor { public static final TableCollector INSTANCE = new TableCollector(); + private static final Logger LOG = LogManager.getLogger(TableCollector.class); @Override - public Void visit(Plan plan, TableCollectorContext context) { - if (plan instanceof CatalogRelation) { - TableIf table = ((CatalogRelation) plan).getTable(); - if (context.getTargetTableTypes().isEmpty() || context.getTargetTableTypes().contains(table.getType())) { - context.getCollectedTables().add(table); - } + public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, TableCollectorContext context) { + TableIf table = catalogRelation.getTable(); + if (context.getTargetTableTypes().isEmpty() || context.getTargetTableTypes().contains(table.getType())) { + context.getCollectedTables().add(table); + } + if (table instanceof MTMV) { + expandMvAndCollect((MTMV) table, context); + } + return catalogRelation; + } + + @Override + public Plan visitPhysicalCatalogRelation(PhysicalCatalogRelation catalogRelation, TableCollectorContext context) { + TableIf table = catalogRelation.getTable(); + if (context.getTargetTableTypes().isEmpty() || context.getTargetTableTypes().contains(table.getType())) { + context.getCollectedTables().add(table); + } + if (table instanceof MTMV) { + expandMvAndCollect((MTMV) table, context); + } + return catalogRelation; + } + + private void expandMvAndCollect(MTMV mtmv, TableCollectorContext context) { + if (!context.isExpand()) { + return; + } + try { + MTMVCache expandedMv = MTMVCache.from(mtmv, MTMVPlanUtil.createMTMVContext(mtmv)); + expandedMv.getLogicalPlan().accept(this, context); + } catch (AnalysisException e) { + LOG.error(String.format( + "table collector expand fail, mtmv name is %s, targetTableTypes is %s", + mtmv.getName(), context.targetTableTypes), e); + throw new org.apache.doris.nereids.exceptions.AnalysisException( + String.format("expand mv and collect table fail, mv name is %s, mv sql is %s", + mtmv.getName(), mtmv.getQuerySql()), e); } - return super.visit(plan, context); } /** @@ -51,19 +91,26 @@ public Void visit(Plan plan, TableCollectorContext context) { * and the result of collect. */ public static final class TableCollectorContext { - private final List collectedTables = new ArrayList<>(); + private final Set collectedTables = new HashSet<>(); private final Set targetTableTypes; + // if expand the mv or not + private final boolean expand; - public TableCollectorContext(Set targetTableTypes) { + public TableCollectorContext(Set targetTableTypes, boolean expand) { this.targetTableTypes = targetTableTypes; + this.expand = expand; } - public List getCollectedTables() { + public Set getCollectedTables() { return collectedTables; } public Set getTargetTableTypes() { return targetTableTypes; } + + public boolean isExpand() { + return expand; + } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java index 775bc05e1639a5..23cfc1cc452d8c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java @@ -37,7 +37,9 @@ import org.junit.jupiter.api.Test; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; /** @@ -78,9 +80,31 @@ protected void runBeforeAll() throws Exception { + "\"replication_num\" = \"1\"\n" + ");"); + createTable("CREATE TABLE `table3` (\n" + + " `c1` bigint(20) NULL,\n" + + " `c2` bigint(20) NULL,\n" + + " `c3` bigint(20) not NULL,\n" + + " `k4` bitmap BITMAP_UNION NULL,\n" + + " `k5` bitmap BITMAP_UNION NULL\n" + + ") ENGINE=OLAP\n" + + "AGGREGATE KEY(`c1`, `c2`, `c3`)\n" + + "COMMENT 'OLAP'\n" + + "DISTRIBUTED BY HASH(`c2`) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\"\n" + + ");"); + createView("CREATE VIEW `view1` AS SELECT t1.*, random() FROM\n" + "`table1` t1 LEFT JOIN\n" + "`table2` t2 ON t1.c1 = t2.c1;"); + + createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" + + "DISTRIBUTED BY RANDOM BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1') \n" + + "as " + + "select t1.c1, t3.c2 " + + "from table1 t1 " + + "inner join table3 t3 on t1.c1= t3.c2;"); } @Test @@ -97,18 +121,16 @@ public void test1() { Assertions.assertEquals(1, collectResult.size()); Assertions.assertTrue(collectResult.get(0) instanceof Random); // Check get tables - TableCollectorContext collectorContext = - new TableCollector.TableCollectorContext(Sets.newHashSet(TableType.OLAP)); + TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( + Sets.newHashSet(TableType.OLAP), true); physicalPlan.accept(TableCollector.INSTANCE, collectorContext); - Assertions.assertEquals(3, collectorContext.getCollectedTables().size()); - List expectedTables = new ArrayList<>(); + Set expectedTables = new HashSet<>(); expectedTables.add("table1"); expectedTables.add("table2"); - expectedTables.add("table2"); Assertions.assertEquals( collectorContext.getCollectedTables().stream() .map(TableIf::getName) - .collect(Collectors.toList()), + .collect(Collectors.toSet()), expectedTables); }); } @@ -128,21 +150,83 @@ public void test2() { Assertions.assertTrue(collectResult.get(0) instanceof Uuid); Assertions.assertTrue(collectResult.get(1) instanceof Random); // Check get tables - TableCollectorContext collectorContext = - new TableCollector.TableCollectorContext(Sets.newHashSet(TableType.OLAP)); + TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( + Sets.newHashSet(TableType.OLAP), true); physicalPlan.accept(TableCollector.INSTANCE, collectorContext); - Assertions.assertEquals(4, collectorContext.getCollectedTables().size()); - List expectedTables = new ArrayList<>(); + Set expectedTables = new HashSet<>(); expectedTables.add("table1"); expectedTables.add("table2"); + Assertions.assertEquals( + collectorContext.getCollectedTables().stream() + .map(TableIf::getName) + .collect(Collectors.toSet()), + expectedTables); + }); + } + + @Test + public void test3() throws Exception { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT mv1.*, uuid() FROM mv1 " + + "INNER JOIN view1 on mv1.c1 = view1.c2 " + + "LEFT SEMI JOIN table2 ON mv1.c1 = table2.c1 " + + "WHERE mv1.c1 IN (SELECT c1 FROM table2) OR mv1.c1 < 10", + nereidsPlanner -> { + PhysicalPlan physicalPlan = nereidsPlanner.getPhysicalPlan(); + List> collectResult = new ArrayList<>(); + // Check nondeterministic collect + physicalPlan.accept(NondeterministicFunctionCollector.INSTANCE, collectResult); + // Check get tables + TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( + Sets.newHashSet(TableType.OLAP), true); + physicalPlan.accept(TableCollector.INSTANCE, collectorContext); + Set expectedTables = new HashSet<>(); + expectedTables.add("table1"); expectedTables.add("table2"); - expectedTables.add("table2"); + expectedTables.add("table3"); Assertions.assertEquals( collectorContext.getCollectedTables().stream() .map(TableIf::getName) - .collect(Collectors.toList()), + .collect(Collectors.toSet()), expectedTables); + + TableCollectorContext collectorContextWithNoExpand = + new TableCollector.TableCollectorContext(Sets.newHashSet(TableType.OLAP), + false); + physicalPlan.accept(TableCollector.INSTANCE, collectorContextWithNoExpand); + Set expectedTablesWithNoExpand = new HashSet<>(); + expectedTablesWithNoExpand.add("table1"); + expectedTablesWithNoExpand.add("table2"); + Assertions.assertEquals( + collectorContextWithNoExpand.getCollectedTables().stream() + .map(TableIf::getName) + .collect(Collectors.toSet()), + expectedTablesWithNoExpand); + + TableCollectorContext mvCollectorContext = new TableCollector.TableCollectorContext( + Sets.newHashSet(TableType.MATERIALIZED_VIEW), true); + physicalPlan.accept(TableCollector.INSTANCE, mvCollectorContext); + Set expectedMvs = new HashSet<>(); + expectedMvs.add("mv1"); + Assertions.assertEquals( + mvCollectorContext.getCollectedTables().stream() + .map(TableIf::getName) + .collect(Collectors.toSet()), + expectedMvs); + + TableCollectorContext mvCollectorContextWithNoExpand = + new TableCollector.TableCollectorContext( + Sets.newHashSet(TableType.MATERIALIZED_VIEW), false); + physicalPlan.accept(TableCollector.INSTANCE, mvCollectorContextWithNoExpand); + Set expectedMvsWithNoExpand = new HashSet<>(); + expectedMvsWithNoExpand.add("mv1"); + Assertions.assertEquals( + mvCollectorContextWithNoExpand.getCollectedTables().stream() + .map(TableIf::getName) + .collect(Collectors.toSet()), + expectedMvsWithNoExpand); }); + dropMvByNereids("drop materialized view mv1"); } @Test