diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 41cfbb659a9ad62..046686255c14892 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -446,7 +446,7 @@ private PhysicalPlan chooseBestPlan(Group rootGroup, PhysicalProperties physical GroupExpression groupExpression = rootGroup.getLowestCostPlan(physicalProperties).orElseThrow( () -> new AnalysisException("lowestCostPlans with physicalProperties(" + physicalProperties + ") doesn't exist in root group")).second; - if (rootGroup.getEnforcers().contains(groupExpression)) { + if (rootGroup.getEnforcers().containsKey(groupExpression)) { rootGroup.addChosenEnforcerId(groupExpression.getId().asInt()); rootGroup.addChosenEnforcerProperties(physicalProperties); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java index eb4f86bb0cabbb2..18b9b86dae798ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java @@ -74,6 +74,11 @@ public final void execute() throws AnalysisException { GroupExpressionMatching groupExpressionMatching = new GroupExpressionMatching(rule.getPattern(), groupExpression); for (Plan plan : groupExpressionMatching) { + if (rule.isExploration() + && context.getCascadesContext().getMemo().getGroupExpressionsSize() > context.getCascadesContext() + .getConnectContext().getSessionVariable().memoMaxGroupExpressionSize) { + break; + } List newPlans = rule.transform(plan, context.getCascadesContext()); for (Plan newPlan : newPlans) { if (newPlan == plan) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java index 2f1e34fae5a012a..a01db5546cb761c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java @@ -59,7 +59,7 @@ public class Group { private final List logicalExpressions = Lists.newArrayList(); private final List physicalExpressions = Lists.newArrayList(); - private final List enforcers = Lists.newArrayList(); + private final Map enforcers = Maps.newHashMap(); private boolean isStatsReliable = true; private LogicalProperties logicalProperties; @@ -239,10 +239,10 @@ public GroupExpression getBestPlan(PhysicalProperties properties) { public void addEnforcer(GroupExpression enforcer) { enforcer.setOwnerGroup(this); - enforcers.add(enforcer); + enforcers.put(enforcer, enforcer); } - public List getEnforcers() { + public Map getEnforcers() { return enforcers; } @@ -346,9 +346,9 @@ public void mergeTo(Group target) { parentExpressions.keySet().forEach(parent -> target.addParentExpression(parent)); // move enforcers Ownership - enforcers.forEach(ge -> ge.children().set(0, target)); + enforcers.forEach((k, v) -> k.children().set(0, target)); // TODO: dedup? - enforcers.forEach(enforcer -> target.addEnforcer(enforcer)); + enforcers.forEach((k, v) -> target.addEnforcer(k)); enforcers.clear(); // move LogicalExpression PhysicalExpression Ownership @@ -458,7 +458,7 @@ public String toString() { str.append(" ").append(physicalExpression).append("\n"); } str.append(" enforcers:\n"); - for (GroupExpression enforcer : enforcers) { + for (GroupExpression enforcer : enforcers.keySet()) { str.append(" ").append(enforcer).append("\n"); } if (!chosenEnforcerIdList.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java index 6c530c6aa2fd4ab..c34aa00a05379e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java @@ -552,8 +552,7 @@ public void mergeGroup(Group source, Group destination, HashMap pla return; } Group parentOwnerGroup = srcParent.getOwnerGroup(); - HashSet enforcers = new HashSet<>(parentOwnerGroup.getEnforcers()); - if (enforcers.contains(srcParent)) { + if (parentOwnerGroup.getEnforcers().containsKey(srcParent)) { continue; } needReplaceChild.add(srcParent); @@ -946,7 +945,7 @@ private List extractGroupExpressionSatisfyProp(Group group, Phy List exprs = Lists.newArrayList(bestExpr); Set hasVisited = new HashSet<>(); hasVisited.add(bestExpr); - Stream.concat(group.getPhysicalExpressions().stream(), group.getEnforcers().stream()) + Stream.concat(group.getPhysicalExpressions().stream(), group.getEnforcers().keySet().stream()) .forEach(groupExpression -> { if (!groupExpression.getInputPropertiesListOrEmpty(prop).isEmpty() && !groupExpression.equals(bestExpr) && !hasVisited.contains(groupExpression)) { @@ -969,7 +968,7 @@ private List> extractInputProperties(GroupExpression gr res.add(groupExpression.getInputPropertiesList(prop)); // return optimized input for enforcer - if (groupExpression.getOwnerGroup().getEnforcers().contains(groupExpression)) { + if (groupExpression.getOwnerGroup().getEnforcers().containsKey(groupExpression)) { return res; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rule.java index 207dd6458c9202e..7d5b4001d9ae8cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rule.java @@ -64,6 +64,10 @@ public boolean isRewrite() { return ruleType.getRuleTypeClass() == RuleTypeClass.REWRITE; } + public boolean isExploration() { + return ruleType.getRuleTypeClass() == RuleTypeClass.EXPLORATION; + } + @Override public String toString() { return getRuleType().toString(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 7525d2a960d2221..f2c572f7779e916 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -384,29 +384,29 @@ public enum RuleType { EAGER_SPLIT(RuleTypeClass.EXPLORATION), EXPLORATION_SENTINEL(RuleTypeClass.EXPLORATION), - MATERIALIZED_VIEW_PROJECT_JOIN(RuleTypeClass.EXPLORATION), - MATERIALIZED_VIEW_FILTER_JOIN(RuleTypeClass.EXPLORATION), - MATERIALIZED_VIEW_PROJECT_FILTER_JOIN(RuleTypeClass.EXPLORATION), - MATERIALIZED_VIEW_FILTER_PROJECT_JOIN(RuleTypeClass.EXPLORATION), - MATERIALIZED_VIEW_ONLY_JOIN(RuleTypeClass.EXPLORATION), - - MATERIALIZED_VIEW_PROJECT_AGGREGATE(RuleTypeClass.EXPLORATION), - MATERIALIZED_VIEW_FILTER_AGGREGATE(RuleTypeClass.EXPLORATION), - MATERIALIZED_VIEW_PROJECT_FILTER_AGGREGATE(RuleTypeClass.EXPLORATION), - MATERIALIZED_VIEW_FILTER_PROJECT_AGGREGATE(RuleTypeClass.EXPLORATION), - MATERIALIZED_VIEW_ONLY_AGGREGATE(RuleTypeClass.EXPLORATION), - - MATERIALIZED_VIEW_PROJECT_AGGREGATE_ON_NONE_AGGREGATE(RuleTypeClass.EXPLORATION), - MATERIALIZED_VIEW_FILTER_AGGREGATE_ON_NONE_AGGREGATE(RuleTypeClass.EXPLORATION), - MATERIALIZED_VIEW_PROJECT_FILTER_AGGREGATE_ON_NONE_AGGREGATE(RuleTypeClass.EXPLORATION), - MATERIALIZED_VIEW_FILTER_PROJECT_AGGREGATE_ON_NONE_AGGREGATE(RuleTypeClass.EXPLORATION), - MATERIALIZED_VIEW_ONLY_AGGREGATE_ON_NONE_AGGREGATE(RuleTypeClass.EXPLORATION), - - MATERIALIZED_VIEW_FILTER_SCAN(RuleTypeClass.EXPLORATION), - MATERIALIZED_VIEW_PROJECT_SCAN(RuleTypeClass.EXPLORATION), - MATERIALIZED_VIEW_FILTER_PROJECT_SCAN(RuleTypeClass.EXPLORATION), - MATERIALIZED_VIEW_PROJECT_FILTER_SCAN(RuleTypeClass.EXPLORATION), - MATERIALIZED_VIEW_ONLY_SCAN(RuleTypeClass.EXPLORATION), + MATERIALIZED_VIEW_PROJECT_JOIN(RuleTypeClass.MATERIALIZE_VIEW), + MATERIALIZED_VIEW_FILTER_JOIN(RuleTypeClass.MATERIALIZE_VIEW), + MATERIALIZED_VIEW_PROJECT_FILTER_JOIN(RuleTypeClass.MATERIALIZE_VIEW), + MATERIALIZED_VIEW_FILTER_PROJECT_JOIN(RuleTypeClass.MATERIALIZE_VIEW), + MATERIALIZED_VIEW_ONLY_JOIN(RuleTypeClass.MATERIALIZE_VIEW), + + MATERIALIZED_VIEW_PROJECT_AGGREGATE(RuleTypeClass.MATERIALIZE_VIEW), + MATERIALIZED_VIEW_FILTER_AGGREGATE(RuleTypeClass.MATERIALIZE_VIEW), + MATERIALIZED_VIEW_PROJECT_FILTER_AGGREGATE(RuleTypeClass.MATERIALIZE_VIEW), + MATERIALIZED_VIEW_FILTER_PROJECT_AGGREGATE(RuleTypeClass.MATERIALIZE_VIEW), + MATERIALIZED_VIEW_ONLY_AGGREGATE(RuleTypeClass.MATERIALIZE_VIEW), + + MATERIALIZED_VIEW_PROJECT_AGGREGATE_ON_NONE_AGGREGATE(RuleTypeClass.MATERIALIZE_VIEW), + MATERIALIZED_VIEW_FILTER_AGGREGATE_ON_NONE_AGGREGATE(RuleTypeClass.MATERIALIZE_VIEW), + MATERIALIZED_VIEW_PROJECT_FILTER_AGGREGATE_ON_NONE_AGGREGATE(RuleTypeClass.MATERIALIZE_VIEW), + MATERIALIZED_VIEW_FILTER_PROJECT_AGGREGATE_ON_NONE_AGGREGATE(RuleTypeClass.MATERIALIZE_VIEW), + MATERIALIZED_VIEW_ONLY_AGGREGATE_ON_NONE_AGGREGATE(RuleTypeClass.MATERIALIZE_VIEW), + + MATERIALIZED_VIEW_FILTER_SCAN(RuleTypeClass.MATERIALIZE_VIEW), + MATERIALIZED_VIEW_PROJECT_SCAN(RuleTypeClass.MATERIALIZE_VIEW), + MATERIALIZED_VIEW_FILTER_PROJECT_SCAN(RuleTypeClass.MATERIALIZE_VIEW), + MATERIALIZED_VIEW_PROJECT_FILTER_SCAN(RuleTypeClass.MATERIALIZE_VIEW), + MATERIALIZED_VIEW_ONLY_SCAN(RuleTypeClass.MATERIALIZE_VIEW), // implementation rules LOGICAL_ONE_ROW_RELATION_TO_PHYSICAL_ONE_ROW_RELATION(RuleTypeClass.IMPLEMENTATION), @@ -494,6 +494,7 @@ public Rule build( enum RuleTypeClass { REWRITE, EXPLORATION, + MATERIALIZE_VIEW, // This type is used for unit test only. CHECK, IMPLEMENTATION, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index 44b97086aa0f683..e3125dfa5bd51d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -36,6 +36,8 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableSet; @@ -46,6 +48,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; /** * Logical project plan. @@ -54,6 +57,7 @@ public class LogicalProject extends LogicalUnary projects; + private final Supplier> projectsSet; private final List excepts; private final boolean isDistinct; @@ -83,6 +87,7 @@ private LogicalProject(List projects, List exc this.projects = projects.isEmpty() ? ImmutableList.of(ExpressionUtils.selectMinimumColumn(child.get(0).getOutput())) : projects; + this.projectsSet = Suppliers.memoize(() -> ImmutableSet.copyOf(projects)); this.excepts = Utils.fastToImmutableList(excepts); this.isDistinct = isDistinct; } @@ -138,7 +143,7 @@ public boolean equals(Object o) { return false; } LogicalProject that = (LogicalProject) o; - boolean equal = projects.equals(that.projects) + boolean equal = projectsSet.get().equals(that.projectsSet.get()) && excepts.equals(that.excepts) && isDistinct == that.isDistinct; // TODO: should add exprId for UnBoundStar and BoundStar for equality comparison @@ -150,7 +155,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(projects); + return Objects.hash(projectsSet.get()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java index 02769e475248451..d34ba7206ea48ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java @@ -34,12 +34,16 @@ import org.apache.doris.statistics.Statistics; import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; /** * Physical project plan. @@ -47,6 +51,7 @@ public class PhysicalProject extends PhysicalUnary implements Project { private final List projects; + private final Supplier> projectsSet; //multiLayerProjects is used to extract common expressions // projects: (A+B) * 2, (A+B) * 3 // multiLayerProjects: @@ -62,6 +67,7 @@ public PhysicalProject(List projects, Optional LogicalProperties logicalProperties, CHILD_TYPE child) { super(PlanType.PHYSICAL_PROJECT, groupExpression, logicalProperties, child); this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, "projects can not be null")); + this.projectsSet = Suppliers.memoize(() -> ImmutableSet.copyOf(projects)); } public PhysicalProject(List projects, Optional groupExpression, @@ -70,6 +76,7 @@ public PhysicalProject(List projects, Optional super(PlanType.PHYSICAL_PROJECT, groupExpression, logicalProperties, physicalProperties, statistics, child); this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, "projects can not be null")); + this.projectsSet = Suppliers.memoize(() -> ImmutableSet.copyOf(projects)); } public List getProjects() { @@ -96,13 +103,13 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } - PhysicalProject that = (PhysicalProject) o; - return projects.equals(that.projects); + PhysicalProject that = (PhysicalProject) o; + return projectsSet.get().equals(that.projectsSet.get()); } @Override public int hashCode() { - return Objects.hash(projects); + return Objects.hash(projectsSet.get()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java index b3b3eb9e5f702dd..ba20c9267059f16 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java @@ -88,7 +88,7 @@ public R accept(PlanVisitor visitor, C context) { @Override public String toString() { - return Utils.toSqlString("PhysicalUnion" + getGroupIdWithPrefix(), + return Utils.toSqlString("PhysicalUnion" + "[" + id.asInt() + "]" + getGroupIdWithPrefix(), "qualifier", qualifier, "outputs", outputs, "regularChildrenOutputs", regularChildrenOutputs, @@ -98,7 +98,7 @@ public String toString() { @Override public PhysicalUnion withChildren(List children) { - return new PhysicalUnion(qualifier, outputs, regularChildrenOutputs, constantExprsList, + return new PhysicalUnion(qualifier, outputs, regularChildrenOutputs, constantExprsList, groupExpression, getLogicalProperties(), children); } @@ -119,7 +119,7 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr public PhysicalUnion withPhysicalPropertiesAndStats( PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalUnion(qualifier, outputs, regularChildrenOutputs, constantExprsList, - Optional.empty(), getLogicalProperties(), physicalProperties, statistics, children); + groupExpression, getLogicalProperties(), physicalProperties, statistics, children); } @Override