diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/AnalysisException.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/AnalysisException.java index 18f89725e2f8cd..e8b5d142f8f102 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/AnalysisException.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/AnalysisException.java @@ -23,35 +23,48 @@ /** Nereids's AnalysisException. */ public class AnalysisException extends RuntimeException { + private final ErrorCode errorCode; private final String message; private final Optional line; private final Optional startPosition; private final Optional plan; - public AnalysisException(String message, Throwable cause, Optional line, + /** Constructor of AnalysisException. */ + public AnalysisException(ErrorCode errorCode, String message, Throwable cause, Optional line, Optional startPosition, Optional plan) { super(message, cause); + this.errorCode = errorCode; this.message = message; this.line = line; this.startPosition = startPosition; this.plan = plan; } - public AnalysisException(String message, Optional line, + /** Constructor of AnalysisException. */ + public AnalysisException(ErrorCode errorCode, String message, Optional line, Optional startPosition, Optional plan) { super(message); + this.errorCode = errorCode; this.message = message; this.line = line; this.startPosition = startPosition; this.plan = plan; } + public AnalysisException(ErrorCode errorCode, String message, Throwable cause) { + this(errorCode, message, cause, Optional.empty(), Optional.empty(), Optional.empty()); + } + + public AnalysisException(ErrorCode errorCode, String message) { + this(errorCode, message, Optional.empty(), Optional.empty(), Optional.empty()); + } + public AnalysisException(String message, Throwable cause) { - this(message, cause, Optional.empty(), Optional.empty(), Optional.empty()); + this(ErrorCode.NONE, message, cause); } public AnalysisException(String message) { - this(message, Optional.empty(), Optional.empty(), Optional.empty()); + this(ErrorCode.NONE, message); } @Override @@ -70,5 +83,16 @@ private String getSimpleMessage() { } } - // TODO: support ErrorCode + /** get error code. + */ + public ErrorCode getErrorCode() { + return errorCode; + } + + /** error code enum. + */ + public enum ErrorCode { + NONE, + EXPRESSION_EXCEEDS_LIMIT, + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/ParseException.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/ParseException.java index 697af739f89963..e3c72f1d6e7f49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/ParseException.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/ParseException.java @@ -37,7 +37,7 @@ public ParseException(String message) { } public ParseException(String message, Origin start, Optional command) { - super(message, start.line, start.startPosition, Optional.empty()); + super(ErrorCode.NONE, message, start.line, start.startPosition, Optional.empty()); this.message = message; this.start = start; this.command = command; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/MergeProjectPostProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/MergeProjectPostProcessor.java index 7466889fde64ab..b1f4284fb42ef1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/MergeProjectPostProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/MergeProjectPostProcessor.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import java.util.List; +import java.util.Optional; /** * merge consecutive projects @@ -34,10 +35,12 @@ public PhysicalProject visitPhysicalProject(PhysicalProject proj project = (PhysicalProject) super.visit(project, ctx); Plan child = project.child(); if (child instanceof PhysicalProject && project.canMergeProjections((PhysicalProject) child)) { - List projections = project.mergeProjections((PhysicalProject) child); - return (PhysicalProject) project - .withProjectionsAndChild(projections, child.child(0)) - .copyStatsAndGroupIdFrom(project); + Optional> projections = project.mergeProjections((PhysicalProject) child); + if (projections.isPresent()) { + return (PhysicalProject) project + .withProjectionsAndChild(projections.get(), child.child(0)) + .copyStatsAndGroupIdFrom(project); + } } return project; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java index 765081ae016afe..18ff8bfeb2f907 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java @@ -187,8 +187,11 @@ public List buildRules() { ).when(project -> project.canMergeProjections(project.child().child()))).then(r -> { LogicalProject upperProject = r.child(); LogicalProject bottomProject = r.child().child().child(); - List projections = upperProject.mergeProjections(bottomProject); - LogicalProject project = upperProject.withProjects(projections); + Optional> projections = upperProject.mergeProjections(bottomProject); + if (!projections.isPresent()) { + return null; + } + LogicalProject project = upperProject.withProjects(projections.get()); return deferMaterialize(r, r.child().child(), Optional.of(project), Optional.empty(), bottomProject.child()); }) @@ -246,8 +249,11 @@ public List buildRules() { ).when(project -> project.canMergeProjections(project.child().child()))).then(r -> { LogicalProject upperProject = r.child(); LogicalProject> bottomProject = r.child().child().child(); - List projections = upperProject.mergeProjections(bottomProject); - LogicalProject project = upperProject.withProjects(projections); + Optional> projections = upperProject.mergeProjections(bottomProject); + if (!projections.isPresent()) { + return null; + } + LogicalProject project = upperProject.withProjects(projections.get()); LogicalFilter filter = bottomProject.child(); return deferMaterialize(r, r.child().child(), Optional.of(project), Optional.of(filter), filter.child()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java index bb6ca154136a63..572ff88ea598df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import java.util.List; +import java.util.Optional; /** * this rule aims to merge consecutive project. For example: @@ -43,13 +44,17 @@ public Rule build() { // here we just don't merge two projects if there is any window function return logicalProject(logicalProject()) .when(project -> project.canMergeProjections(project.child())) - .then(MergeProjects::mergeProjects).toRule(RuleType.MERGE_PROJECTS); + .then(MergeProjects::mergeProjects) + .toRule(RuleType.MERGE_PROJECTS); } /** merge projects */ public static Plan mergeProjects(LogicalProject project) { LogicalProject childProject = (LogicalProject) project.child(); - List projectExpressions = project.mergeProjections(childProject); - return project.withProjectsAndChild(projectExpressions, childProject.child(0)); + Optional> projectExpressions = project.mergeProjections(childProject); + if (!projectExpressions.isPresent()) { + return project; + } + return project.withProjectsAndChild(projectExpressions.get(), childProject.child(0)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java index 90cdfda818d05f..707bf05bb8e84d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.analyzer.Unbound; import org.apache.doris.nereids.analyzer.UnboundVariable; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.exceptions.AnalysisException.ErrorCode; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.trees.AbstractTreeNode; import org.apache.doris.nereids.trees.expressions.ArrayItemReference.ArrayItemSlot; @@ -179,12 +180,13 @@ protected Expression(List children, boolean inferred) { private void checkLimit() { if (depth > Config.expr_depth_limit) { - throw new AnalysisException(String.format("Exceeded the maximum depth of an " - + "expression tree (%s).", Config.expr_depth_limit)); + throw new AnalysisException(ErrorCode.EXPRESSION_EXCEEDS_LIMIT, + String.format("Exceeded the maximum depth of an expression tree (%s).", Config.expr_depth_limit)); } if (width > Config.expr_children_limit) { - throw new AnalysisException(String.format("Exceeded the maximum children of an " - + "expression tree (%s).", Config.expr_children_limit)); + throw new AnalysisException(ErrorCode.EXPRESSION_EXCEEDS_LIMIT, + String.format("Exceeded the maximum children of an expression tree (%s).", + Config.expr_children_limit)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java index 33b351ba4b31ad..eeacd02468debf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java @@ -26,12 +26,13 @@ import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.PlanUtils; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Optional; /** * Common interface for logical/physical project. @@ -55,23 +56,30 @@ default Map getAliasToProducer() { } /** - * combine upper level and bottom level projections + * combine upper level and bottom level projections. + * if the combined expressions too huge, will return empty. * 1. alias combination, for example * proj(x as y, b) --> proj(a as x, b, c) =>(a as y, b) * 2. remove used projection in bottom project * @param childProject bottom project * @return project list for merged project */ - default List mergeProjections(Project childProject) { - List projects = new ArrayList<>(); - projects.addAll(PlanUtils.mergeProjections(childProject.getProjects(), getProjects())); + default Optional> mergeProjections(Project childProject) { + Optional> parentProjectsOpt + = PlanUtils.tryMergeProjections(childProject.getProjects(), getProjects()); + if (!parentProjectsOpt.isPresent()) { + return Optional.empty(); + } + ImmutableList.Builder projectsBuilder + = ImmutableList.builderWithExpectedSize(parentProjectsOpt.get().size()); + projectsBuilder.addAll(parentProjectsOpt.get()); for (NamedExpression expression : childProject.getProjects()) { // keep NoneMovableFunction for later use if (expression.containsType(NoneMovableFunction.class)) { - projects.add(expression); + projectsBuilder.add(expression); } } - return projects; + return Optional.of(projectsBuilder.build()); } /** check can merge two projects */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 5c5a4e8d06002c..f05060d58a7a6f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -368,7 +368,7 @@ public static Optional extractSlotOrCastOnSlot(Expression expr) { /** * Generate replaceMap Slot -> Expression from NamedExpression[Expression as name] */ - public static Map generateReplaceMap(List namedExpressions) { + public static Map generateReplaceMap(List namedExpressions) { Map replaceMap = Maps.newLinkedHashMapWithExpectedSize(namedExpressions.size()); for (NamedExpression namedExpression : namedExpressions) { if (namedExpression instanceof Alias) { @@ -457,7 +457,7 @@ public static Set replace(Set exprs, /** * Replace expression node in the expression tree by `replaceMap` in top-down manner. */ - public static List replaceNamedExpressions(List namedExpressions, + public static List replaceNamedExpressions(List namedExpressions, Map replaceMap) { Builder replaceExprs = ImmutableList.builderWithExpectedSize(namedExpressions.size()); for (NamedExpression namedExpression : namedExpressions) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java index 9ab1cf0ae1c72f..0e0a39d01c5558 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.util; import org.apache.doris.catalog.TableIf; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -120,10 +121,26 @@ public static List adjustNullableForRepeat( } /** - * merge childProjects with parentProjects + * try merge childProjects with parentProjects. if merged expression exceeds limit, return empty. */ - public static List mergeProjections(List childProjects, - List parentProjects) { + public static Optional> tryMergeProjections(List childProjects, + List parentProjects) { + try { + return Optional.of(mergeProjections(childProjects, parentProjects)); + } catch (AnalysisException e) { + if (e.getErrorCode() == AnalysisException.ErrorCode.EXPRESSION_EXCEEDS_LIMIT) { + return Optional.empty(); + } else { + throw e; + } + } + } + + /** + * merge childProjects with parentProjects. if merged expression exceeds limit, will throw AnalysisException. + */ + public static List mergeProjections(List childProjects, + List parentProjects) { Map replaceMap = ExpressionUtils.generateReplaceMap(childProjects); return ExpressionUtils.replaceNamedExpressions(parentProjects, replaceMap); } diff --git a/regression-test/data/nereids_rules_p0/merge_project/test_merge_project.out b/regression-test/data/nereids_rules_p0/merge_project/test_merge_project.out new file mode 100644 index 00000000000000..56c097d00f8310 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/merge_project/test_merge_project.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !exceeds_expression_limit_shape -- +PhysicalResultSink +--PhysicalProject +----PhysicalProject +------PhysicalProject +--------PhysicalProject +----------PhysicalOlapScan[tbl_test_merge_project] + +-- !exceeds_expression_limit_result -- +1024 1023 +512 511 + diff --git a/regression-test/suites/nereids_rules_p0/merge_project/test_merge_project.groovy b/regression-test/suites/nereids_rules_p0/merge_project/test_merge_project.groovy new file mode 100644 index 00000000000000..f522dcdfa7dbdc --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/merge_project/test_merge_project.groovy @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('test_merge_project', 'nonConcurrent') { + setFeConfigTemporary([expr_children_limit : 200]) { + def tbl = 'tbl_test_merge_project' + multi_sql """ + SET ignore_shape_nodes='PhysicalDistribute'; + drop table if exists ${tbl} force; + create table ${tbl} (a int, b int) properties('replication_num' = '1'); + insert into ${tbl} values (2, 1), (4, 3); + """ + + explainAndOrderResult 'exceeds_expression_limit', """ + select + case + when k15 > k16 then k15 + k16 + 1 + when k15 > k16 - 1 then k15 + k16 - 1 + when k15 > k16 - 2 then k15 + k16 - 2 + else 0 + end as k17, + k15 + k16 as k18 + from + (select + case + when k13 > k14 then k13 + k14 + 1 + when k13 > k14 - 1 then k13 + k14 - 1 + when k13 > k14 - 2 then k13 + k14 - 2 + else 0 + end as k15, + k13 + k14 as k16 + from + (select + case + when k11 > k12 then k11 + k12 + 1 + when k11 > k12 - 1 then k11 + k12 - 1 + when k11 > k12 - 2 then k11 + k12 - 2 + else 0 + end as k13, + k11 + k12 as k14 + from + (select + case + when k9 > k10 then k9 + k10 + 1 + when k9 > k10 - 1 then k9 + k10 - 1 + when k9 > k10 - 2 then k9 + k10 - 2 + else 0 + end as k11, + k9 + k10 as k12 + from + (select + case + when k7 > k8 then k7 + k8 + 1 + when k7 > k8 - 1 then k7 + k8 - 1 + when k7 > k8 - 2 then k7 + k8 - 2 + else 0 + end as k9, + k7 + k8 as k10 + from + (select + case + when k5 > k6 then k5 + k6 + 1 + when k5 > k6 - 1 then k5 + k6 - 1 + when k5 > k6 - 2 then k5 + k6 - 2 + else 0 + end as k7, + k5 + k6 as k8 + from + (select + case + when k3 > k4 then k3 + k4 + 1 + when k3 > k4 - 1 then k3 + k4 - 1 + when k3 > k4 - 2 then k3 + k4 - 2 + else 0 + end as k5, + k3 + k4 as k6 + from + (select + case + when k1 > k2 then k1 + k2 + 1 + when k1 > k2 - 1 then k1 + k2 - 1 + when k1 > k2 - 2 then k1 + k2 - 2 + else 0 + end as k3, + k1 + k2 as k4 + from + (select a as k1, b as k2 + from ${tbl}) t1) t2) t3) t4) t5) t6) t7) t8; + """ + } +}