branch-3.1: [fix](nereids) stop merge project when generating huge expression #55293

fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/AnalysisException.java

@@ -23,35 +23,48 @@
 
 /** Nereids's AnalysisException. */
 public class AnalysisException extends RuntimeException {
+    private final ErrorCode errorCode;
     private final String message;
     private final Optional<Integer> line;
     private final Optional<Integer> startPosition;
     private final Optional<LogicalPlan> plan;
 
-    public AnalysisException(String message, Throwable cause, Optional<Integer> line,
+    /** Constructor of AnalysisException. */
+    public AnalysisException(ErrorCode errorCode, String message, Throwable cause, Optional<Integer> line,
             Optional<Integer> startPosition, Optional<LogicalPlan> plan) {
         super(message, cause);
+        this.errorCode = errorCode;
         this.message = message;
         this.line = line;
         this.startPosition = startPosition;
         this.plan = plan;
     }
 
-    public AnalysisException(String message, Optional<Integer> line,
+    /** Constructor of AnalysisException. */
+    public AnalysisException(ErrorCode errorCode, String message, Optional<Integer> line,
             Optional<Integer> startPosition, Optional<LogicalPlan> plan) {
         super(message);
+        this.errorCode = errorCode;
         this.message = message;
         this.line = line;
         this.startPosition = startPosition;
         this.plan = plan;
     }
 
+    public AnalysisException(ErrorCode errorCode, String message, Throwable cause) {
+        this(errorCode, message, cause, Optional.empty(), Optional.empty(), Optional.empty());
+    }
+
+    public AnalysisException(ErrorCode errorCode, String message) {
+        this(errorCode, message, Optional.empty(), Optional.empty(), Optional.empty());
+    }
+
     public AnalysisException(String message, Throwable cause) {
-        this(message, cause, Optional.empty(), Optional.empty(), Optional.empty());
+        this(ErrorCode.NONE, message, cause);
     }
 
     public AnalysisException(String message) {
-        this(message, Optional.empty(), Optional.empty(), Optional.empty());
+        this(ErrorCode.NONE, message);
     }
 
     @Override
@@ -70,5 +83,16 @@ private String getSimpleMessage() {
         }
     }
 
-    // TODO: support ErrorCode
+    /** get error code.
+     */
+    public ErrorCode getErrorCode() {
+        return errorCode;
+    }
+
+    /** error code enum.
+     */
+    public enum ErrorCode {
+        NONE,
+        EXPRESSION_EXCEEDS_LIMIT,
+    }
 }

fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/ParseException.java

@@ -37,7 +37,7 @@ public ParseException(String message) {
     }
 
     public ParseException(String message, Origin start, Optional<String> command) {
-        super(message, start.line, start.startPosition, Optional.empty());
+        super(ErrorCode.NONE, message, start.line, start.startPosition, Optional.empty());
         this.message = message;
         this.start = start;
         this.command = command;

fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/MergeProjectPostProcessor.java

@@ -23,6 +23,7 @@
 import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
 
 import java.util.List;
+import java.util.Optional;
 
 /**
  * merge consecutive projects
@@ -34,10 +35,12 @@ public PhysicalProject visitPhysicalProject(PhysicalProject<? extends Plan> proj
         project = (PhysicalProject<? extends Plan>) super.visit(project, ctx);
         Plan child = project.child();
         if (child instanceof PhysicalProject && project.canMergeProjections((PhysicalProject) child)) {
-            List<NamedExpression> projections = project.mergeProjections((PhysicalProject) child);
-            return (PhysicalProject) project
-                    .withProjectionsAndChild(projections, child.child(0))
-                    .copyStatsAndGroupIdFrom(project);
+            Optional<List<NamedExpression>> projections = project.mergeProjections((PhysicalProject) child);
+            if (projections.isPresent()) {
+                return (PhysicalProject) project
+                        .withProjectionsAndChild(projections.get(), child.child(0))
+                        .copyStatsAndGroupIdFrom(project);
+            }
         }
         return project;
     }

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java

@@ -187,8 +187,11 @@ public List<Rule> buildRules() {
                         ).when(project -> project.canMergeProjections(project.child().child()))).then(r -> {
                             LogicalProject<?> upperProject = r.child();
                             LogicalProject<LogicalOlapScan> bottomProject = r.child().child().child();
-                            List<NamedExpression> projections = upperProject.mergeProjections(bottomProject);
-                            LogicalProject<?> project = upperProject.withProjects(projections);
+                            Optional<List<NamedExpression>> projections = upperProject.mergeProjections(bottomProject);
+                            if (!projections.isPresent()) {
+                                return null;
+                            }
+                            LogicalProject<?> project = upperProject.withProjects(projections.get());
                             return deferMaterialize(r, r.child().child(), Optional.of(project),
                                     Optional.empty(), bottomProject.child());
                         })
@@ -246,8 +249,11 @@ public List<Rule> buildRules() {
                         ).when(project -> project.canMergeProjections(project.child().child()))).then(r -> {
                             LogicalProject<?> upperProject = r.child();
                             LogicalProject<LogicalFilter<LogicalOlapScan>> bottomProject = r.child().child().child();
-                            List<NamedExpression> projections = upperProject.mergeProjections(bottomProject);
-                            LogicalProject<?> project = upperProject.withProjects(projections);
+                            Optional<List<NamedExpression>> projections = upperProject.mergeProjections(bottomProject);
+                            if (!projections.isPresent()) {
+                                return null;
+                            }
+                            LogicalProject<?> project = upperProject.withProjects(projections.get());
                             LogicalFilter<LogicalOlapScan> filter = bottomProject.child();
                             return deferMaterialize(r, r.child().child(), Optional.of(project),
                                     Optional.of(filter), filter.child());

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java

@@ -24,6 +24,7 @@
 import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
 
 import java.util.List;
+import java.util.Optional;
 
 /**
  * this rule aims to merge consecutive project. For example:
@@ -43,13 +44,17 @@ public Rule build() {
         // here we just don't merge two projects if there is any window function
         return logicalProject(logicalProject())
                 .when(project -> project.canMergeProjections(project.child()))
-                .then(MergeProjects::mergeProjects).toRule(RuleType.MERGE_PROJECTS);
+                .then(MergeProjects::mergeProjects)
+                .toRule(RuleType.MERGE_PROJECTS);
     }
 
     /** merge projects */
     public static Plan mergeProjects(LogicalProject<?> project) {
         LogicalProject<? extends Plan> childProject = (LogicalProject<?>) project.child();
-        List<NamedExpression> projectExpressions = project.mergeProjections(childProject);
-        return project.withProjectsAndChild(projectExpressions, childProject.child(0));
+        Optional<List<NamedExpression>> projectExpressions = project.mergeProjections(childProject);
+        if (!projectExpressions.isPresent()) {
+            return project;
+        }
+        return project.withProjectsAndChild(projectExpressions.get(), childProject.child(0));
     }
 }

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java

@@ -21,6 +21,7 @@
 import org.apache.doris.nereids.analyzer.Unbound;
 import org.apache.doris.nereids.analyzer.UnboundVariable;
 import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.exceptions.AnalysisException.ErrorCode;
 import org.apache.doris.nereids.exceptions.UnboundException;
 import org.apache.doris.nereids.trees.AbstractTreeNode;
 import org.apache.doris.nereids.trees.expressions.ArrayItemReference.ArrayItemSlot;
@@ -179,12 +180,13 @@ protected Expression(List<Expression> children, boolean inferred) {
 
     private void checkLimit() {
         if (depth > Config.expr_depth_limit) {
-            throw new AnalysisException(String.format("Exceeded the maximum depth of an "
-                    + "expression tree (%s).", Config.expr_depth_limit));
+            throw new AnalysisException(ErrorCode.EXPRESSION_EXCEEDS_LIMIT,
+                    String.format("Exceeded the maximum depth of an expression tree (%s).", Config.expr_depth_limit));
         }
         if (width > Config.expr_children_limit) {
-            throw new AnalysisException(String.format("Exceeded the maximum children of an "
-                    + "expression tree (%s).", Config.expr_children_limit));
+            throw new AnalysisException(ErrorCode.EXPRESSION_EXCEEDS_LIMIT,
+                    String.format("Exceeded the maximum children of an expression tree (%s).",
+                            Config.expr_children_limit));
         }
     }
 

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java

@@ -26,12 +26,13 @@
 import org.apache.doris.nereids.util.ExpressionUtils;
 import org.apache.doris.nereids.util.PlanUtils;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 /**
  * Common interface for logical/physical project.
@@ -55,23 +56,30 @@ default Map<Slot, Expression> getAliasToProducer() {
     }
 
     /**
-     * combine upper level and bottom level projections
+     * combine upper level and bottom level projections.
+     * if the combined expressions too huge, will return empty.
      * 1. alias combination, for example
      * proj(x as y, b) --> proj(a as x, b, c) =>(a as y, b)
      * 2. remove used projection in bottom project
      * @param childProject bottom project
      * @return project list for merged project
      */
-    default List<NamedExpression> mergeProjections(Project childProject) {
-        List<NamedExpression> projects = new ArrayList<>();
-        projects.addAll(PlanUtils.mergeProjections(childProject.getProjects(), getProjects()));
+    default Optional<List<NamedExpression>> mergeProjections(Project childProject) {
+        Optional<List<NamedExpression>> parentProjectsOpt
+                = PlanUtils.tryMergeProjections(childProject.getProjects(), getProjects());
+        if (!parentProjectsOpt.isPresent()) {
+            return Optional.empty();
+        }
+        ImmutableList.Builder<NamedExpression> projectsBuilder
+                = ImmutableList.builderWithExpectedSize(parentProjectsOpt.get().size());
+        projectsBuilder.addAll(parentProjectsOpt.get());
         for (NamedExpression expression : childProject.getProjects()) {
             // keep NoneMovableFunction for later use
             if (expression.containsType(NoneMovableFunction.class)) {
-                projects.add(expression);
+                projectsBuilder.add(expression);
             }
         }
-        return projects;
+        return Optional.of(projectsBuilder.build());
     }
 
     /** check can merge two projects */

fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java

@@ -368,7 +368,7 @@ public static Optional<Slot> extractSlotOrCastOnSlot(Expression expr) {
     /**
      * Generate replaceMap Slot -> Expression from NamedExpression[Expression as name]
      */
-    public static Map<Slot, Expression> generateReplaceMap(List<NamedExpression> namedExpressions) {
+    public static Map<Slot, Expression> generateReplaceMap(List<? extends NamedExpression> namedExpressions) {
         Map<Slot, Expression> replaceMap = Maps.newLinkedHashMapWithExpectedSize(namedExpressions.size());
         for (NamedExpression namedExpression : namedExpressions) {
             if (namedExpression instanceof Alias) {
@@ -457,7 +457,7 @@ public static Set<Expression> replace(Set<Expression> exprs,
     /**
      * Replace expression node in the expression tree by `replaceMap` in top-down manner.
      */
-    public static List<NamedExpression> replaceNamedExpressions(List<NamedExpression> namedExpressions,
+    public static List<NamedExpression> replaceNamedExpressions(List<? extends NamedExpression> namedExpressions,
             Map<? extends Expression, ? extends Expression> replaceMap) {
         Builder<NamedExpression> replaceExprs = ImmutableList.builderWithExpectedSize(namedExpressions.size());
         for (NamedExpression namedExpression : namedExpressions) {

fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java

@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.util;
 
 import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -120,10 +121,26 @@ public static List<NamedExpression> adjustNullableForRepeat(
     }
 
     /**
-     * merge childProjects with parentProjects
+     * try merge childProjects with parentProjects. if merged expression exceeds limit, return empty.
      */
-    public static List<NamedExpression> mergeProjections(List<NamedExpression> childProjects,
-            List<NamedExpression> parentProjects) {
+    public static Optional<List<NamedExpression>> tryMergeProjections(List<? extends NamedExpression> childProjects,
+            List<? extends NamedExpression> parentProjects) {
+        try {
+            return Optional.of(mergeProjections(childProjects, parentProjects));
+        } catch (AnalysisException e) {
+            if (e.getErrorCode() == AnalysisException.ErrorCode.EXPRESSION_EXCEEDS_LIMIT) {
+                return Optional.empty();
+            } else {
+                throw e;
+            }
+        }
+    }
+
+    /**
+     * merge childProjects with parentProjects. if merged expression exceeds limit, will throw AnalysisException.
+     */
+    public static List<NamedExpression> mergeProjections(List<? extends NamedExpression> childProjects,
+            List<? extends NamedExpression> parentProjects) {
         Map<Slot, Expression> replaceMap = ExpressionUtils.generateReplaceMap(childProjects);
         return ExpressionUtils.replaceNamedExpressions(parentProjects, replaceMap);
     }

regression-test/data/nereids_rules_p0/merge_project/test_merge_project.out

@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !exceeds_expression_limit_shape --
+PhysicalResultSink
+--PhysicalProject
+----PhysicalProject
+------PhysicalProject
+--------PhysicalProject
+----------PhysicalOlapScan[tbl_test_merge_project]
+
+-- !exceeds_expression_limit_result --
+1024	1023
+512	511
+