diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableSnapshot.java index efae08263c55eb..d5a904dc950923 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableSnapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableSnapshot.java @@ -68,4 +68,12 @@ public String toString() { return " FOR TIME AS OF '" + value + "'"; } } + + public String toDigest() { + if (this.type == VersionType.VERSION) { + return " FOR VERSION AS OF " + '?'; + } else { + return " FOR TIME AS OF '" + '?' + "'"; + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundAlias.java index 692f8f28318044..232c705df98e31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundAlias.java @@ -104,6 +104,16 @@ public String toString() { return stringBuilder.toString(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(child().toDigest()); + if (alias.isPresent()) { + sb.append(" AS " + alias.get()); + } + return sb.toString(); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitUnboundAlias(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java index 9b46293086ad73..deca7ebcd98463 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java @@ -128,6 +128,20 @@ public String toString() { return "'" + getName() + "(" + (isDistinct ? "distinct " : "") + params + ")"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(getName().toUpperCase()); + sb.append("("); + sb.append(isDistinct ? "distinct " : ""); + sb.append( + children.stream().map(Expression::toDigest) + .collect(Collectors.joining(", ")) + ); + sb.append(")"); + return sb.toString(); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitUnboundFunction(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundInlineTable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundInlineTable.java index 42d637d676fae2..3a6e4be2ec877b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundInlineTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundInlineTable.java @@ -84,4 +84,10 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr public List computeOutput() { throw new UnboundException("output"); } + + @Override + public String toDigest() { + // TODO handle exprList? + return "VALUES ?"; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java index bb61bc93574208..1bee39655f20cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java @@ -37,6 +37,7 @@ import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; /** * A relation that contains only one row consist of some constant expressions. @@ -108,4 +109,15 @@ public String toString() { "projects", projects ); } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT "); + sb.append( + projects.stream().map(Expression::toDigest) + .collect(Collectors.joining(", ")) + ); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index 3ba9aab002677f..0006782362a1a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -42,6 +42,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; /** * Represent a relation plan node that has not been bound. @@ -181,6 +182,27 @@ public String toString() { } @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + if (nameParts.size() > 0) { + sb.append(nameParts.stream().collect(Collectors.joining("."))); + sb.append(" "); + } + if (indexName.isPresent()) { + sb.append("INDEX ").append(indexName.get()).append(" "); + } + if (tabletIds.size() > 0) { + sb.append("TABLET(?)").append(" "); + } + if (scanParams != null) { + sb.append("@").append(scanParams.getParamType()).append(" "); + } + if (tableSnapshot.isPresent()) { + sb.append(tableSnapshot.get().toDigest()).append(" "); + } + return sb.substring(0, sb.length() - 1); + } + public R accept(PlanVisitor visitor, C context) { return visitor.visitUnboundRelation(this, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java index e6fd3eedf0df88..747049d3d8c634 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java @@ -91,6 +91,11 @@ public String toString() { return Utils.toSqlString("UnboundResultSink[" + id.asInt() + "]"); } + @Override + public String toDigest() { + return child().toDigest(); + } + @Override public StmtType stmtType() { return StmtType.SELECT; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java index 3c5608f4f288dd..7d48fba1dd46c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java @@ -70,6 +70,11 @@ public String getName() { }).reduce((left, right) -> left + "." + right).orElse(""); } + @Override + public String toDigest() { + return computeToSql(); + } + @Override public List getQualifier() { return nameParts.subList(0, nameParts.size() - 1); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java index 33ea9dafc36561..79b880450a43d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java @@ -66,6 +66,17 @@ public String toString() { return toSql(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + if (qualifier.isEmpty()) { + sb.append("*"); + } else { + sb.append(qualifier.get(0) + ".*"); + } + return sb.toString(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java index 3024058edc7a5d..ee8212b9049540 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java @@ -107,4 +107,11 @@ public String toString() { "arguments", properties ); } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(functionName); + sb.append("(?)"); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java index f2b6821c8043cf..b9290c796c2406 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java @@ -196,4 +196,15 @@ public String toString() { "colNames", colNames, "hints", hints); } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("INSERT INTO ").append(StringUtils.join(nameParts, ".")); + if (colNames != null && colNames.size() > 0) { + sb.append(" (").append(StringUtils.join(colNames, ", ")).append(")"); + } + sb.append(" ").append(child().toDigest()); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundVariable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundVariable.java index 340008bcc0a2de..66e19e486c124a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundVariable.java @@ -80,6 +80,11 @@ public String toString() throws UnboundException { } } + @Override + public String toDigest() { + return toString(); + } + /** * variable type */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 7c981bc1c5570e..beb4160e482eb1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -3005,6 +3005,13 @@ protected LogicalPlan withSelectQuerySpecification( List orderKeys = Lists.newArrayList(); LogicalPlan aggregate = withAggregate(filter, selectColumnCtx, aggClause, orderKeys); boolean isDistinct = (selectClause.DISTINCT() != null); + if (!isDistinct) { + if (aggregate instanceof LogicalRepeat) { + aggregate = ((LogicalRepeat) aggregate).withInProjection(false); + } else if (aggregate instanceof LogicalAggregate) { + aggregate = ((LogicalAggregate) aggregate).withInProjection(false); + } + } LogicalPlan selectPlan; if (!(aggregate instanceof Aggregate) && havingClause.isPresent()) { // create a project node for pattern match of ProjectToGlobalAggregate rule diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java index e3f06c7d1ddad7..1e0654aec46e1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.properties; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import java.util.Objects; @@ -78,6 +79,23 @@ public String toString() { return expr.toString() + (isAsc ? " asc" : " desc") + (nullFirst ? " null first" : ""); } + /** + * generate digest for order by key + * + * @return String + */ + public String toDigest() { + StringBuilder sb = new StringBuilder(); + if (expr instanceof Literal) { + sb.append(expr.toString()); + } else { + sb.append(expr.toDigest()); + } + sb.append(isAsc ? " ASC" : " DESC"); + sb.append(nullFirst ? " NULLS FIRST" : ""); + return sb.toString(); + } + @Override public int hashCode() { return Objects.hash(expr, isAsc, nullFirst); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java index 2c5decb9f9a1fc..5edbec493baaaa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java @@ -357,4 +357,8 @@ default boolean deepEquals(TreeNode that) { // If the "that" tree hasn't been fully traversed, return false. return thatDeque.isEmpty(); } + + default String toDigest() { + return ""; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java index 5bd084870258f9..4f0ca0cd909029 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.expressions; import org.apache.doris.nereids.exceptions.UnboundException; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; @@ -148,6 +149,17 @@ public String toString() { return child().toString() + " AS `" + name.get() + "`#" + exprId; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + if (child() instanceof Literal) { + sb.append("?"); + } else { + sb.append(child().toDigest()).append(" AS ").append(getName()); + } + return sb.toString(); + } + @Override public Alias withChildren(List children) { Preconditions.checkArgument(children.size() == 1); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java index f699e7531f6207..e51d16f57a3085 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java @@ -59,6 +59,14 @@ public String toString() { } @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(left().toDigest()); + sb.append(" ").append(symbol).append(" "); + sb.append(right().toDigest()); + return sb.toString(); + } + public String shapeInfo() { return "(" + left().shapeInfo() + " " + symbol + " " + right().shapeInfo() + ")"; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java index 0c3687f57153f2..cf30185998716f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java @@ -110,6 +110,20 @@ public String toString() { return output.toString(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder("CASE"); + for (Expression child : children()) { + if (child instanceof WhenClause) { + sb.append(child.toDigest()); + } else { + sb.append(" ELSE ").append(child.toDigest()); + } + } + sb.append(" END"); + return sb.toString(); + } + @Override public String computeToSql() throws UnboundException { StringBuilder output = new StringBuilder("CASE"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java index d38e18b0e61002..127cdabb001d75 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java @@ -103,6 +103,17 @@ public String toString() { return "cast(" + child() + " as " + targetType + ")"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("cast(") + .append(child().toDigest()) + .append(" as ") + .append(targetType) + .append(")"); + return sb.toString(); + } + @Override public boolean equals(Object o) { if (!super.equals(o)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CompoundPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CompoundPredicate.java index 3e04ee53d90d96..368a870c3b671a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CompoundPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CompoundPredicate.java @@ -121,6 +121,15 @@ public String toString() { } @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("("); + sb.append(children().stream().map(c -> c.toDigest()) + .collect(Collectors.joining(" " + symbol + " "))); + sb.append(")"); + return sb.toString(); + } + public String shapeInfo() { StringBuilder sb = new StringBuilder(); children().forEach(c -> sb.append(c.shapeInfo()).append(",")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/DefaultValueSlot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/DefaultValueSlot.java index a66428dc982708..56d7790fd92c6c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/DefaultValueSlot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/DefaultValueSlot.java @@ -64,6 +64,11 @@ public String toString() { return "DEFAULT_VALUE"; } + @Override + public String toDigest() { + return "?"; + } + public Slot withIndexInSql(Pair index) { return this; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java index c95882a71e2a56..80989377d69fc4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java @@ -75,6 +75,16 @@ public String toString() { return "EXISTS (SUBQUERY) " + super.toString(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(isNot ? "NOT " : "") + .append("EXISTS (") + .append(queryPlan.toDigest()) + .append(")"); + return sb.toString(); + } + public R accept(ExpressionVisitor visitor, C context) { return visitor.visitExistsSubquery(this, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java index 98cdca653683cc..4bade5707dd0a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java @@ -179,6 +179,31 @@ public String toString() { .collect(Collectors.joining(", ", "(", ")")); } + private boolean isLiteralOptions() { + for (Expression option : options) { + if (!(option instanceof Literal)) { + return false; + } + } + return true; + } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(compareExpr.toDigest()); + sb.append(" IN "); + if (isLiteralOptions()) { + sb.append("(?)"); + } else { + sb.append( + options.stream().map(Expression::toDigest) + .collect(Collectors.joining(", ", "(", ")")) + ); + } + return sb.toString(); + } + @Override public String computeToSql() { return compareExpr.toSql() + " IN " + options.stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java index 6b77700a4c8d49..20d327dcab13f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java @@ -78,6 +78,16 @@ public String toString() { return this.child() + " IN (INSUBQUERY) " + super.toString(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(getCompareExpr().toDigest()); + sb.append(isNot ? " NOT IN (" : " IN ("); + sb.append(queryPlan.toDigest()); + sb.append(')'); + return sb.toString(); + } + public R accept(ExpressionVisitor visitor, C context) { return visitor.visitInSubquery(this, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java index a5ea5fd9495ff8..a587ac6f8e2966 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java @@ -64,6 +64,14 @@ public String toString() { return child().toString() + " IS NULL"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(child().toDigest()); + sb.append(" IS NULL"); + return sb.toString(); + } + @Override public boolean equals(Object o) { if (!super.equals(o)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Not.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Not.java index 25602f08e8ab00..12c0252d3a302f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Not.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Not.java @@ -101,6 +101,13 @@ public String toString() { return "( not " + child().toString() + ")"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("NOT ").append(child().toDigest()); + return sb.toString(); + } + @Override public String computeToSql() { return "( not " + child().toSql() + ")"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/OrderExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/OrderExpression.java index b0564b7073761e..dffac46e5b96e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/OrderExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/OrderExpression.java @@ -80,6 +80,11 @@ public String toString() { return orderKey.toString(); } + @Override + public String toDigest() { + return orderKey.toDigest(); + } + @Override public String computeToSql() { return orderKey.toSql(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java index caa3fd07e30322..ce92a0e36c8249 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java @@ -66,6 +66,10 @@ public String toString() { return "$" + placeholderId.asInt(); } + public String toDigest() { + return "?"; + } + @Override public String computeToSql() { return "?"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java index 09dc6a5c216178..afffa324424a0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java @@ -69,6 +69,11 @@ public String toString() { return "Properties(" + toSql() + ")"; } + @Override + public String toDigest() { + return "?"; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ScalarSubquery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ScalarSubquery.java index ed608135254972..6e608ee5e4d11e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ScalarSubquery.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ScalarSubquery.java @@ -113,6 +113,15 @@ public String toString() { return " (SCALARSUBQUERY) " + super.toString(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("(") + .append(queryPlan.toDigest()) + .append(")"); + return sb.toString(); + } + public R accept(ExpressionVisitor visitor, C context) { return visitor.visitScalarSubquery(this, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StringRegexPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StringRegexPredicate.java index 5a62be54f93d81..21833fe8780ee3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StringRegexPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StringRegexPredicate.java @@ -64,6 +64,17 @@ public String toString() { return "(" + left() + " " + getName() + " " + right() + ")"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(left().toDigest()) + .append(' ') + .append(getName()) + .append(' ') + .append(right().toDigest()); + return sb.toString(); + } + public R accept(ExpressionVisitor visitor, C context) { return visitor.visitStringRegexPredicate(this, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Subtract.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Subtract.java index 8e80e5e673c42c..b569471162fd15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Subtract.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Subtract.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.ArithmeticExpr.Operator; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DecimalV3Type; @@ -63,4 +64,16 @@ public DataType getDataTypeForOthers(DataType t1, DataType t2) { public R accept(ExpressionVisitor visitor, C context) { return visitor.visitSubtract(this, context); } + + @Override + public String toDigest() { + if (left() instanceof IntegerLiteral) { + IntegerLiteral left = (IntegerLiteral) left(); + if (left.getValue() == 0) { + // nereids parser change - operator to subtract, so compactible with that + return " -" + right().toDigest(); + } + } + return super.toDigest(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TimestampArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TimestampArithmetic.java index 40a727eb1757ba..c9abc12a4145ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TimestampArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TimestampArithmetic.java @@ -128,6 +128,21 @@ public String toString() { return toSql(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + if (funcName != null) { + sb.append(funcName.toUpperCase()).append("("); + sb.append(child(0).toDigest()).append(", "); + sb.append(child(1).toDigest()).append(")"); + } else { + sb.append(child(0).toDigest()); + sb.append(" ").append(op.toString()).append(" "); + sb.append(child(1).toDigest()); + } + return sb.toString(); + } + @Override public String computeToSql() { StringBuilder strBuilder = new StringBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/UnaryOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/UnaryOperator.java index 299e00d0dc4fe4..e881e0584fe47a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/UnaryOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/UnaryOperator.java @@ -55,6 +55,14 @@ public String toString() { return "(" + symbol + " " + child().toString() + ")"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(" ").append(symbol).append(" "); + sb.append(child().toDigest()); + return sb.toString(); + } + @Override public int computeHashCode() { return Objects.hash(symbol, child()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WhenClause.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WhenClause.java index 6e10701faaad08..3a458fc8380a7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WhenClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WhenClause.java @@ -98,4 +98,11 @@ public int computeHashCode() { public String toString() { return " WHEN " + left().toString() + " THEN " + right().toString(); } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(" WHEN ").append(left().toDigest()).append(" THEN ").append(right().toDigest()); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java index 7c0f4651a42d97..2ea9ca3199ab4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java @@ -214,6 +214,24 @@ public String toString() { return sb.toString().trim() + ")"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(function.toDigest()).append(" OVER ("); + if (!partitionKeys.isEmpty()) { + sb.append("PARTITION BY ").append(partitionKeys.stream() + .map(Expression::toDigest) + .collect(Collectors.joining(", ", "", " "))); + } + if (!orderKeys.isEmpty()) { + sb.append("ORDER BY ").append(orderKeys.stream() + .map(OrderExpression::toDigest) + .collect(Collectors.joining(", ", "", " "))); + } + windowFrame.ifPresent(wf -> sb.append(" ").append(wf.toDigest())); + return sb.toString().trim() + ")"; + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitWindow(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java index 2c0d7d65b80a5f..b61b3ef1a43568 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java @@ -119,6 +119,18 @@ public String toString() { return sb.toString(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(frameUnits + " "); + if (rightBoundary != null) { + sb.append("BETWEEN " + leftBoundary.toDigest() + " AND " + rightBoundary.toDigest()); + } else { + sb.append(leftBoundary.toDigest()); + } + return sb.toString(); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitWindowFrame(this, context); @@ -215,6 +227,32 @@ public String toString() { return sb.toString(); } + /** to digest */ + public String toDigest() { + StringBuilder sb = new StringBuilder(); + boundOffset.ifPresent(value -> sb.append(value.toDigest()).append(" ")); + switch (frameBoundType) { + case UNBOUNDED_PRECEDING: + sb.append("UNBOUNDED PRECEDING"); + break; + case UNBOUNDED_FOLLOWING: + sb.append("UNBOUNDED FOLLOWING"); + break; + case CURRENT_ROW: + sb.append("CURRENT ROW"); + break; + case PRECEDING: + sb.append("PRECEDING"); + break; + case FOLLOWING: + sb.append("FOLLOWING"); + break; + default: + break; + } + return sb.toString(); + } + /** toSql*/ public String toSql() { StringBuilder sb = new StringBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BoundFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BoundFunction.java index cf936976e82c4d..b5df577dfecff2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BoundFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BoundFunction.java @@ -107,6 +107,17 @@ public String toString() { return getName() + "(" + args + ")"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(getName().toUpperCase()); + sb.append( + children().stream().map(Expression::toDigest) + .collect(Collectors.joining(", ", "(", ")")) + ); + return sb.toString(); + } + /** * checkOrderExprIsValid. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java index ce8ec70e765745..f9575eba49058c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java @@ -135,6 +135,20 @@ public String toString() { return getName() + "(" + (distinct ? "DISTINCT " : "") + args + ")"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(getName()).append("("); + if (distinct) { + sb.append("DISTINCT "); + } + sb.append( + children.stream().map(Expression::toDigest) + .collect(Collectors.joining(", ")) + ); + sb.append(")"); + return sb.toString(); + } + public boolean supportAggregatePhase(AggregatePhase aggregatePhase) { return true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Count.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Count.java index ba16b07ed5fafa..890860660518a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Count.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Count.java @@ -134,6 +134,14 @@ public String toString() { return super.toString(); } + @Override + public String toDigest() { + if (isStar) { + return "count(*)"; + } + return super.toDigest(); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitCount(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Lambda.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Lambda.java index 2ecab6090d8d3f..35463aa829bcae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Lambda.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Lambda.java @@ -153,6 +153,18 @@ public String toString() { return builder.toString(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + if (argumentNames.size() == 1) { + sb.append(argumentNames.get(0)); + } else { + sb.append(argumentNames.stream().collect(Collectors.joining(", ", "(", ")"))); + } + sb.append(" -> ").append(getLambdaFunction().toDigest()); + return sb.toString(); + } + @Override public Lambda withChildren(List children) { return new Lambda(argumentNames, children); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/ArrayLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/ArrayLiteral.java index be84a5b32e35cf..7b6a136a06559a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/ArrayLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/ArrayLiteral.java @@ -93,6 +93,13 @@ public String toString() { return "[" + items + "]"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("[?]"); + return sb.toString(); + } + @Override public String computeToSql() { String items = this.items.stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Interval.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Interval.java index 246122e59dbe5a..f1a404c336e31f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Interval.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Interval.java @@ -70,6 +70,15 @@ public R accept(ExpressionVisitor visitor, C context) { return visitor.visitInterval(this, context); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("INTERVAL "); + sb.append(value().toDigest()); + sb.append(" ").append(timeUnit); + return sb.toString(); + } + /** * Supported time unit. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index afa3b00a2bbbf2..8a8ded537c8a0b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -683,4 +683,9 @@ private static Literal handleVarcharLiteral(ByteBuffer data) { // different environment return new VarcharLiteral(new String(bytes, StandardCharsets.UTF_8)); } + + @Override + public String toDigest() { + return "?"; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java index ea805f6cb0ceb6..be5be3828ce9e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java @@ -126,4 +126,12 @@ public boolean showPlanProcess() { public StmtType stmtType() { return StmtType.EXPLAIN; } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("EXPLAIN "); + sb.append(logicalPlan.toDigest()); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java index d5e436f8c559a2..5ed5046a050954 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java @@ -61,6 +61,7 @@ import java.util.Objects; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; /** * EXPORT statement, export data to dirs by broker. @@ -410,5 +411,17 @@ public StmtType stmtType() { public boolean needAuditEncryption() { return true; } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder("EXPORT TABLE "); + sb.append(nameParts.stream().collect(Collectors.joining("."))); + if (expr.isPresent()) { + sb.append(" WHERE ") + .append(expr.get().toDigest()); + } + sb.append(" TO ?"); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java index 4fb42a21fd780d..b99eb39e296497 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java @@ -257,4 +257,11 @@ private List> reorderUnionData(List sinkE public StmtType stmtType() { return StmtType.INSERT; } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(originLogicalQuery.toDigest()); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index 407ef36d394811..88e83a6974600a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -558,4 +558,15 @@ public BuildInsertExecutorResult(NereidsPlanner planner, AbstractInsertExecutor this.physicalSink = physicalSink; } } + + @Override + public String toDigest() { + // if with cte, query will be print twice + StringBuilder sb = new StringBuilder(); + sb.append(originLogicalQuery.toDigest()); + if (cte.isPresent()) { + sb.append(" ").append(cte.get().toDigest()); + } + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java index 7ca08e6bc49eba..4619f2f6e321f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java @@ -430,4 +430,16 @@ public R accept(PlanVisitor visitor, C context) { public StmtType stmtType() { return StmtType.INSERT; } + + @Override + public String toDigest() { + // if with cte, query will be print twice + StringBuilder sb = new StringBuilder(); + sb.append("OVERWRITE TABLE "); // there is no way add overwrite flag in sink(logic query), so add it here + sb.append(originLogicalQuery.toDigest()); + if (cte.isPresent()) { + sb.append(" (").append(cte.get().toDigest()).append(")"); + } + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index fd224426c2cc6f..fdd5c59afef518 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.AggregatePhase; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.agg.Ndv; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; @@ -74,6 +75,7 @@ public class LogicalAggregate private final boolean ordinalIsResolved; private final boolean generated; private final boolean hasPushed; + private final boolean withInProjection; /** * Desc: Constructor for LogicalAggregate. @@ -90,19 +92,20 @@ public LogicalAggregate( * Distinct Agg */ public LogicalAggregate(List namedExpressions, boolean generated, CHILD_TYPE child) { - this(ImmutableList.copyOf(namedExpressions), namedExpressions, false, true, generated, false, Optional.empty(), + this(ImmutableList.copyOf(namedExpressions), namedExpressions, + false, true, generated, false, true, Optional.empty(), Optional.empty(), Optional.empty(), child); } public LogicalAggregate(List namedExpressions, boolean generated, boolean hasPushed, CHILD_TYPE child) { - this(ImmutableList.copyOf(namedExpressions), namedExpressions, false, true, generated, hasPushed, + this(ImmutableList.copyOf(namedExpressions), namedExpressions, false, true, generated, hasPushed, true, Optional.empty(), Optional.empty(), Optional.empty(), child); } public LogicalAggregate(List groupByExpressions, List outputExpressions, boolean ordinalIsResolved, CHILD_TYPE child) { - this(groupByExpressions, outputExpressions, false, ordinalIsResolved, false, false, Optional.empty(), + this(groupByExpressions, outputExpressions, false, ordinalIsResolved, false, false, true, Optional.empty(), Optional.empty(), Optional.empty(), child); } @@ -124,7 +127,7 @@ public LogicalAggregate( boolean normalized, Optional> sourceRepeat, CHILD_TYPE child) { - this(groupByExpressions, outputExpressions, normalized, false, false, false, sourceRepeat, + this(groupByExpressions, outputExpressions, normalized, false, false, false, true, sourceRepeat, Optional.empty(), Optional.empty(), child); } @@ -138,6 +141,7 @@ private LogicalAggregate( boolean ordinalIsResolved, boolean generated, boolean hasPushed, + boolean withInProjection, Optional> sourceRepeat, Optional groupExpression, Optional logicalProperties, @@ -154,6 +158,7 @@ private LogicalAggregate( this.generated = generated; this.hasPushed = hasPushed; this.sourceRepeat = Objects.requireNonNull(sourceRepeat, "sourceRepeat cannot be null"); + this.withInProjection = withInProjection; } @Override @@ -192,6 +197,32 @@ public String toString() { } @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + // org.apache.doris.nereids.parser.LogicalPlanBuilder.withProjection will generate different plan for + // distinct aggregation so use withInProjection flag to control whether to generate a select statement + // eg: select distinct type, id from tb_book group by type; + // select type, id from tb_book group by type; + if (!withInProjection) { + sb.append("SELECT "); + sb.append( + outputExpressions.stream().map(Expression::toDigest) + .collect(Collectors.joining(", ")) + ); + sb.append(" FROM "); + } + sb.append(child().toDigest()); + sb.append(" GROUP BY "); + sb.append( + groupByExpressions.stream() + .map(it -> + (it instanceof Literal) ? it.toString() : it.toDigest() + ) + .collect(Collectors.joining(", ")) + ); + return sb.toString(); + } + public List computeOutput() { ImmutableList.Builder outputSlots = ImmutableList.builderWithExpectedSize(outputExpressions.size()); for (NamedExpression outputExpression : outputExpressions) { @@ -248,13 +279,14 @@ public int hashCode() { public LogicalAggregate withChildren(List children) { Preconditions.checkArgument(children.size() == 1); return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, - hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), children.get(0)); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), children.get(0)); } @Override public LogicalAggregate withGroupExpression(Optional groupExpression) { return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, - hasPushed, sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), children.get(0)); + hasPushed, withInProjection, + sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), children.get(0)); } @Override @@ -262,37 +294,38 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr Optional logicalProperties, List children) { Preconditions.checkArgument(children.size() == 1); return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, - hasPushed, sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), children.get(0)); + hasPushed, withInProjection, + sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), children.get(0)); } public LogicalAggregate withGroupBy(List groupByExprList) { return new LogicalAggregate<>(groupByExprList, outputExpressions, normalized, ordinalIsResolved, generated, - hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), child()); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), child()); } public LogicalAggregate withGroupByAndOutput(List groupByExprList, List outputExpressionList) { return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, ordinalIsResolved, generated, - hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), child()); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), child()); } public LogicalAggregate withChildGroupByAndOutput(List groupByExprList, List outputExpressionList, Plan newChild) { return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, ordinalIsResolved, generated, - hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), newChild); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), newChild); } public LogicalAggregate withChildGroupByAndOutputAndSourceRepeat(List groupByExprList, List outputExpressionList, Plan newChild, Optional> sourceRepeat) { return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, ordinalIsResolved, generated, - hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), newChild); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), newChild); } public LogicalAggregate withChildAndOutput(CHILD_TYPE child, List outputExpressionList) { return new LogicalAggregate<>(groupByExpressions, outputExpressionList, normalized, ordinalIsResolved, - generated, hasPushed, sourceRepeat, Optional.empty(), + generated, hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), child); } @@ -304,23 +337,29 @@ public List getOutputs() { @Override public LogicalAggregate withAggOutput(List newOutput) { return new LogicalAggregate<>(groupByExpressions, newOutput, normalized, ordinalIsResolved, generated, - hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), child()); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), child()); } public LogicalAggregate withAggOutputChild(List newOutput, Plan newChild) { return new LogicalAggregate<>(groupByExpressions, newOutput, normalized, ordinalIsResolved, generated, - hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), newChild); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), newChild); } public LogicalAggregate withNormalized(List normalizedGroupBy, List normalizedOutput, Plan normalizedChild) { return new LogicalAggregate<>(normalizedGroupBy, normalizedOutput, true, ordinalIsResolved, generated, - hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), normalizedChild); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), normalizedChild); + } + + public LogicalAggregate withInProjection(boolean withInProjection) { + return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, + generated, hasPushed, withInProjection, + sourceRepeat, Optional.empty(), Optional.empty(), child()); } public LogicalAggregate withSourceRepeat(LogicalRepeat sourceRepeat) { return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, - generated, hasPushed, Optional.ofNullable(sourceRepeat), + generated, hasPushed, withInProjection, Optional.ofNullable(sourceRepeat), Optional.empty(), Optional.empty(), child()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java index 7a75dd5c1a4abf..4f810c3b6def8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; /** * Logical Node for CTE @@ -77,6 +78,19 @@ public String toString() { ); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("WITH\n"); + sb.append( + aliasQueries.stream().map(LogicalSubQueryAlias::toDigest) + .collect(Collectors.joining(", ")) + ); + sb.append("\n"); + sb.append(child().toDigest()); + return sb.toString(); + } + @Override public boolean displayExtraPlanFirst() { return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java index 31220fc13e18b8..87b6f510c64a81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java @@ -87,6 +87,11 @@ public String toString() { return Utils.toSqlString("LogicalCheckPolicy"); } + @Override + public String toDigest() { + return child().toDigest(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java index 65b3bf801d57d0..9fd44fcef21495 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java @@ -66,6 +66,15 @@ public String toString() { "regularChildrenOutputs", regularChildrenOutputs); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("(").append(child(0).toDigest()).append(")"); + sb.append(" EXCEPT ").append(qualifier).append(" "); + sb.append("(").append(child(1).toDigest()).append(")"); + return sb.toString(); + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitLogicalExcept(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java index f5e49cdba9e690..47a1ed6ff5bbb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java @@ -126,4 +126,14 @@ public Map getProperties() { public boolean needAuditEncryption() { return true; } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(child().toDigest()); + sb.append(" INTO OUTFILE '").append(" ? ").append(" FORMAT AS ").append(" ? "); + if (properties != null && !properties.isEmpty()) { + sb.append(" PROPERTIES(").append(" ? ").append(")"); + } + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java index efd7e90c13615e..c379a43bb7242d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java @@ -93,6 +93,18 @@ public String toString() { ); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(child().toDigest()); + sb.append(" WHERE "); + sb.append( + conjuncts.stream().map(Expression::toDigest) + .collect(Collectors.joining(" AND ")) + ); + return sb.toString(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java index 3dc0577779c699..9fb79809006cca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java @@ -37,6 +37,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; /** * plan for table generator, the statement like: SELECT * FROM tbl LATERAL VIEW EXPLODE(c1) g as (gc1); @@ -136,6 +137,27 @@ public String toString() { ); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + String generateName = ""; + try { + generateName = generatorOutput.get(0).getQualifier().get(0); + } catch (Throwable e) { + generateName = generatorOutput.get(0).toDigest(); + } + sb.append(child().toDigest()); + sb.append(" LATERAL VIEW ") + .append(generators.get(0).toDigest()) + .append(" ") + .append(generateName) + .append(" AS ") + .append( + expandColumnAlias.get(0).stream().collect(Collectors.joining(", ")) + ); + return sb.toString(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java index fedd2b9c62e008..b268fd265eb4cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java @@ -39,6 +39,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * Logical Having plan @@ -156,4 +157,16 @@ public void computeFd(Builder builder) { public String toString() { return Utils.toSqlString("LogicalHaving", "predicates", getPredicate()); } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(child().toDigest()); + sb.append(" HAVING "); + sb.append( + conjuncts.stream().map(Expression::toDigest) + .collect(Collectors.joining(" AND ")) + ); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java index 610a40ac96af3c..6f27da32b5043e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java @@ -66,6 +66,15 @@ public String toString() { "regularChildrenOutputs", regularChildrenOutputs); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("(").append(child(0).toDigest()).append(")"); + sb.append(" INTERSECT ").append(qualifier).append(" "); + sb.append("(").append(child(1).toDigest()).append(")"); + return sb.toString(); + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitLogicalIntersect(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java index 3cae93cd512c36..2f4cbe4681e2bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java @@ -272,6 +272,24 @@ public String toString() { return Utils.toSqlString("LogicalJoin[" + id.asInt() + "]", args.toArray()); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(left().toDigest()); + sb.append(" ").append(joinType).append(" "); + sb.append(right().toDigest()); + if (!hashJoinConjuncts.isEmpty() || !otherJoinConjuncts.isEmpty()) { + sb.append(" ON "); + sb.append( + hashJoinConjuncts.stream().map(Expression::toDigest).collect(Collectors.joining(" AND ")) + ); + sb.append( + otherJoinConjuncts.stream().map(Expression::toDigest).collect(Collectors.joining(" AND ")) + ); + } + return sb.toString(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java index 071e93ca48de45..39c878f62579fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java @@ -92,6 +92,17 @@ public String toString() { ); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(child().toDigest()); + sb.append(" LIMIT ? "); + if (offset != 0) { + sb.append(" OFFSET ?"); + } + return sb.toString(); + } + @Override public int hashCode() { return Objects.hash(limit, offset); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index d437319540df35..30563c4ba1df00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -52,6 +52,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * Logical project plan. @@ -127,6 +128,32 @@ public String toString() { ); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT "); + if (isDistinct) { + sb.append("DISTINCT "); + } + sb.append( + projects.stream().map(NamedExpression::toDigest) + .collect(Collectors.joining(", ")) + ); + if (!excepts.isEmpty()) { + sb.append(" EXCEPT("); + sb.append( + excepts.stream().map(Expression::toDigest) + .collect(Collectors.joining(", ")) + ); + sb.append(")"); + } + if (child().getType() != PlanType.LOGICAL_ONE_ROW_RELATION) { + sb.append(" FROM "); + } + sb.append(child().toDigest()); + return sb.toString(); + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitLogicalProject(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java index 6773c1c1a9641a..761845fbf9446d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java @@ -37,6 +37,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; /** * LogicalRepeat. @@ -49,6 +50,7 @@ public class LogicalRepeat extends LogicalUnary> groupingSets; private final List outputExpressions; + private final boolean withInProjection; /** * Desc: Constructor for LogicalRepeat. @@ -57,7 +59,7 @@ public LogicalRepeat( List> groupingSets, List outputExpressions, CHILD_TYPE child) { - this(groupingSets, outputExpressions, Optional.empty(), Optional.empty(), child); + this(groupingSets, outputExpressions, Optional.empty(), Optional.empty(), true, child); } /** @@ -65,6 +67,7 @@ public LogicalRepeat( */ public LogicalRepeat(List> groupingSets, List outputExpressions, Optional groupExpression, Optional logicalProperties, + boolean withInProjection, CHILD_TYPE child) { super(PlanType.LOGICAL_REPEAT, groupExpression, logicalProperties, child); this.groupingSets = Objects.requireNonNull(groupingSets, "groupingSets can not be null") @@ -73,6 +76,7 @@ public LogicalRepeat(List> groupingSets, List .collect(ImmutableList.toImmutableList()); this.outputExpressions = ImmutableList.copyOf( Objects.requireNonNull(outputExpressions, "outputExpressions can not be null")); + this.withInProjection = withInProjection; } @Override @@ -98,6 +102,36 @@ public String toString() { ); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + // org.apache.doris.nereids.parser.LogicalPlanBuilder.withProjection will generate different plan for + // distinct aggregation so use withInProjection flag to control whether to generate a select statement + // eg: select distinct log_time from example_tbl_duplicate group by log_time,log_type with rollup; + // select log_time from example_tbl_duplicate group by log_time,log_type with rollup; + if (!withInProjection) { + sb.append("SELECT "); + sb.append( + outputExpressions.stream().map(Expression::toDigest) + .collect(Collectors.joining(", ")) + ); + sb.append(" FROM "); + } + sb.append(child().toDigest()); + sb.append(" GROUP BY GROUPING SETS ("); + for (int i = 0; i < groupingSets.size(); i++) { + List groupingSet = groupingSets.get(i); + String subSet = groupingSet.stream().map(Expression::toDigest) + .collect(Collectors.joining(",", "(", ")")); + sb.append(subSet); + if (i != groupingSets.size() - 1) { + sb.append(", "); + } + } + sb.append(")"); + return sb.toString(); + } + @Override public List computeOutput() { return outputExpressions.stream() @@ -144,7 +178,7 @@ public LogicalRepeat withChildren(List children) { @Override public LogicalRepeat withGroupExpression(Optional groupExpression) { return new LogicalRepeat<>(groupingSets, outputExpressions, groupExpression, - Optional.of(getLogicalProperties()), child()); + Optional.of(getLogicalProperties()), withInProjection, child()); } @Override @@ -152,7 +186,7 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr Optional logicalProperties, List children) { Preconditions.checkArgument(children.size() == 1); return new LogicalRepeat<>(groupingSets, outputExpressions, groupExpression, logicalProperties, - children.get(0)); + withInProjection, children.get(0)); } public LogicalRepeat withGroupSets(List> groupingSets) { @@ -178,6 +212,11 @@ public LogicalRepeat withAggOutputAndChild(List newOutput return new LogicalRepeat<>(groupingSets, newOutput, child); } + public LogicalRepeat withInProjection(boolean withInProjection) { + return new LogicalRepeat<>(groupingSets, outputExpressions, + Optional.empty(), Optional.empty(), withInProjection, child()); + } + public boolean canBindVirtualSlot() { return bound() && outputExpressions.stream() .noneMatch(output -> output.containsType(VirtualSlotReference.class)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java index 127889ea7ed471..d043117645dfff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java @@ -113,4 +113,9 @@ public String toString() { .collect(Collectors.joining(", ")); return "LogicalSelectHint (" + hintStr + ")"; } + + @Override + public String toDigest() { + return child().toDigest(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java index 607fcf25bca7fe..45e0e081a3cdac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java @@ -37,6 +37,7 @@ import java.util.Objects; import java.util.Optional; import java.util.function.Supplier; +import java.util.stream.Collectors; /** * Logical Sort plan. @@ -89,6 +90,17 @@ public String toString() { "orderKeys", orderKeys); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(child().toDigest()).append(" ORDER BY "); + sb.append( + orderKeys.stream().map(OrderKey::toDigest) + .collect(Collectors.joining(", ")) + ); + return sb.toString(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java index 331ceda315918f..e019d0e3084053 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java @@ -41,6 +41,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * The node of logical plan for sub query and alias @@ -127,6 +128,18 @@ public String toString() { )); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("(").append(child().toDigest()).append(") AS "); + sb.append(qualifier.get(0)); + if (columnAliases.isPresent()) { + columnAliases.get().stream() + .collect(Collectors.joining(", ", "(", ")")); + } + return sb.toString(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java index d9fae844c48912..9935331cf0e6b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java @@ -112,6 +112,15 @@ public String toString() { "hasPushedFilter", hasPushedFilter); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("(").append(child(0).toDigest()).append(")"); + sb.append(" UNION ").append(qualifier).append(" "); + sb.append("(").append(child(1).toDigest()).append(")"); + return sb.toString(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/UsingJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/UsingJoin.java index e206b79a6b33e5..32246de282ba53 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/UsingJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/UsingJoin.java @@ -36,6 +36,7 @@ import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; /** * select col1 from t1 join t2 using(col1); @@ -177,4 +178,23 @@ public Optional getOnClauseCondition() { public boolean hasDistributeHint() { return hint != null; } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(left().toDigest()); + sb.append(" ").append(joinType).append(" "); + sb.append(right().toDigest()); + sb.append(" USING ("); + sb.append( + hashJoinConjuncts.stream().map(Expression::toDigest) + .collect(Collectors.joining(", ")) + ); + sb.append( + otherJoinConjuncts.stream().map(Expression::toDigest) + .collect(Collectors.joining(", ")) + ); + sb.append(")"); + return sb.toString(); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserDigestTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserDigestTest.java new file mode 100644 index 00000000000000..5b19aba7229c84 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserDigestTest.java @@ -0,0 +1,286 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +public class NereidsParserDigestTest extends ParserTestBase { + + private void assertDigestEquals(String expected, + List> logicalPlanList) { + String digest = logicalPlanList.get(0).first.toDigest(); + Assertions.assertEquals(expected, digest); + } + + @Test + public void testDigest() { + NereidsParser nereidsParser = new NereidsParser(); + // test simple query + String sql + = "SELECT (a+1) as b, c as d, abs(f) as f FROM test where not exists(select d from test2) " + + "and c in (1,2,3) and e in (select e from test3) and d is null " + + "and f = (select f from testf) and e = [1, 3, 5];"; + List> logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT a + ? AS b, c AS d, ABS(f) AS f FROM test WHERE " + + "NOT EXISTS (SELECT d FROM test2) AND c IN (?) AND e IN (SELECT e FROM test3) " + + "AND d IS NULL AND f = (SELECT f FROM testf) AND e = [?]", + logicalPlanList); + + // test group by and order by + sql = "select a,b from test_table group by 1,2 order by 1"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT a, b FROM test_table GROUP BY 1, 2 ORDER BY 1 ASC NULLS FIRST", + logicalPlanList); + + // test explain + sql = "explain select a from test"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("EXPLAIN SELECT a FROM test", logicalPlanList); + + // test variable + sql = "SELECT @@session.auto_increment_increment AS auto_increment_increment, @query_timeout AS query_timeout;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals( + "SELECT @@auto_increment_increment AS auto_increment_increment, @query_timeout AS query_timeout", + logicalPlanList); + + // test one row relation + sql = "select 100, 'value'"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT ?, ?", logicalPlanList); + + // test select tablet + sql = "select * from test tablet(1024)"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT * FROM test TABLET(?)", logicalPlanList); + + // test except + sql = "select * except(age) from student;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT * EXCEPT(age) FROM student", logicalPlanList); + + // test lateral view + sql = "SELECT * FROM person LATERAL VIEW EXPLODE(ARRAY(30, 60)) tableName AS c_age;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT * FROM person LATERAL VIEW EXPLODE(ARRAY(?, ?)) tableName AS c_age", + logicalPlanList); + + // test lambda + sql = "SELECT ARRAY_MAP(x->x+1, ARRAY(87, 33, -49))"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT ARRAY_MAP(x -> x + ?, ARRAY(?, ?, ?)) " + + "AS ARRAY_MAP(x->x+1, ARRAY(87, 33, -49))", logicalPlanList); + + // test set operation + sql = "SELECT student_id, name\n" + + "FROM students\n" + + "EXCEPT\n" + + "SELECT student_id, name\n" + + "FROM graduated_students;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("(SELECT student_id, name FROM students) " + + "EXCEPT DISTINCT (SELECT student_id, name FROM graduated_students)", logicalPlanList); + + sql = "SELECT student_id, name\n" + + "FROM math_students\n" + + "INTERSECT\n" + + "SELECT student_id, name\n" + + "FROM physics_students;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("(SELECT student_id, name FROM math_students) " + + "INTERSECT DISTINCT (SELECT student_id, name FROM physics_students)", logicalPlanList); + + sql = "SELECT student_id, name, age\n" + + "FROM class1_students\n" + + "UNION\n" + + "SELECT student_id, name, age\n" + + "FROM class2_students;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("(SELECT student_id, name, age FROM class1_students) " + + "UNION DISTINCT (SELECT student_id, name, age FROM class2_students)", logicalPlanList); + + // test tvf + sql = "SELECT cast(id as INT) as id, name, cast (age as INT) as age FROM " + + "s3(\"s3.access_key\"= \"ak\", \"s3.secret_key\" = \"sk\");"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT cast(id as INT) AS id, name, cast(age as INT) AS age FROM s3(?)", logicalPlanList); + + // test filter + subquery + in list + sql = "select id, concat(firstname, ' ', lastname) as fullname from student where age in (18,20,25) and " + + "name like('_h%') and id in (select id from application) order by id desc limit 1,3;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT id, CONCAT(firstname, ?, lastname) AS fullname FROM student " + + "WHERE age IN (?) AND name like ? AND id IN (SELECT id FROM application) " + + "ORDER BY id DESC LIMIT ? OFFSET ?", + logicalPlanList); + + // test agg + sql = "select sum(price) as total,type, count(1) as t_count from tb_book where level > 1 group by type;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals( + "SELECT SUM(price) AS total, type, COUNT(?) AS t_count FROM tb_book WHERE level > ? GROUP BY type", + logicalPlanList); + + // test distinct agg + sql = "select distinct type, id from tb_book group by type;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + // NOTE: this is special for distinct group by + assertDigestEquals("SELECT DISTINCT * FROM tb_book GROUP BY type", logicalPlanList); + + sql = "select type, id from tb_book group by type;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT type, id FROM tb_book GROUP BY type", logicalPlanList); + + // test distinct + sql = "select distinct type, id from tb_book"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT DISTINCT type, id FROM tb_book", logicalPlanList); + + // test case when + sql = "select d_week_seq,\n" + + " sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales,\n" + + " sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales,\n" + + " sum(case when (d_day_name='Tuesday') then sales_price else null end) tue_sales,\n" + + " sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales,\n" + + " sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales,\n" + + " sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales,\n" + + " sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales\n" + + " from wscs\n" + + " ,date_dim\n" + + " where d_date_sk = sold_date_sk\n" + + " group by d_week_seq"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals( + "SELECT d_week_seq, " + + "SUM(CASE WHEN d_day_name = ? THEN sales_price ELSE ? END) AS sun_sales, " + + "SUM(CASE WHEN d_day_name = ? THEN sales_price ELSE ? END) AS mon_sales, " + + "SUM(CASE WHEN d_day_name = ? THEN sales_price ELSE ? END) AS tue_sales, " + + "SUM(CASE WHEN d_day_name = ? THEN sales_price ELSE ? END) AS wed_sales, " + + "SUM(CASE WHEN d_day_name = ? THEN sales_price ELSE ? END) AS thu_sales, " + + "SUM(CASE WHEN d_day_name = ? THEN sales_price ELSE ? END) AS fri_sales, " + + "SUM(CASE WHEN d_day_name = ? THEN sales_price ELSE ? END) AS sat_sales " + + "FROM wscs CROSS_JOIN date_dim WHERE d_date_sk = sold_date_sk GROUP BY d_week_seq", + logicalPlanList); + + // test cte + sql = "WITH\n" + + " cte1 AS (SELECT a，b FROM table1),\n" + + " cte2 AS (SELECT c，d FROM table2)\n" + + "SELECT b，d FROM cte1 JOIN cte2\n" + + "WHERE cte1.a = cte2.c;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("WITH\n" + + "(SELECT a，b FROM table1) AS cte1, (SELECT c，d FROM table2) AS cte2\n" + + "SELECT b，d FROM cte1 CROSS_JOIN cte2 WHERE cte1.a = cte2.c", logicalPlanList); + + // test hint + sql = "select /*+ leading(t1 {t2 t3}) */ * from t1 left join t2 on c1 = c2 join t3 on c2 = c3;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT * FROM t1 LEFT_OUTER_JOIN t2 ON c1 = c2 INNER_JOIN t3 ON c2 = c3", logicalPlanList); + + // test using join + sql = "SELECT order_id, name, order_date\n" + + "FROM orders\n" + + "JOIN customers\n" + + "USING (customer_id);"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT order_id, name, order_date FROM orders " + + "INNER_JOIN customers USING (customer_id)", logicalPlanList); + + // test window function + sql = "select k1, sum(k2), rank() over(partition by k1 order by k1) as ranking from t1 group by k1"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals( + "SELECT k1, SUM(k2) AS sum(k2), RANK() OVER (PARTITION BY k1 ORDER BY k1 ASC NULLS FIRST) AS ranking FROM t1 GROUP BY k1", + logicalPlanList); + + // test binary keyword + sql = "SELECT BINARY 'abc' FROM t"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT ? FROM t", logicalPlanList); + + // test rollup + sql = "SELECT a, b, sum(c) from test group by a ASC, b ASC WITH ROLLUP"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals( + "SELECT a, b, SUM(c) AS sum(c) FROM test GROUP BY GROUPING SETS ((a,b), (a), ()) ORDER BY a ASC NULLS FIRST, b ASC NULLS FIRST", + logicalPlanList); + sql = "SELECT a, b from test group by a, b WITH ROLLUP"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT a, b FROM test GROUP BY GROUPING SETS ((a,b), (a), ())", logicalPlanList); + sql = "SELECT distinct a from test group by a, b WITH ROLLUP"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT DISTINCT * FROM test GROUP BY GROUPING SETS ((a,b), (a), ())", logicalPlanList); + + // test export + sql = "EXPORT TABLE test\n" + + "WHERE k1 < 50\n" + + "TO \"s3://bucket/export\"\n" + + "PROPERTIES (\n" + + " \"columns\" = \"k1,k2\",\n" + + " \"column_separator\"=\",\"\n" + + ") WITH s3 (\n" + + " \"s3.endpoint\" = \"xxxxx\",\n" + + " \"s3.region\" = \"xxxxx\"\n" + + ")"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("EXPORT TABLE test WHERE k1 < ? TO ?", logicalPlanList); + + // test insert + FeConstants.runningUnitTest = true; + ConnectContext ctx = ConnectContext.get(); + ctx.setCurrentUserIdentity(UserIdentity.ROOT); + ctx.setRemoteIP("127.0.0.1"); + ctx.setEnv(Env.getCurrentEnv()); + ctx.setDatabase("mysql"); + + ctx.setThreadLocalInfo(); + sql = "INSERT INTO test VALUES (1, 2);"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("INSERT INTO test VALUES ?", logicalPlanList); + + sql = "INSERT INTO test (c1, c2) VALUES (1, 2), (3, 2 * 2);"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("INSERT INTO test (c1, c2) VALUES ?", logicalPlanList); + + sql = "INSERT INTO test PARTITION(p1, p2) WITH LABEL `label1` SELECT * FROM test2;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("INSERT INTO test SELECT * FROM test2", logicalPlanList); + + sql = "INSERT OVERWRITE table test (c1, c2) VALUES (1, 2), (3, 2 * 2);"; + logicalPlanList = nereidsParser.parseMultiple(sql); + // special for insert overwrite + assertDigestEquals("OVERWRITE TABLE INSERT INTO test (c1, c2) VALUES ?", logicalPlanList); + + sql = "INSERT OVERWRITE table test (c1, c2) SELECT * from test2;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + // special for insert overwrite + assertDigestEquals("OVERWRITE TABLE INSERT INTO test (c1, c2) SELECT * FROM test2", logicalPlanList); + } +}