diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableSnapshot.java index efae08263c55eb..d5a904dc950923 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableSnapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableSnapshot.java @@ -68,4 +68,12 @@ public String toString() { return " FOR TIME AS OF '" + value + "'"; } } + + public String toDigest() { + if (this.type == VersionType.VERSION) { + return " FOR VERSION AS OF " + '?'; + } else { + return " FOR TIME AS OF '" + '?' + "'"; + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundAlias.java index 692f8f28318044..232c705df98e31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundAlias.java @@ -104,6 +104,16 @@ public String toString() { return stringBuilder.toString(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(child().toDigest()); + if (alias.isPresent()) { + sb.append(" AS " + alias.get()); + } + return sb.toString(); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitUnboundAlias(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java index fae76a5c0384d7..666455b7ed7c94 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java @@ -130,6 +130,20 @@ public String toString() { return "'" + getName() + "(" + (isDistinct ? "distinct " : "") + params + ")"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(getName().toUpperCase()); + sb.append("("); + sb.append(isDistinct ? "distinct " : ""); + sb.append( + children.stream().map(Expression::toDigest) + .collect(Collectors.joining(", ")) + ); + sb.append(")"); + return sb.toString(); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitUnboundFunction(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundInlineTable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundInlineTable.java index 42d637d676fae2..3a6e4be2ec877b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundInlineTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundInlineTable.java @@ -84,4 +84,10 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr public List computeOutput() { throw new UnboundException("output"); } + + @Override + public String toDigest() { + // TODO handle exprList? + return "VALUES ?"; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java index bb61bc93574208..1bee39655f20cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java @@ -37,6 +37,7 @@ import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; /** * A relation that contains only one row consist of some constant expressions. @@ -108,4 +109,15 @@ public String toString() { "projects", projects ); } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT "); + sb.append( + projects.stream().map(Expression::toDigest) + .collect(Collectors.joining(", ")) + ); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index 963986bf383479..f02c2bd3b2bfcf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -42,6 +42,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; /** * Represent a relation plan node that has not been bound. @@ -187,6 +188,28 @@ public String toString() { return Utils.toSqlString("UnboundRelation", args.toArray()); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + if (nameParts.size() > 0) { + sb.append(nameParts.stream().collect(Collectors.joining("."))); + sb.append(" "); + } + if (indexName.isPresent()) { + sb.append("INDEX ").append(indexName.get()).append(" "); + } + if (tabletIds.size() > 0) { + sb.append("TABLET(?)").append(" "); + } + if (scanParams != null) { + sb.append("@").append(scanParams.getParamType()).append(" "); + } + if (tableSnapshot.isPresent()) { + sb.append(tableSnapshot.get().toDigest()).append(" "); + } + return sb.substring(0, sb.length() - 1); + } + @Override public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java index e6fd3eedf0df88..747049d3d8c634 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java @@ -91,6 +91,11 @@ public String toString() { return Utils.toSqlString("UnboundResultSink[" + id.asInt() + "]"); } + @Override + public String toDigest() { + return child().toDigest(); + } + @Override public StmtType stmtType() { return StmtType.SELECT; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java index 9f60aa2e68e0ab..2181b1cdc5f71f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java @@ -70,6 +70,11 @@ public String getName() { }).reduce((left, right) -> left + "." + right).orElse(""); } + @Override + public String toDigest() { + return computeToSql(); + } + @Override public List getQualifier() { return nameParts.subList(0, nameParts.size() - 1); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java index a78ca007d6d561..4a2985dc2810c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java @@ -156,6 +156,21 @@ public String toString() { return toSql(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + if (qualifier.isEmpty()) { + sb.append("*"); + } else { + sb.append(qualifier.get(0) + ".*"); + } + if (!exceptedSlots.isEmpty()) { + sb.append(exceptedSlots.stream().map(NamedExpression::toDigest) + .collect(Collectors.joining(", ", " EXCEPT(", ")"))); + } + return sb.toString(); + } + public Optional> getIndexInSqlString() { return indexInSqlString; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java index 3024058edc7a5d..ee8212b9049540 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java @@ -107,4 +107,11 @@ public String toString() { "arguments", properties ); } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(functionName); + sb.append("(?)"); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java index f2b6821c8043cf..b9290c796c2406 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java @@ -196,4 +196,15 @@ public String toString() { "colNames", colNames, "hints", hints); } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("INSERT INTO ").append(StringUtils.join(nameParts, ".")); + if (colNames != null && colNames.size() > 0) { + sb.append(" (").append(StringUtils.join(colNames, ", ")).append(")"); + } + sb.append(" ").append(child().toDigest()); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundVariable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundVariable.java index 340008bcc0a2de..66e19e486c124a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundVariable.java @@ -80,6 +80,11 @@ public String toString() throws UnboundException { } } + @Override + public String toDigest() { + return toString(); + } + /** * variable type */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 2e56d769e84144..70c964d8558feb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -4120,6 +4120,13 @@ protected LogicalPlan withSelectQuerySpecification( List orderKeys = Lists.newArrayList(); LogicalPlan aggregate = withAggregate(filter, selectColumnCtx, aggClause, orderKeys); boolean isDistinct = (selectClause.DISTINCT() != null); + if (!isDistinct) { + if (aggregate instanceof LogicalRepeat) { + aggregate = ((LogicalRepeat) aggregate).withInProjection(false); + } else if (aggregate instanceof LogicalAggregate) { + aggregate = ((LogicalAggregate) aggregate).withInProjection(false); + } + } LogicalPlan selectPlan; if (!(aggregate instanceof Aggregate) && havingClause.isPresent()) { // create a project node for pattern match of ProjectToGlobalAggregate rule diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java index e3f06c7d1ddad7..1e0654aec46e1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.properties; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import java.util.Objects; @@ -78,6 +79,23 @@ public String toString() { return expr.toString() + (isAsc ? " asc" : " desc") + (nullFirst ? " null first" : ""); } + /** + * generate digest for order by key + * + * @return String + */ + public String toDigest() { + StringBuilder sb = new StringBuilder(); + if (expr instanceof Literal) { + sb.append(expr.toString()); + } else { + sb.append(expr.toDigest()); + } + sb.append(isAsc ? " ASC" : " DESC"); + sb.append(nullFirst ? " NULLS FIRST" : ""); + return sb.toString(); + } + @Override public int hashCode() { return Objects.hash(expr, isAsc, nullFirst); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java index 74215be9c4ea6b..a5e3ee9c5e11f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java @@ -389,4 +389,8 @@ default boolean deepEquals(TreeNode that) { // If the "that" tree hasn't been fully traversed, return false. return thatDeque.isEmpty(); } + + default String toDigest() { + return ""; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java index 59d90b6668a091..0650a7cbf86f05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.expressions; import org.apache.doris.nereids.exceptions.UnboundException; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; @@ -153,6 +154,17 @@ public String toString() { return child().toString() + " AS `" + name.get() + "`#" + exprId; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + if (child() instanceof Literal) { + sb.append("?"); + } else { + sb.append(child().toDigest()).append(" AS ").append(getName()); + } + return sb.toString(); + } + @Override public Alias withChildren(List children) { Preconditions.checkArgument(children.size() == 1); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java index c081fe5dc4acb9..0b8ee94e8b815d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java @@ -59,6 +59,15 @@ public String toString() { return "(" + left().toString() + " " + symbol + " " + right().toString() + ")"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(left().toDigest()); + sb.append(" ").append(symbol).append(" "); + sb.append(right().toDigest()); + return sb.toString(); + } + @Override public String getFingerprint() { String leftFingerprint = left().toString(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java index 0c3687f57153f2..cf30185998716f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java @@ -110,6 +110,20 @@ public String toString() { return output.toString(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder("CASE"); + for (Expression child : children()) { + if (child instanceof WhenClause) { + sb.append(child.toDigest()); + } else { + sb.append(" ELSE ").append(child.toDigest()); + } + } + sb.append(" END"); + return sb.toString(); + } + @Override public String computeToSql() throws UnboundException { StringBuilder output = new StringBuilder("CASE"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java index edff4a5e9d08df..69280d383219b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java @@ -221,6 +221,17 @@ public String toString() { return "cast(" + child() + " as " + targetType + ")"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("cast(") + .append(child().toDigest()) + .append(" as ") + .append(targetType) + .append(")"); + return sb.toString(); + } + @Override public boolean equals(Object o) { if (!super.equals(o)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CompoundPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CompoundPredicate.java index 8b08fd39c2e95e..13f99839ffb119 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CompoundPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CompoundPredicate.java @@ -116,6 +116,16 @@ public String toString() { return symbol + "[" + sb + "]"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("("); + sb.append(children().stream().map(c -> c.toDigest()) + .collect(Collectors.joining(" " + symbol + " "))); + sb.append(")"); + return sb.toString(); + } + @Override public String getFingerprint() { StringBuilder sb = new StringBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/DefaultValueSlot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/DefaultValueSlot.java index a66428dc982708..56d7790fd92c6c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/DefaultValueSlot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/DefaultValueSlot.java @@ -64,6 +64,11 @@ public String toString() { return "DEFAULT_VALUE"; } + @Override + public String toDigest() { + return "?"; + } + public Slot withIndexInSql(Pair index) { return this; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java index c95882a71e2a56..80989377d69fc4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java @@ -75,6 +75,16 @@ public String toString() { return "EXISTS (SUBQUERY) " + super.toString(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(isNot ? "NOT " : "") + .append("EXISTS (") + .append(queryPlan.toDigest()) + .append(")"); + return sb.toString(); + } + public R accept(ExpressionVisitor visitor, C context) { return visitor.visitExistsSubquery(this, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java index 81c0bd7a83908a..df972af18325a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java @@ -179,6 +179,31 @@ public String toString() { .collect(Collectors.joining(", ", "(", ")")); } + private boolean isLiteralOptions() { + for (Expression option : options) { + if (!(option instanceof Literal)) { + return false; + } + } + return true; + } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(compareExpr.toDigest()); + sb.append(" IN "); + if (isLiteralOptions()) { + sb.append("(?)"); + } else { + sb.append( + options.stream().map(Expression::toDigest) + .collect(Collectors.joining(", ", "(", ")")) + ); + } + return sb.toString(); + } + @Override public String getFingerprint() { return compareExpr + " IN " + options.stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java index 6b77700a4c8d49..20d327dcab13f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java @@ -78,6 +78,16 @@ public String toString() { return this.child() + " IN (INSUBQUERY) " + super.toString(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(getCompareExpr().toDigest()); + sb.append(isNot ? " NOT IN (" : " IN ("); + sb.append(queryPlan.toDigest()); + sb.append(')'); + return sb.toString(); + } + public R accept(ExpressionVisitor visitor, C context) { return visitor.visitInSubquery(this, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java index a5ea5fd9495ff8..a587ac6f8e2966 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java @@ -64,6 +64,14 @@ public String toString() { return child().toString() + " IS NULL"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(child().toDigest()); + sb.append(" IS NULL"); + return sb.toString(); + } + @Override public boolean equals(Object o) { if (!super.equals(o)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Not.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Not.java index 25602f08e8ab00..12c0252d3a302f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Not.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Not.java @@ -101,6 +101,13 @@ public String toString() { return "( not " + child().toString() + ")"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("NOT ").append(child().toDigest()); + return sb.toString(); + } + @Override public String computeToSql() { return "( not " + child().toSql() + ")"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/OrderExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/OrderExpression.java index b0564b7073761e..dffac46e5b96e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/OrderExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/OrderExpression.java @@ -80,6 +80,11 @@ public String toString() { return orderKey.toString(); } + @Override + public String toDigest() { + return orderKey.toDigest(); + } + @Override public String computeToSql() { return orderKey.toSql(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java index caa3fd07e30322..ce92a0e36c8249 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java @@ -66,6 +66,10 @@ public String toString() { return "$" + placeholderId.asInt(); } + public String toDigest() { + return "?"; + } + @Override public String computeToSql() { return "?"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java index 09dc6a5c216178..afffa324424a0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java @@ -69,6 +69,11 @@ public String toString() { return "Properties(" + toSql() + ")"; } + @Override + public String toDigest() { + return "?"; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ScalarSubquery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ScalarSubquery.java index ed608135254972..6e608ee5e4d11e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ScalarSubquery.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ScalarSubquery.java @@ -113,6 +113,15 @@ public String toString() { return " (SCALARSUBQUERY) " + super.toString(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("(") + .append(queryPlan.toDigest()) + .append(")"); + return sb.toString(); + } + public R accept(ExpressionVisitor visitor, C context) { return visitor.visitScalarSubquery(this, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StringRegexPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StringRegexPredicate.java index 5a62be54f93d81..21833fe8780ee3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StringRegexPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StringRegexPredicate.java @@ -64,6 +64,17 @@ public String toString() { return "(" + left() + " " + getName() + " " + right() + ")"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(left().toDigest()) + .append(' ') + .append(getName()) + .append(' ') + .append(right().toDigest()); + return sb.toString(); + } + public R accept(ExpressionVisitor visitor, C context) { return visitor.visitStringRegexPredicate(this, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Subtract.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Subtract.java index 35f4358c79fde0..46e8174a2db808 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Subtract.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Subtract.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.ArithmeticExpr.Operator; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DecimalV3Type; @@ -57,4 +58,16 @@ public DecimalV3Type getDataTypeForDecimalV3(DecimalV3Type t1, DecimalV3Type t2) public R accept(ExpressionVisitor visitor, C context) { return visitor.visitSubtract(this, context); } + + @Override + public String toDigest() { + if (left() instanceof IntegerLiteral) { + IntegerLiteral left = (IntegerLiteral) left(); + if (left.getValue() == 0) { + // nereids parser change - operator to subtract, so compactible with that + return " -" + right().toDigest(); + } + } + return super.toDigest(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TimestampArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TimestampArithmetic.java index 2bf6dc501aa1b0..b028df70c245b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TimestampArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TimestampArithmetic.java @@ -129,6 +129,21 @@ public String toString() { return toSql(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + if (funcName != null) { + sb.append(funcName.toUpperCase()).append("("); + sb.append(child(0).toDigest()).append(", "); + sb.append(child(1).toDigest()).append(")"); + } else { + sb.append(child(0).toDigest()); + sb.append(" ").append(op.toString()).append(" "); + sb.append(child(1).toDigest()); + } + return sb.toString(); + } + @Override public String computeToSql() { StringBuilder strBuilder = new StringBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/UnaryOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/UnaryOperator.java index 299e00d0dc4fe4..e881e0584fe47a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/UnaryOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/UnaryOperator.java @@ -55,6 +55,14 @@ public String toString() { return "(" + symbol + " " + child().toString() + ")"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(" ").append(symbol).append(" "); + sb.append(child().toDigest()); + return sb.toString(); + } + @Override public int computeHashCode() { return Objects.hash(symbol, child()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WhenClause.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WhenClause.java index 6e10701faaad08..3a458fc8380a7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WhenClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WhenClause.java @@ -98,4 +98,11 @@ public int computeHashCode() { public String toString() { return " WHEN " + left().toString() + " THEN " + right().toString(); } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(" WHEN ").append(left().toDigest()).append(" THEN ").append(right().toDigest()); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java index 659e4066113b14..fa9337598d2caa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java @@ -227,6 +227,24 @@ public String toString() { return sb.toString().trim(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(function.toDigest()).append(" OVER ("); + if (!partitionKeys.isEmpty()) { + sb.append("PARTITION BY ").append(partitionKeys.stream() + .map(Expression::toDigest) + .collect(Collectors.joining(", ", "", " "))); + } + if (!orderKeys.isEmpty()) { + sb.append("ORDER BY ").append(orderKeys.stream() + .map(OrderExpression::toDigest) + .collect(Collectors.joining(", ", "", " "))); + } + windowFrame.ifPresent(wf -> sb.append(" ").append(wf.toDigest())); + return sb.toString().trim() + ")"; + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitWindow(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java index 2c0d7d65b80a5f..b61b3ef1a43568 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java @@ -119,6 +119,18 @@ public String toString() { return sb.toString(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(frameUnits + " "); + if (rightBoundary != null) { + sb.append("BETWEEN " + leftBoundary.toDigest() + " AND " + rightBoundary.toDigest()); + } else { + sb.append(leftBoundary.toDigest()); + } + return sb.toString(); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitWindowFrame(this, context); @@ -215,6 +227,32 @@ public String toString() { return sb.toString(); } + /** to digest */ + public String toDigest() { + StringBuilder sb = new StringBuilder(); + boundOffset.ifPresent(value -> sb.append(value.toDigest()).append(" ")); + switch (frameBoundType) { + case UNBOUNDED_PRECEDING: + sb.append("UNBOUNDED PRECEDING"); + break; + case UNBOUNDED_FOLLOWING: + sb.append("UNBOUNDED FOLLOWING"); + break; + case CURRENT_ROW: + sb.append("CURRENT ROW"); + break; + case PRECEDING: + sb.append("PRECEDING"); + break; + case FOLLOWING: + sb.append("FOLLOWING"); + break; + default: + break; + } + return sb.toString(); + } + /** toSql*/ public String toSql() { StringBuilder sb = new StringBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BoundFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BoundFunction.java index 2cd9b27a0b93df..16d7740cf9c2a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BoundFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BoundFunction.java @@ -108,6 +108,17 @@ public String toString() { return getName() + "(" + args + ")"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(getName().toUpperCase()); + sb.append( + children().stream().map(Expression::toDigest) + .collect(Collectors.joining(", ", "(", ")")) + ); + return sb.toString(); + } + @Override public Expression withChildren(List children) { throw new UnsupportedOperationException( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java index a7bb33f39247a6..41ed2bbb2c68f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java @@ -157,6 +157,20 @@ public String toString() { return getName() + "(" + (distinct ? "DISTINCT " : "") + args + ")"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(getName()).append("("); + if (distinct) { + sb.append("DISTINCT "); + } + sb.append( + children.stream().map(Expression::toDigest) + .collect(Collectors.joining(", ")) + ); + sb.append(")"); + return sb.toString(); + } + public boolean supportAggregatePhase(AggregatePhase aggregatePhase) { return true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Count.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Count.java index efbfaaf6bb9250..175c25c7ce4394 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Count.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Count.java @@ -148,6 +148,14 @@ public String toString() { return super.toString(); } + @Override + public String toDigest() { + if (isStar) { + return "count(*)"; + } + return super.toDigest(); + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitCount(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Lambda.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Lambda.java index 2ecab6090d8d3f..35463aa829bcae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Lambda.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Lambda.java @@ -153,6 +153,18 @@ public String toString() { return builder.toString(); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + if (argumentNames.size() == 1) { + sb.append(argumentNames.get(0)); + } else { + sb.append(argumentNames.stream().collect(Collectors.joining(", ", "(", ")"))); + } + sb.append(" -> ").append(getLambdaFunction().toDigest()); + return sb.toString(); + } + @Override public Lambda withChildren(List children) { return new Lambda(argumentNames, children); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/ArrayLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/ArrayLiteral.java index 57c4bba82e6d78..e8ff885a5da8be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/ArrayLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/ArrayLiteral.java @@ -126,6 +126,13 @@ public String toString() { return "[" + items + "]"; } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("[?]"); + return sb.toString(); + } + @Override public String computeToSql() { String items = this.items.stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Interval.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Interval.java index 1fe7dfcd1a4422..fd751b64c9a724 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Interval.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Interval.java @@ -70,6 +70,15 @@ public R accept(ExpressionVisitor visitor, C context) { return visitor.visitInterval(this, context); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("INTERVAL "); + sb.append(value().toDigest()); + sb.append(" ").append(timeUnit); + return sb.toString(); + } + /** * Supported time unit. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index dc4a0870dc5730..21708dc6cd367d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -810,4 +810,9 @@ public static Literal convertToTypedLiteral(Object value, DataType dataType) { } return null; } + + @Override + public String toDigest() { + return "?"; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java index 8d8ece4fe064dc..c13b284e2b43b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java @@ -133,4 +133,12 @@ public boolean showPlanProcess() { public StmtType stmtType() { return StmtType.EXPLAIN; } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("EXPLAIN "); + sb.append(logicalPlan.toDigest()); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java index 36f7d9bc7d9774..5582a4f1143736 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java @@ -59,6 +59,7 @@ import java.util.Objects; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; /** * EXPORT statement, export data to dirs by broker. @@ -402,4 +403,16 @@ public StmtType stmtType() { public boolean needAuditEncryption() { return true; } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder("EXPORT TABLE "); + sb.append(nameParts.stream().collect(Collectors.joining("."))); + if (expr.isPresent()) { + sb.append(" WHERE ") + .append(expr.get().toDigest()); + } + sb.append(" TO ?"); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java index 4fb42a21fd780d..b99eb39e296497 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java @@ -257,4 +257,11 @@ private List> reorderUnionData(List sinkE public StmtType stmtType() { return StmtType.INSERT; } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(originLogicalQuery.toDigest()); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index d0badaf2d78336..1f2f497d1acaaf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -685,4 +685,15 @@ public BuildInsertExecutorResult(NereidsPlanner planner, AbstractInsertExecutor this.physicalSink = physicalSink; } } + + @Override + public String toDigest() { + // if with cte, query will be print twice + StringBuilder sb = new StringBuilder(); + sb.append(originLogicalQuery.toDigest()); + if (cte.isPresent()) { + sb.append(" ").append(cte.get().toDigest()); + } + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java index 454e9a476d1806..8a3db16f5b39e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java @@ -443,4 +443,16 @@ public StmtType stmtType() { public boolean needAuditEncryption() { return originLogicalQuery.anyMatch(node -> node instanceof TVFRelation); } + + @Override + public String toDigest() { + // if with cte, query will be print twice + StringBuilder sb = new StringBuilder(); + sb.append("OVERWRITE TABLE "); // there is no way add overwrite flag in sink(logic query), so add it here + sb.append(originLogicalQuery.toDigest()); + if (cte.isPresent()) { + sb.append(" (").append(cte.get().toDigest()).append(")"); + } + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index 3ca6d433228ed7..4e43631c95a688 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -31,6 +31,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.AggregatePhase; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.agg.Ndv; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; @@ -77,6 +78,7 @@ public class LogicalAggregate private final boolean ordinalIsResolved; private final boolean generated; private final boolean hasPushed; + private final boolean withInProjection; /** * Desc: Constructor for LogicalAggregate. @@ -93,19 +95,20 @@ public LogicalAggregate( * Distinct Agg */ public LogicalAggregate(List namedExpressions, boolean generated, CHILD_TYPE child) { - this(ImmutableList.copyOf(namedExpressions), namedExpressions, false, true, generated, false, Optional.empty(), + this(ImmutableList.copyOf(namedExpressions), namedExpressions, + false, true, generated, false, true, Optional.empty(), Optional.empty(), Optional.empty(), child); } public LogicalAggregate(List namedExpressions, boolean generated, boolean hasPushed, CHILD_TYPE child) { - this(ImmutableList.copyOf(namedExpressions), namedExpressions, false, true, generated, hasPushed, + this(ImmutableList.copyOf(namedExpressions), namedExpressions, false, true, generated, hasPushed, true, Optional.empty(), Optional.empty(), Optional.empty(), child); } public LogicalAggregate(List groupByExpressions, List outputExpressions, boolean ordinalIsResolved, CHILD_TYPE child) { - this(groupByExpressions, outputExpressions, false, ordinalIsResolved, false, false, Optional.empty(), + this(groupByExpressions, outputExpressions, false, ordinalIsResolved, false, false, true, Optional.empty(), Optional.empty(), Optional.empty(), child); } @@ -127,7 +130,7 @@ public LogicalAggregate( boolean normalized, Optional> sourceRepeat, CHILD_TYPE child) { - this(groupByExpressions, outputExpressions, normalized, false, false, false, sourceRepeat, + this(groupByExpressions, outputExpressions, normalized, false, false, false, true, sourceRepeat, Optional.empty(), Optional.empty(), child); } @@ -141,6 +144,7 @@ private LogicalAggregate( boolean ordinalIsResolved, boolean generated, boolean hasPushed, + boolean withInProjection, Optional> sourceRepeat, Optional groupExpression, Optional logicalProperties, @@ -157,6 +161,7 @@ private LogicalAggregate( this.generated = generated; this.hasPushed = hasPushed; this.sourceRepeat = Objects.requireNonNull(sourceRepeat, "sourceRepeat cannot be null"); + this.withInProjection = withInProjection; } @Override @@ -199,6 +204,33 @@ public String toString() { ); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + // org.apache.doris.nereids.parser.LogicalPlanBuilder.withProjection will generate different plan for + // distinct aggregation so use withInProjection flag to control whether to generate a select statement + // eg: select distinct type, id from tb_book group by type; + // select type, id from tb_book group by type; + if (!withInProjection) { + sb.append("SELECT "); + sb.append( + outputExpressions.stream().map(Expression::toDigest) + .collect(Collectors.joining(", ")) + ); + sb.append(" FROM "); + } + sb.append(child().toDigest()); + sb.append(" GROUP BY "); + sb.append( + groupByExpressions.stream() + .map(it -> + (it instanceof Literal) ? it.toString() : it.toDigest() + ) + .collect(Collectors.joining(", ")) + ); + return sb.toString(); + } + @Override public String getFingerprint() { StringBuilder builder = new StringBuilder(); @@ -284,13 +316,14 @@ public int hashCode() { public LogicalAggregate withChildren(List children) { Preconditions.checkArgument(children.size() == 1); return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, - hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), children.get(0)); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), children.get(0)); } @Override public LogicalAggregate withGroupExpression(Optional groupExpression) { return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, - hasPushed, sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), children.get(0)); + hasPushed, withInProjection, + sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), children.get(0)); } @Override @@ -298,30 +331,31 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr Optional logicalProperties, List children) { Preconditions.checkArgument(children.size() == 1); return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, - hasPushed, sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), children.get(0)); + hasPushed, withInProjection, + sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), children.get(0)); } public LogicalAggregate withGroupByAndOutput(List groupByExprList, List outputExpressionList) { return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, ordinalIsResolved, generated, - hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), child()); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), child()); } public LogicalAggregate withGroupBy(List groupByExprList) { return new LogicalAggregate<>(groupByExprList, outputExpressions, normalized, ordinalIsResolved, generated, - hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), child()); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), child()); } public LogicalAggregate withChildGroupByAndOutput(List groupByExprList, List outputExpressionList, Plan newChild) { return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, ordinalIsResolved, generated, - hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), newChild); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), newChild); } public LogicalAggregate withChildAndOutput(CHILD_TYPE child, List outputExpressionList) { return new LogicalAggregate<>(groupByExpressions, outputExpressionList, normalized, ordinalIsResolved, - generated, hasPushed, sourceRepeat, Optional.empty(), + generated, hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), child); } @@ -333,18 +367,24 @@ public List getOutputs() { @Override public LogicalAggregate withAggOutput(List newOutput) { return new LogicalAggregate<>(groupByExpressions, newOutput, normalized, ordinalIsResolved, generated, - hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), child()); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), child()); } public LogicalAggregate withAggOutputChild(List newOutput, Plan newChild) { return new LogicalAggregate<>(groupByExpressions, newOutput, normalized, ordinalIsResolved, generated, - hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), newChild); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), newChild); } public LogicalAggregate withNormalized(List normalizedGroupBy, List normalizedOutput, Plan normalizedChild) { return new LogicalAggregate<>(normalizedGroupBy, normalizedOutput, true, ordinalIsResolved, generated, - hasPushed, sourceRepeat, Optional.empty(), Optional.empty(), normalizedChild); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), normalizedChild); + } + + public LogicalAggregate withInProjection(boolean withInProjection) { + return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, + generated, hasPushed, withInProjection, + sourceRepeat, Optional.empty(), Optional.empty(), child()); } private boolean isUniqueGroupByUnique(NamedExpression namedExpression) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java index 7a75dd5c1a4abf..4f810c3b6def8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; /** * Logical Node for CTE @@ -77,6 +78,19 @@ public String toString() { ); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("WITH\n"); + sb.append( + aliasQueries.stream().map(LogicalSubQueryAlias::toDigest) + .collect(Collectors.joining(", ")) + ); + sb.append("\n"); + sb.append(child().toDigest()); + return sb.toString(); + } + @Override public boolean displayExtraPlanFirst() { return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java index b0ae1e1c9264bd..750e36d41a61af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java @@ -90,6 +90,11 @@ public String toString() { return Utils.toSqlString("LogicalCheckPolicy"); } + @Override + public String toDigest() { + return child().toDigest(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java index ec99833c39ed29..253fff791ece06 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java @@ -67,6 +67,15 @@ public String toString() { "stats", statistics); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("(").append(child(0).toDigest()).append(")"); + sb.append(" EXCEPT ").append(qualifier).append(" "); + sb.append("(").append(child(1).toDigest()).append(")"); + return sb.toString(); + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitLogicalExcept(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java index f5e49cdba9e690..47a1ed6ff5bbb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java @@ -126,4 +126,14 @@ public Map getProperties() { public boolean needAuditEncryption() { return true; } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(child().toDigest()); + sb.append(" INTO OUTFILE '").append(" ? ").append(" FORMAT AS ").append(" ? "); + if (properties != null && !properties.isEmpty()) { + sb.append(" PROPERTIES(").append(" ? ").append(")"); + } + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java index 03515a7d3844e9..b7d72b3476b543 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java @@ -119,6 +119,18 @@ public String toString() { ); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(child().toDigest()); + sb.append(" WHERE "); + sb.append( + conjuncts.stream().map(Expression::toDigest) + .collect(Collectors.joining(" AND ")) + ); + return sb.toString(); + } + @Override public String getFingerprint() { return Utils.toSqlString("Filter[" + getGroupIdWithPrefix() + "]", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java index 38dccfbcb9836c..974317f1942f03 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java @@ -38,6 +38,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; /** * plan for table generator, the statement like: SELECT * FROM tbl LATERAL VIEW EXPLODE(c1) g as (gc1); @@ -147,6 +148,27 @@ public String toString() { ); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + String generateName = ""; + try { + generateName = generatorOutput.get(0).getQualifier().get(0); + } catch (Throwable e) { + generateName = generatorOutput.get(0).toDigest(); + } + sb.append(child().toDigest()); + sb.append(" LATERAL VIEW ") + .append(generators.get(0).toDigest()) + .append(" ") + .append(generateName) + .append(" AS ") + .append( + expandColumnAlias.get(0).stream().collect(Collectors.joining(", ")) + ); + return sb.toString(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java index 3847ffdf10fa12..339a2bb046e1b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java @@ -39,6 +39,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * Logical Having plan @@ -157,4 +158,16 @@ public String toString() { return Utils.toSqlStringSkipNull("LogicalHaving", "predicates", getPredicate(), "stats", statistics); } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(child().toDigest()); + sb.append(" HAVING "); + sb.append( + conjuncts.stream().map(Expression::toDigest) + .collect(Collectors.joining(" AND ")) + ); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java index f3e9f9db3f18f3..c8bce9823975ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java @@ -67,6 +67,15 @@ public String toString() { "stats", statistics); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("(").append(child(0).toDigest()).append(")"); + sb.append(" INTERSECT ").append(qualifier).append(" "); + sb.append("(").append(child(1).toDigest()).append(")"); + return sb.toString(); + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitLogicalIntersect(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java index b226c4fd62628b..f7bf44b5518cc4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java @@ -314,6 +314,24 @@ public String toString() { return Utils.toSqlStringSkipNull("LogicalJoin[" + id.asInt() + "]", args.toArray()); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(left().toDigest()); + sb.append(" ").append(joinType).append(" "); + sb.append(right().toDigest()); + if (!hashJoinConjuncts.isEmpty() || !otherJoinConjuncts.isEmpty()) { + sb.append(" ON "); + sb.append( + hashJoinConjuncts.stream().map(Expression::toDigest).collect(Collectors.joining(" AND ")) + ); + sb.append( + otherJoinConjuncts.stream().map(Expression::toDigest).collect(Collectors.joining(" AND ")) + ); + } + return sb.toString(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java index 32c6705f60ea95..e651fb75272ab4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java @@ -93,6 +93,17 @@ public String toString() { ); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(child().toDigest()); + sb.append(" LIMIT ? "); + if (offset != 0) { + sb.append(" OFFSET ?"); + } + return sb.toString(); + } + @Override public int hashCode() { return Objects.hash(limit, offset); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index 6af69f001ae9e5..6925bcf8fe7f9b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -51,6 +51,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * Logical project plan. @@ -124,6 +125,24 @@ public String toString() { ); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT "); + if (isDistinct) { + sb.append("DISTINCT "); + } + sb.append( + projects.stream().map(NamedExpression::toDigest) + .collect(Collectors.joining(", ")) + ); + if (child().getType() != PlanType.LOGICAL_UNBOUND_ONE_ROW_RELATION) { + sb.append(" FROM "); + } + sb.append(child().toDigest()); + return sb.toString(); + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitLogicalProject(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalQualify.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalQualify.java index 904c66f6482ee5..665990b3fdcf96 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalQualify.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalQualify.java @@ -39,6 +39,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * Logical qualify plan. @@ -90,6 +91,15 @@ public String toString() { ); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(child().toDigest()); + sb.append(" QUALIFY "); + sb.append(conjuncts.stream().map(Expression::toDigest).collect(Collectors.joining(" AND "))); + return sb.toString(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java index 23c8be7417c87d..f2b4a18e46b691 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java @@ -39,6 +39,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * LogicalRepeat. @@ -51,6 +52,7 @@ public class LogicalRepeat extends LogicalUnary> groupingSets; private final List outputExpressions; + private final boolean withInProjection; /** * Desc: Constructor for LogicalRepeat. @@ -59,7 +61,7 @@ public LogicalRepeat( List> groupingSets, List outputExpressions, CHILD_TYPE child) { - this(groupingSets, outputExpressions, Optional.empty(), Optional.empty(), child); + this(groupingSets, outputExpressions, Optional.empty(), Optional.empty(), true, child); } /** @@ -67,6 +69,7 @@ public LogicalRepeat( */ public LogicalRepeat(List> groupingSets, List outputExpressions, Optional groupExpression, Optional logicalProperties, + boolean withInProjection, CHILD_TYPE child) { super(PlanType.LOGICAL_REPEAT, groupExpression, logicalProperties, child); this.groupingSets = Objects.requireNonNull(groupingSets, "groupingSets can not be null") @@ -75,6 +78,7 @@ public LogicalRepeat(List> groupingSets, List .collect(ImmutableList.toImmutableList()); this.outputExpressions = ImmutableList.copyOf( Objects.requireNonNull(outputExpressions, "outputExpressions can not be null")); + this.withInProjection = withInProjection; } @Override @@ -100,6 +104,36 @@ public String toString() { ); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + // org.apache.doris.nereids.parser.LogicalPlanBuilder.withProjection will generate different plan for + // distinct aggregation so use withInProjection flag to control whether to generate a select statement + // eg: select distinct log_time from example_tbl_duplicate group by log_time,log_type with rollup; + // select log_time from example_tbl_duplicate group by log_time,log_type with rollup; + if (!withInProjection) { + sb.append("SELECT "); + sb.append( + outputExpressions.stream().map(Expression::toDigest) + .collect(Collectors.joining(", ")) + ); + sb.append(" FROM "); + } + sb.append(child().toDigest()); + sb.append(" GROUP BY GROUPING SETS ("); + for (int i = 0; i < groupingSets.size(); i++) { + List groupingSet = groupingSets.get(i); + String subSet = groupingSet.stream().map(Expression::toDigest) + .collect(Collectors.joining(",", "(", ")")); + sb.append(subSet); + if (i != groupingSets.size() - 1) { + sb.append(", "); + } + } + sb.append(")"); + return sb.toString(); + } + @Override public List computeOutput() { return outputExpressions.stream() @@ -146,7 +180,7 @@ public LogicalRepeat withChildren(List children) { @Override public LogicalRepeat withGroupExpression(Optional groupExpression) { return new LogicalRepeat<>(groupingSets, outputExpressions, groupExpression, - Optional.of(getLogicalProperties()), child()); + Optional.of(getLogicalProperties()), withInProjection, child()); } @Override @@ -154,7 +188,7 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr Optional logicalProperties, List children) { Preconditions.checkArgument(children.size() == 1); return new LogicalRepeat<>(groupingSets, outputExpressions, groupExpression, logicalProperties, - children.get(0)); + withInProjection, children.get(0)); } public LogicalRepeat withGroupSets(List> groupingSets) { @@ -180,6 +214,11 @@ public LogicalRepeat withAggOutputAndChild(List newOutput return new LogicalRepeat<>(groupingSets, newOutput, child); } + public LogicalRepeat withInProjection(boolean withInProjection) { + return new LogicalRepeat<>(groupingSets, outputExpressions, + Optional.empty(), Optional.empty(), withInProjection, child()); + } + public boolean canBindVirtualSlot() { return bound() && outputExpressions.stream() .noneMatch(output -> output.containsType(VirtualSlotReference.class)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java index 10b67a84f4e54c..2257444cc1f0f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java @@ -127,4 +127,9 @@ public String toString() { .collect(Collectors.joining(", ")); return "LogicalSelectHint (" + hintStr + ")"; } + + @Override + public String toDigest() { + return child().toDigest(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java index 607fcf25bca7fe..45e0e081a3cdac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java @@ -37,6 +37,7 @@ import java.util.Objects; import java.util.Optional; import java.util.function.Supplier; +import java.util.stream.Collectors; /** * Logical Sort plan. @@ -89,6 +90,17 @@ public String toString() { "orderKeys", orderKeys); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(child().toDigest()).append(" ORDER BY "); + sb.append( + orderKeys.stream().map(OrderKey::toDigest) + .collect(Collectors.joining(", ")) + ); + return sb.toString(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java index 328508fb3c70e7..2f49dc1ce85102 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java @@ -42,6 +42,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * The node of logical plan for sub query and alias @@ -138,6 +139,18 @@ public String toString() { )); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("(").append(child().toDigest()).append(") AS "); + sb.append(qualifier.get(0)); + if (columnAliases.isPresent()) { + columnAliases.get().stream() + .collect(Collectors.joining(", ", "(", ")")); + } + return sb.toString(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java index ebad64dc16eadf..1e563d5676dd21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java @@ -123,6 +123,15 @@ public String toString() { "stats", statistics); } + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append("(").append(child(0).toDigest()).append(")"); + sb.append(" UNION ").append(qualifier).append(" "); + sb.append("(").append(child(1).toDigest()).append(")"); + return sb.toString(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUsingJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUsingJoin.java index 78331438a8490e..0b056da686c8de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUsingJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUsingJoin.java @@ -38,6 +38,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; /** * select col1 from t1 join t2 using(col1); @@ -147,4 +148,19 @@ public String toString() { } return Utils.toSqlStringSkipNull("UsingJoin[" + id.asInt() + "]", args.toArray()); } + + @Override + public String toDigest() { + StringBuilder sb = new StringBuilder(); + sb.append(left().toDigest()); + sb.append(" ").append(joinType).append(" "); + sb.append(right().toDigest()); + sb.append(" USING ("); + sb.append( + usingSlots.stream().map(Expression::toDigest) + .collect(Collectors.joining(", ")) + ); + sb.append(")"); + return sb.toString(); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserDigestTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserDigestTest.java new file mode 100644 index 00000000000000..ddbdcbdf0c4736 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserDigestTest.java @@ -0,0 +1,300 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +public class NereidsParserDigestTest extends ParserTestBase { + + private void assertDigestEquals(String expected, + List> logicalPlanList) { + String digest = logicalPlanList.get(0).first.toDigest(); + Assertions.assertEquals(expected, digest); + } + + @Test + public void testDigest() { + NereidsParser nereidsParser = new NereidsParser(); + // test simple query + String sql + = "SELECT (a+1) as b, c as d, abs(f) as f FROM test where not exists(select d from test2) " + + "and c in (1,2,3) and e in (select e from test3) and d is null " + + "and f = (select f from testf) and e = [1, 3, 5];"; + List> logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT a + ? AS b, c AS d, ABS(f) AS f FROM test WHERE " + + "(NOT EXISTS (SELECT d FROM test2) AND c IN (?) AND e IN (SELECT e FROM test3) " + + "AND d IS NULL AND f = (SELECT f FROM testf) AND e = [?])", + logicalPlanList); + + // test group by and order by + sql = "select a,b from test_table group by 1,2 order by 1"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT a, b FROM test_table GROUP BY 1, 2 ORDER BY 1 ASC NULLS FIRST", + logicalPlanList); + + // test explain + sql = "explain select a from test"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("EXPLAIN SELECT a FROM test", logicalPlanList); + + // test variable + sql = "SELECT @@session.auto_increment_increment AS auto_increment_increment, @query_timeout AS query_timeout;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals( + "SELECT @@auto_increment_increment AS auto_increment_increment, @query_timeout AS query_timeout", + logicalPlanList); + + // test one row relation + sql = "select 100, 'value'"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT ?, ?", logicalPlanList); + + // test select tablet + sql = "select * from test tablet(1024)"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT * FROM test TABLET(?)", logicalPlanList); + + // test except + sql = "select * except(age) from student;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT * EXCEPT(age) FROM student", logicalPlanList); + + // test lateral view + sql = "SELECT * FROM person LATERAL VIEW EXPLODE(ARRAY(30, 60)) tableName AS c_age;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT * FROM person LATERAL VIEW EXPLODE(ARRAY(?, ?)) tableName AS c_age", + logicalPlanList); + + // test lambda + sql = "SELECT ARRAY_MAP(x->x+1, ARRAY(87, 33, -49))"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT ARRAY_MAP(x -> x + ?, ARRAY(?, ?, ?)) " + + "AS ARRAY_MAP(x->x+1, ARRAY(87, 33, -49))", logicalPlanList); + + // test set operation + sql = "SELECT student_id, name\n" + + "FROM students\n" + + "EXCEPT\n" + + "SELECT student_id, name\n" + + "FROM graduated_students;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("(SELECT student_id, name FROM students) " + + "EXCEPT DISTINCT (SELECT student_id, name FROM graduated_students)", logicalPlanList); + + sql = "SELECT student_id, name\n" + + "FROM math_students\n" + + "INTERSECT\n" + + "SELECT student_id, name\n" + + "FROM physics_students;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("(SELECT student_id, name FROM math_students) " + + "INTERSECT DISTINCT (SELECT student_id, name FROM physics_students)", logicalPlanList); + + sql = "SELECT student_id, name, age\n" + + "FROM class1_students\n" + + "UNION\n" + + "SELECT student_id, name, age\n" + + "FROM class2_students;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("(SELECT student_id, name, age FROM class1_students) " + + "UNION DISTINCT (SELECT student_id, name, age FROM class2_students)", logicalPlanList); + + // test tvf + sql = "SELECT cast(id as INT) as id, name, cast (age as INT) as age FROM " + + "s3(\"s3.access_key\"= \"ak\", \"s3.secret_key\" = \"sk\");"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT cast(id as INT) AS id, name, cast(age as INT) AS age FROM s3(?)", logicalPlanList); + + // test filter + subquery + in list + sql = "select id, concat(firstname, ' ', lastname) as fullname from student where age in (18,20,25) and " + + "name like('_h%') and id in (select id from application) order by id desc limit 1,3;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT id, CONCAT(firstname, ?, lastname) AS fullname FROM student " + + "WHERE (age IN (?) AND name like ? AND id IN (SELECT id FROM application)) " + + "ORDER BY id DESC LIMIT ? OFFSET ?", + logicalPlanList); + + // test agg + sql = "select sum(price) as total,type, count(1) as t_count from tb_book where level > 1 group by type;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals( + "SELECT SUM(price) AS total, type, COUNT(?) AS t_count FROM tb_book WHERE level > ? GROUP BY type", + logicalPlanList); + + // test distinct agg + sql = "select distinct type, id from tb_book group by type;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + // NOTE: this is special for distinct group by + assertDigestEquals("SELECT DISTINCT * FROM tb_book GROUP BY type", logicalPlanList); + + sql = "select type, id from tb_book group by type;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT type, id FROM tb_book GROUP BY type", logicalPlanList); + + // test distinct + sql = "select distinct type, id from tb_book"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT DISTINCT type, id FROM tb_book", logicalPlanList); + + // test case when + sql = "select d_week_seq,\n" + + " sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales,\n" + + " sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales,\n" + + " sum(case when (d_day_name='Tuesday') then sales_price else null end) tue_sales,\n" + + " sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales,\n" + + " sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales,\n" + + " sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales,\n" + + " sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales\n" + + " from wscs\n" + + " ,date_dim\n" + + " where d_date_sk = sold_date_sk\n" + + " group by d_week_seq"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals( + "SELECT d_week_seq, " + + "SUM(CASE WHEN d_day_name = ? THEN sales_price ELSE ? END) AS sun_sales, " + + "SUM(CASE WHEN d_day_name = ? THEN sales_price ELSE ? END) AS mon_sales, " + + "SUM(CASE WHEN d_day_name = ? THEN sales_price ELSE ? END) AS tue_sales, " + + "SUM(CASE WHEN d_day_name = ? THEN sales_price ELSE ? END) AS wed_sales, " + + "SUM(CASE WHEN d_day_name = ? THEN sales_price ELSE ? END) AS thu_sales, " + + "SUM(CASE WHEN d_day_name = ? THEN sales_price ELSE ? END) AS fri_sales, " + + "SUM(CASE WHEN d_day_name = ? THEN sales_price ELSE ? END) AS sat_sales " + + "FROM wscs CROSS_JOIN date_dim WHERE d_date_sk = sold_date_sk GROUP BY d_week_seq", + logicalPlanList); + + // test cte + sql = "WITH\n" + + " cte1 AS (SELECT a，b FROM table1),\n" + + " cte2 AS (SELECT c，d FROM table2)\n" + + "SELECT b，d FROM cte1 JOIN cte2\n" + + "WHERE cte1.a = cte2.c;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("WITH\n" + + "(SELECT a，b FROM table1) AS cte1, (SELECT c，d FROM table2) AS cte2\n" + + "SELECT b，d FROM cte1 CROSS_JOIN cte2 WHERE cte1.a = cte2.c", logicalPlanList); + + // test hint + sql = "select /*+ leading(t1 {t2 t3}) */ * from t1 left join t2 on c1 = c2 join t3 on c2 = c3;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT * FROM t1 LEFT_OUTER_JOIN t2 ON c1 = c2 INNER_JOIN t3 ON c2 = c3", logicalPlanList); + + // test using join + sql = "SELECT order_id, name, order_date\n" + + "FROM orders\n" + + "JOIN customers\n" + + "USING (customer_id);"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT order_id, name, order_date FROM orders " + + "INNER_JOIN customers USING (customer_id)", logicalPlanList); + + // test window function + sql = "select k1, sum(k2), rank() over(partition by k1 order by k1) as ranking from t1 group by k1"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals( + "SELECT k1, SUM(k2) AS sum(k2), RANK() OVER (PARTITION BY k1 ORDER BY k1 ASC NULLS FIRST) AS ranking FROM t1 GROUP BY k1", + logicalPlanList); + + // test binary keyword + sql = "SELECT BINARY 'abc' FROM t"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT ? FROM t", logicalPlanList); + + // test rollup + sql = "SELECT a, b, sum(c) from test group by a ASC, b ASC WITH ROLLUP"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals( + "SELECT a, b, SUM(c) AS sum(c) FROM test GROUP BY GROUPING SETS ((a,b), (a), ()) ORDER BY a ASC NULLS FIRST, b ASC NULLS FIRST", + logicalPlanList); + sql = "SELECT a, b from test group by a, b WITH ROLLUP"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT a, b FROM test GROUP BY GROUPING SETS ((a,b), (a), ())", logicalPlanList); + sql = "SELECT distinct a from test group by a, b WITH ROLLUP"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT DISTINCT * FROM test GROUP BY GROUPING SETS ((a,b), (a), ())", logicalPlanList); + + // test qualify + sql = "select country, sum(profit) as total, row_number() over (order by country) as rk from sales " + + "where year >= 2000 group by country having sum(profit) > 100 qualify rk = 1"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT country, SUM(profit) AS total, " + + "ROW_NUMBER() OVER (ORDER BY country ASC NULLS FIRST) AS rk " + + "FROM sales WHERE year >= ? GROUP BY country HAVING SUM(profit) > ? QUALIFY rk = ?", + logicalPlanList); + + sql = "select country, sum(profit) as total from sales where year >= 2000 group by country " + + "having sum(profit) > 100 qualify row_number() over (order by country) = 1"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("SELECT country, SUM(profit) AS total FROM sales WHERE year >= ? GROUP BY country " + + "HAVING SUM(profit) > ? QUALIFY ROW_NUMBER() OVER (ORDER BY country ASC NULLS FIRST) = ?", + logicalPlanList); + + // test export + sql = "EXPORT TABLE test\n" + + "WHERE k1 < 50\n" + + "TO \"s3://bucket/export\"\n" + + "PROPERTIES (\n" + + " \"columns\" = \"k1,k2\",\n" + + " \"column_separator\"=\",\"\n" + + ") WITH s3 (\n" + + " \"s3.endpoint\" = \"xxxxx\",\n" + + " \"s3.region\" = \"xxxxx\"\n" + + ")"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("EXPORT TABLE test WHERE k1 < ? TO ?", logicalPlanList); + + // test insert + ConnectContext ctx = ConnectContext.get(); + ctx.setCurrentUserIdentity(UserIdentity.ROOT); + ctx.setRemoteIP("127.0.0.1"); + ctx.setEnv(Env.getCurrentEnv()); + ctx.setDatabase("mysql"); + + ctx.setThreadLocalInfo(); + sql = "INSERT INTO test VALUES (1, 2);"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("INSERT INTO test VALUES ?", logicalPlanList); + + sql = "INSERT INTO test (c1, c2) VALUES (1, 2), (3, 2 * 2);"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("INSERT INTO test (c1, c2) VALUES ?", logicalPlanList); + + sql = "INSERT INTO test PARTITION(p1, p2) WITH LABEL `label1` SELECT * FROM test2;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + assertDigestEquals("INSERT INTO test SELECT * FROM test2", logicalPlanList); + + sql = "INSERT OVERWRITE table test (c1, c2) VALUES (1, 2), (3, 2 * 2);"; + logicalPlanList = nereidsParser.parseMultiple(sql); + // special for insert overwrite + assertDigestEquals("OVERWRITE TABLE INSERT INTO test (c1, c2) VALUES ?", logicalPlanList); + + sql = "INSERT OVERWRITE table test (c1, c2) SELECT * from test2;"; + logicalPlanList = nereidsParser.parseMultiple(sql); + // special for insert overwrite + assertDigestEquals("OVERWRITE TABLE INSERT INTO test (c1, c2) SELECT * FROM test2", logicalPlanList); + } +}