Skip to content

branch-3.0: [fix](Nereids) not generate duplicate exprid after convert outer to anti rule #52798 #53898

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dataroaring merged 1 commit into from
Aug 12, 2025
Merged

branch-3.0: [fix](Nereids) not generate duplicate exprid after convert outer to anti rule #52798 #53898

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
import org.apache.doris.nereids.rules.rewrite.CollectFilterAboveConsumer;
import org.apache.doris.nereids.rules.rewrite.ColumnPruning;
import org.apache.doris.nereids.rules.rewrite.ConvertInnerOrCrossJoin;
import org.apache.doris.nereids.rules.rewrite.ConvertOuterJoinToAntiJoin;
import org.apache.doris.nereids.rules.rewrite.CountDistinctRewrite;
import org.apache.doris.nereids.rules.rewrite.CountLiteralRewrite;
import org.apache.doris.nereids.rules.rewrite.CreatePartitionTopNFromWindow;
Expand Down Expand Up @@ -446,15 +447,15 @@ public class Rewriter extends AbstractBatchJobExecutor {
new CollectCteConsumerOutput()
)
),
topic("Collect used column", custom(RuleType.COLLECT_COLUMNS, QueryColumnCollector::new)
)
topic("Collect used column", custom(RuleType.COLLECT_COLUMNS, QueryColumnCollector::new))
)
);

private static final List<RewriteJob> CTE_CHILDREN_REWRITE_JOBS_AFTER_SUB_PATH_PUSH_DOWN = notTraverseChildrenOf(
ImmutableSet.of(LogicalCTEAnchor.class),
() -> jobs(
// after variant sub path pruning, we need do column pruning again
bottomUp(RuleSet.PUSH_DOWN_FILTERS),
custom(RuleType.COLUMN_PRUNING, ColumnPruning::new),
bottomUp(ImmutableList.of(
new PushDownFilterThroughProject(),
Expand Down Expand Up @@ -548,6 +549,8 @@ private static List<RewriteJob> getWholeTreeRewriteJobs(
topic("rewrite cte sub-tree before sub path push down",
custom(RuleType.REWRITE_CTE_CHILDREN, () -> new RewriteCteChildren(beforePushDownJobs))
)));
rewriteJobs.addAll(jobs(topic("convert outer join to anti",
custom(RuleType.CONVERT_OUTER_JOIN_TO_ANTI, ConvertOuterJoinToAntiJoin::new))));
if (needOrExpansion) {
rewriteJobs.addAll(jobs(topic("or expansion",
custom(RuleType.OR_EXPANSION, () -> OrExpansion.INSTANCE))));
Expand Down
2 changes: 0 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,6 @@
import org.apache.doris.nereids.rules.implementation.LogicalTopNToPhysicalTopN;
import org.apache.doris.nereids.rules.implementation.LogicalUnionToPhysicalUnion;
import org.apache.doris.nereids.rules.implementation.LogicalWindowToPhysicalWindow;
import org.apache.doris.nereids.rules.rewrite.ConvertOuterJoinToAntiJoin;
import org.apache.doris.nereids.rules.rewrite.CreatePartitionTopNFromWindow;
import org.apache.doris.nereids.rules.rewrite.EliminateFilter;
import org.apache.doris.nereids.rules.rewrite.EliminateOuterJoin;
Expand Down Expand Up @@ -148,7 +147,6 @@ public class RuleSet {
new PushDownFilterThroughGenerate(),
new PushDownProjectThroughLimit(),
new EliminateOuterJoin(),
new ConvertOuterJoinToAntiJoin(),
new MergeProjects(),
new MergeFilters(),
new MergeGenerates(),
Expand Down
190 changes: 188 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.pattern.ExpressionPatternRules;
import org.apache.doris.nereids.pattern.ExpressionPatternTraverseListeners;
import org.apache.doris.nereids.pattern.MatchingContext;
import org.apache.doris.nereids.properties.OrderKey;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
Expand All @@ -28,25 +29,39 @@
import org.apache.doris.nereids.trees.expressions.EqualPredicate;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.OrderExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate;
import org.apache.doris.nereids.trees.plans.logical.LogicalHaving;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
import org.apache.doris.nereids.trees.plans.logical.LogicalSort;
import org.apache.doris.nereids.trees.plans.logical.LogicalTopN;
import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.Utils;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;

Expand Down Expand Up @@ -79,7 +94,19 @@ public List<Rule> buildRules() {
new JoinExpressionRewrite().build(),
new SortExpressionRewrite().build(),
new LogicalRepeatRewrite().build(),
new HavingExpressionRewrite().build());
new HavingExpressionRewrite().build(),
new LogicalPartitionTopNExpressionRewrite().build(),
new LogicalTopNExpressionRewrite().build(),
new LogicalSetOperationRewrite().build(),
new LogicalWindowRewrite().build(),
new LogicalCteConsumerRewrite().build(),
new LogicalResultSinkRewrite().build(),
new LogicalFileSinkRewrite().build(),
new LogicalHiveTableSinkRewrite().build(),
new LogicalIcebergTableSinkRewrite().build(),
new LogicalJdbcTableSinkRewrite().build(),
new LogicalOlapTableSinkRewrite().build(),
new LogicalDeferMaterializeResultSinkRewrite().build());
}

private class GenerateExpressionRewrite extends OneRewriteRuleFactory {
Expand Down Expand Up @@ -264,7 +291,166 @@ public Rule build() {
}
}

private class LogicalRepeatRewrite extends OneRewriteRuleFactory {
private class LogicalWindowRewrite extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalWindow().thenApply(ctx -> {
LogicalWindow<Plan> window = ctx.root;
List<NamedExpression> windowExpressions = window.getWindowExpressions();
ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext);
List<NamedExpression> result = rewriteAll(windowExpressions, rewriter, context);
return window.withExpressionsAndChild(result, window.child());
})
.toRule(RuleType.REWRITE_WINDOW_EXPRESSION);
}
}

private class LogicalSetOperationRewrite extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalSetOperation().thenApply(ctx -> {
LogicalSetOperation setOperation = ctx.root;
List<List<SlotReference>> slotsList = setOperation.getRegularChildrenOutputs();
List<List<SlotReference>> newSlotsList = new ArrayList<>();
ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext);
for (List<SlotReference> slots : slotsList) {
List<SlotReference> result = rewriteAll(slots, rewriter, context);
newSlotsList.add(result);
}
return setOperation.withChildrenAndTheirOutputs(setOperation.children(), newSlotsList);
})
.toRule(RuleType.REWRITE_SET_OPERATION_EXPRESSION);
}
}

private class LogicalTopNExpressionRewrite extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalTopN().thenApply(ctx -> {
LogicalTopN<Plan> topN = ctx.root;
List<OrderKey> orderKeys = topN.getOrderKeys();
ImmutableList.Builder<OrderKey> rewrittenOrderKeys
= ImmutableList.builderWithExpectedSize(orderKeys.size());
ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext);
boolean changed = false;
for (OrderKey k : orderKeys) {
Expression expression = rewriter.rewrite(k.getExpr(), context);
changed |= expression != k.getExpr();
rewrittenOrderKeys.add(new OrderKey(expression, k.isAsc(), k.isNullFirst()));
}
return changed ? topN.withOrderKeys(rewrittenOrderKeys.build()) : topN;
}).toRule(RuleType.REWRITE_TOPN_EXPRESSION);
}
}

private class LogicalPartitionTopNExpressionRewrite extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalPartitionTopN().thenApply(ctx -> {
LogicalPartitionTopN<Plan> partitionTopN = ctx.root;
ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext);
List<OrderExpression> newOrderExpressions = new ArrayList<>();
for (OrderExpression orderExpression : partitionTopN.getOrderKeys()) {
OrderKey orderKey = orderExpression.getOrderKey();
Expression expr = rewriter.rewrite(orderKey.getExpr(), context);
OrderKey newOrderKey = new OrderKey(expr, orderKey.isAsc(), orderKey.isNullFirst());
newOrderExpressions.add(new OrderExpression(newOrderKey));
}
List<Expression> result = rewriteAll(partitionTopN.getPartitionKeys(), rewriter, context);
return partitionTopN.withPartitionKeysAndOrderKeys(result, newOrderExpressions);
}).toRule(RuleType.REWRITE_PARTITION_TOPN_EXPRESSION);
}
}

private class LogicalCteConsumerRewrite extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalCTEConsumer().thenApply(ctx -> {
LogicalCTEConsumer consumer = ctx.root;
boolean changed = false;
ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext);
ImmutableMap.Builder<Slot, Slot> cToPBuilder = ImmutableMap.builder();
ImmutableMultimap.Builder<Slot, Slot> pToCBuilder = ImmutableMultimap.builder();
for (Map.Entry<Slot, Slot> entry : consumer.getConsumerToProducerOutputMap().entrySet()) {
Slot key = (Slot) rewriter.rewrite(entry.getKey(), context);
Slot value = (Slot) rewriter.rewrite(entry.getValue(), context);
cToPBuilder.put(key, value);
pToCBuilder.put(value, key);
if (!key.equals(entry.getKey()) || !value.equals(entry.getValue())) {
changed = true;
}
}
return changed ? consumer.withTwoMaps(cToPBuilder.build(), pToCBuilder.build()) : consumer;
}).toRule(RuleType.REWRITE_TOPN_EXPRESSION);
}
}

private class LogicalResultSinkRewrite extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalResultSink().thenApply(ExpressionRewrite.this::applyRewriteToSink)
.toRule(RuleType.REWRITE_SINK_EXPRESSION);
}
}

private class LogicalFileSinkRewrite extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalFileSink().thenApply(ExpressionRewrite.this::applyRewriteToSink)
.toRule(RuleType.REWRITE_SINK_EXPRESSION);
}
}

private class LogicalHiveTableSinkRewrite extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalHiveTableSink().thenApply(ExpressionRewrite.this::applyRewriteToSink)
.toRule(RuleType.REWRITE_SINK_EXPRESSION);
}
}

private class LogicalIcebergTableSinkRewrite extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalIcebergTableSink().thenApply(ExpressionRewrite.this::applyRewriteToSink)
.toRule(RuleType.REWRITE_SINK_EXPRESSION);
}
}

private class LogicalJdbcTableSinkRewrite extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalJdbcTableSink().thenApply(ExpressionRewrite.this::applyRewriteToSink)
.toRule(RuleType.REWRITE_SINK_EXPRESSION);
}
}

private class LogicalOlapTableSinkRewrite extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalOlapTableSink().thenApply(ExpressionRewrite.this::applyRewriteToSink)
.toRule(RuleType.REWRITE_SINK_EXPRESSION);
}
}

private class LogicalDeferMaterializeResultSinkRewrite extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalDeferMaterializeResultSink().thenApply(ExpressionRewrite.this::applyRewriteToSink)
.toRule(RuleType.REWRITE_SINK_EXPRESSION);
}
}

private LogicalSink<Plan> applyRewriteToSink(MatchingContext<? extends LogicalSink<Plan>> ctx) {
LogicalSink<Plan> sink = ctx.root;
ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext);
List<NamedExpression> outputExprs = sink.getOutputExprs();
List<NamedExpression> result = rewriteAll(outputExprs, rewriter, context);
return sink.withOutputExprs(result);
}

/** LogicalRepeatRewrite */
public class LogicalRepeatRewrite extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalRepeat().thenApply(ctx -> {
Expand Down
Loading
Loading