Skip to content

branch-3.0:[improvement](statistics)Agg table set preagg on when doing sample analyzing. (#49918) #51672

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dataroaring merged 1 commit into from
Jun 13, 2025
Merged

branch-3.0:[improvement](statistics)Agg table set preagg on when doing sample analyzing. (#49918) #51672

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ public enum TableFrom {
private final List<Expression> joinFilters = new ArrayList<>();

private final List<Hint> hints = new ArrayList<>();
private boolean hintForcePreAggOn = false;

// the columns in Plan.getExpressions(), such as columns in join condition or filter condition, group by expression
private final Set<SlotReference> keySlots = Sets.newHashSet();
Expand Down Expand Up @@ -246,6 +247,14 @@ public void setNeedLockTables(boolean needLockTables) {
this.needLockTables = needLockTables;
}

public void setHintForcePreAggOn(boolean preAggOn) {
this.hintForcePreAggOn = preAggOn;
}

public boolean isHintForcePreAggOn() {
return hintForcePreAggOn;
}

/**
* cache view info to avoid view's def and sql mode changed before lock it.
*
Expand Down
4 changes: 3 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.apache.doris.nereids.rules.analysis.CollectJoinConstraint;
import org.apache.doris.nereids.rules.analysis.CollectSubQueryAlias;
import org.apache.doris.nereids.rules.analysis.EliminateDistinctConstant;
import org.apache.doris.nereids.rules.analysis.EliminateLogicalPreAggOnHint;
import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint;
import org.apache.doris.nereids.rules.analysis.FillUpMissingSlots;
import org.apache.doris.nereids.rules.analysis.HavingToFilter;
Expand Down Expand Up @@ -93,7 +94,8 @@ private static List<RewriteJob> buildAnalyzerJobs() {
return jobs(
// we should eliminate hint before "Subquery unnesting".
topDown(new AnalyzeCTE()),
topDown(new EliminateLogicalSelectHint()),
topDown(new EliminateLogicalSelectHint(),
new EliminateLogicalPreAggOnHint()),
bottomUp(
new BindRelation(),
new CheckPolicy()
Expand Down
138 changes: 81 additions & 57 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -459,6 +459,7 @@
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPreAggOnHint;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSelectHint;
Expand Down Expand Up @@ -1460,12 +1461,15 @@ public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationConte
return selectPlan;
}
List<ParserRuleContext> selectHintContexts = Lists.newArrayList();
List<ParserRuleContext> preAggOnHintContexts = Lists.newArrayList();
for (Integer key : selectHintMap.keySet()) {
if (key > selectCtx.getStart().getStopIndex() && key < selectCtx.getStop().getStartIndex()) {
selectHintContexts.add(selectHintMap.get(key));
} else {
preAggOnHintContexts.add(selectHintMap.get(key));
}
}
return withSelectHint(selectPlan, selectHintContexts);
return withHints(selectPlan, selectHintContexts, preAggOnHintContexts);
});
}

Expand Down Expand Up @@ -3274,73 +3278,93 @@ private LogicalPlan withJoinRelations(LogicalPlan input, RelationContext ctx) {
return last;
}

private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List<ParserRuleContext> hintContexts) {
if (hintContexts.isEmpty()) {
private LogicalPlan withHints(LogicalPlan logicalPlan, List<ParserRuleContext> selectHintContexts,
List<ParserRuleContext> preAggOnHintContexts) {
if (selectHintContexts.isEmpty() && preAggOnHintContexts.isEmpty()) {
return logicalPlan;
}
Map<String, SelectHint> hints = Maps.newLinkedHashMap();
for (ParserRuleContext hintContext : hintContexts) {
SelectHintContext selectHintContext = (SelectHintContext) hintContext;
for (HintStatementContext hintStatement : selectHintContext.hintStatements) {
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
switch (hintName) {
case "set_var":
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
Optional<String> value = Optional.empty();
if (kv.constantValue != null) {
Literal literal = (Literal) visit(kv.constantValue);
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
} else if (kv.identifierValue != null) {
// maybe we should throw exception when the identifierValue is quoted identifier
value = Optional.ofNullable(kv.identifierValue.getText());
LogicalPlan newPlan = logicalPlan;
if (!selectHintContexts.isEmpty()) {
Map<String, SelectHint> hints = Maps.newLinkedHashMap();
for (ParserRuleContext hintContext : selectHintContexts) {
SelectHintContext selectHintContext = (SelectHintContext) hintContext;
for (HintStatementContext hintStatement : selectHintContext.hintStatements) {
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
switch (hintName) {
case "set_var":
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
Optional<String> value = Optional.empty();
if (kv.constantValue != null) {
Literal literal = (Literal) visit(kv.constantValue);
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
} else if (kv.identifierValue != null) {
// maybe we should throw exception when the identifierValue is quoted identifier
value = Optional.ofNullable(kv.identifierValue.getText());
}
parameters.put(parameterName, value);
}
parameters.put(parameterName, value);
}
}
hints.put(hintName, new SelectHintSetVar(hintName, parameters));
break;
case "leading":
List<String> leadingParameters = new ArrayList<>();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
leadingParameters.add(parameterName);
hints.put(hintName, new SelectHintSetVar(hintName, parameters));
break;
case "leading":
List<String> leadingParameters = new ArrayList<>();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
leadingParameters.add(parameterName);
}
}
}
hints.put(hintName, new SelectHintLeading(hintName, leadingParameters));
break;
case "ordered":
hints.put(hintName, new SelectHintOrdered(hintName));
break;
case "use_cbo_rule":
List<String> useRuleParameters = new ArrayList<>();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
useRuleParameters.add(parameterName);
hints.put(hintName, new SelectHintLeading(hintName, leadingParameters));
break;
case "ordered":
hints.put(hintName, new SelectHintOrdered(hintName));
break;
case "use_cbo_rule":
List<String> useRuleParameters = new ArrayList<>();
for (HintAssignmentContext kv : hintStatement.parameters) {
if (kv.key != null) {
String parameterName = visitIdentifierOrText(kv.key);
useRuleParameters.add(parameterName);
}
}
}
hints.put(hintName, new SelectHintUseCboRule(hintName, useRuleParameters, false));
break;
case "no_use_cbo_rule":
List<String> noUseRuleParameters = new ArrayList<>();
for (HintAssignmentContext kv : hintStatement.parameters) {
String parameterName = visitIdentifierOrText(kv.key);
if (kv.key != null) {
noUseRuleParameters.add(parameterName);
hints.put(hintName, new SelectHintUseCboRule(hintName, useRuleParameters, false));
break;
case "no_use_cbo_rule":
List<String> noUseRuleParameters = new ArrayList<>();
for (HintAssignmentContext kv : hintStatement.parameters) {
String parameterName = visitIdentifierOrText(kv.key);
if (kv.key != null) {
noUseRuleParameters.add(parameterName);
}
}
hints.put(hintName, new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
break;
default:
break;
}
}
}
newPlan = new LogicalSelectHint<>(hints, newPlan);
}
if (!preAggOnHintContexts.isEmpty()) {
for (ParserRuleContext hintContext : preAggOnHintContexts) {
if (hintContext instanceof SelectHintContext) {
SelectHintContext preAggOnHintContext = (SelectHintContext) hintContext;
if (preAggOnHintContext.hintStatement != null
&& preAggOnHintContext.hintStatement.hintName != null) {
String text = preAggOnHintContext.hintStatement.hintName.getText();
if (text.equalsIgnoreCase("PREAGGOPEN")) {
newPlan = new LogicalPreAggOnHint<>(newPlan);
break;
}
hints.put(hintName, new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
break;
default:
break;
}
}
}
}
return new LogicalSelectHint<>(hints, logicalPlan);
return newPlan;
}

@Override
Expand Down
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ public enum RuleType {
ELIMINATE_GROUP_BY_CONSTANT(RuleTypeClass.REWRITE),

ELIMINATE_LOGICAL_SELECT_HINT(RuleTypeClass.REWRITE),
ELIMINATE_LOGICAL_PRE_AGG_ON_HINT(RuleTypeClass.REWRITE),
ELIMINATE_ORDER_BY_CONSTANT(RuleTypeClass.REWRITE),
ELIMINATE_ORDER_BY_UNDER_SUBQUERY(RuleTypeClass.REWRITE),
ELIMINATE_ORDER_BY_UNDER_VIEW(RuleTypeClass.REWRITE),
Expand Down
11 changes: 9 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,8 @@
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.util.ArrayList;
import java.util.List;
Expand All @@ -104,6 +106,7 @@
* Rule to bind relations in query plan.
*/
public class BindRelation extends OneAnalysisRuleFactory {
private static final Logger LOG = LogManager.getLogger(StatementContext.class);

public BindRelation() {}

Expand Down Expand Up @@ -178,7 +181,8 @@ private LogicalPlan bind(CascadesContext cascadesContext, UnboundRelation unboun
return getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
}

private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List<String> qualifier) {
private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List<String> qualifier,
CascadesContext cascadesContext) {
LogicalOlapScan scan;
List<Long> partIds = getPartitionIds(table, unboundRelation, qualifier);
List<Long> tabletIds = unboundRelation.getTabletIds();
Expand Down Expand Up @@ -214,6 +218,9 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation,
// This tabletIds is set manually, so need to set specifiedTabletIds
scan = scan.withManuallySpecifiedTabletIds(tabletIds);
}
if (cascadesContext.getStatementContext().isHintForcePreAggOn()) {
return scan.withPreAggStatus(PreAggStatus.on());
}
if (needGenerateLogicalAggForRandomDistAggTable(scan)) {
// it's a random distribution agg table
// add agg on olap scan
Expand Down Expand Up @@ -380,7 +387,7 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio
switch (table.getType()) {
case OLAP:
case MATERIALIZED_VIEW:
return makeOlapScan(table, unboundRelation, qualifierWithoutTableName);
return makeOlapScan(table, unboundRelation, qualifierWithoutTableName, cascadesContext);
case VIEW:
View view = (View) table;
isView = true;
Expand Down
39 changes: 39 additions & 0 deletions ...e/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalPreAggOnHint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.nereids.rules.analysis;

import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPreAggOnHint;

/**
* eliminate logical common hint and set them to cascade context
*/
public class EliminateLogicalPreAggOnHint extends OneRewriteRuleFactory {

@Override
public Rule build() {
return logicalPreAggOnHint().thenApply(ctx -> {
LogicalPreAggOnHint<Plan> preAggHintPlan = ctx.root;
ctx.statementContext.setHintForcePreAggOn(true);
return preAggHintPlan.child();
}).toRule(RuleType.ELIMINATE_LOGICAL_PRE_AGG_ON_HINT);
}
}
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ public enum PlanType {
LOGICAL_APPLY,
LOGICAL_ASSERT_NUM_ROWS,
LOGICAL_CHECK_POLICY,
LOGICAL_COMMON_HINT,
LOGICAL_CTE,
LOGICAL_CTE_ANCHOR,
LOGICAL_CTE_PRODUCER,
Expand Down
4 changes: 3 additions & 1 deletion ...e-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.apache.doris.nereids.rules.analysis.BindExpression;
import org.apache.doris.nereids.rules.analysis.BindRelation;
import org.apache.doris.nereids.rules.analysis.CheckPolicy;
import org.apache.doris.nereids.rules.analysis.EliminateLogicalPreAggOnHint;
import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
Expand Down Expand Up @@ -271,7 +272,8 @@ public List<RewriteJob> getJobs() {

private static List<RewriteJob> buildAnalyzeViewJobsForStar() {
return jobs(
topDown(new EliminateLogicalSelectHint()),
topDown(new EliminateLogicalSelectHint(),
new EliminateLogicalPreAggOnHint()),
topDown(new AnalyzeCTE()),
bottomUp(
new BindRelation(),
Expand Down
Loading
Loading