Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Run filter-into-join rule early for subqueries and disable project-filter rule #15511

Merged
merged 5 commits into from
Jan 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Injector;
import com.google.inject.Module;
import org.apache.druid.guice.DruidInjectorBuilder;
Expand Down Expand Up @@ -194,4 +195,40 @@ public void testArrayAggQueryOnComplexDatatypes()
);
}
}

@Test(timeout = 40000)
public void testJoinMultipleTablesWithWhereCondition()
{
testBuilder()
.queryContext(
ImmutableMap.of(
"sqlJoinAlgorithm", "sortMerge"
)
)
.sql(
"SELECT f2.dim3,sum(f6.m1 * (1- f6.m2)) FROM"
+ " druid.foo as f5, "
+ " druid.foo as f6, "
+ " druid.numfoo as f7, "
+ " druid.foo2 as f2, "
+ " druid.numfoo as f3, "
+ " druid.foo as f4, "
+ " druid.numfoo as f1, "
+ " druid.foo2 as f8 "
+ "where true"
+ " and f1.dim1 = f2.dim2 "
+ " and f3.dim1 = f4.dim2 "
+ " and f5.dim1 = f6.dim2 "
+ " and f7.dim2 = f8.dim3 "
+ " and f2.dim1 = f4.dim2 "
+ " and f6.dim1 = f8.dim2 "
+ " and f1.dim1 = f7.dim2 "
+ " and f8.dim2 = 'x' "
+ " and f3.__time >= date '2011-11-11' "
+ " and f3.__time < date '2013-11-11' "
+ "group by 1 "
+ "order by 2 desc limit 1001"
)
.run();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -87,14 +87,15 @@ public class CalciteRulesManager
* those functions).
* 3) {@link CoreRules#JOIN_COMMUTE}, {@link JoinPushThroughJoinRule#RIGHT}, {@link JoinPushThroughJoinRule#LEFT},
* and {@link CoreRules#FILTER_INTO_JOIN}, which are part of {@link #FANCY_JOIN_RULES}.
* 4) {@link CoreRules#PROJECT_FILTER_TRANSPOSE} because PartialDruidQuery would like to have the Project on top of the Filter -
* this rule could create a lot of non-usefull plans.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
* this rule could create a lot of non-usefull plans.
* this rule could create a lot of non-useful plans that unnecessarily increase the planning time.

*/
private static final List<RelOptRule> BASE_RULES =
ImmutableList.of(
CoreRules.AGGREGATE_STAR_TABLE,
CoreRules.AGGREGATE_PROJECT_STAR_TABLE,
CoreRules.PROJECT_MERGE,
CoreRules.FILTER_SCAN,
CoreRules.PROJECT_FILTER_TRANSPOSE,
kgyrtkirk marked this conversation as resolved.
Show resolved Hide resolved
CoreRules.FILTER_PROJECT_TRANSPOSE,
CoreRules.JOIN_PUSH_EXPRESSIONS,
CoreRules.AGGREGATE_EXPAND_WITHIN_DISTINCT,
Expand Down Expand Up @@ -237,7 +238,7 @@ public List<Program> programs(final PlannerContext plannerContext)
prePrograms.add(new LoggingProgram("Finished decorrelate and trim fields program", isDebug));
prePrograms.add(buildCoalesceProgram());
prePrograms.add(new LoggingProgram("Finished coalesce program", isDebug));
prePrograms.add(buildHepProgram(REDUCTION_RULES));
prePrograms.add(buildReductionProgram(plannerContext));
prePrograms.add(new LoggingProgram("Finished expression reduction program", isDebug));

final Program preProgram = Programs.sequence(prePrograms.toArray(new Program[0]));
Expand All @@ -261,6 +262,18 @@ public List<Program> programs(final PlannerContext plannerContext)
);
}

private Program buildReductionProgram(final PlannerContext plannerContext)
{
List<RelOptRule> hepRules = new ArrayList<RelOptRule>(REDUCTION_RULES);
// Apply CoreRules#FILTER_INTO_JOIN early to avoid exploring less optimal plans.
if (plannerContext.getJoinAlgorithm().requiresSubquery()) {
hepRules.add(CoreRules.FILTER_INTO_JOIN);
}
return buildHepProgram(
hepRules
);
}

private static class LoggingProgram implements Program
{
private final String stage;
Expand Down
Loading