diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java index 46edad00f7a0..fa0bbb307a16 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java @@ -26,12 +26,10 @@ import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelShuttle; -import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; -import org.apache.calcite.sql.SqlExplainLevel; import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.util.Util; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; @@ -46,8 +44,6 @@ public class HiveProject extends Project implements HiveRelNode { // Information about correlations within a subquery. private final CorrelationInfoSupplier correlationInfos; - private boolean isSysnthetic; - /** * Creates a HiveProject. * @param cluster @@ -110,22 +106,7 @@ public static HiveProject create(RelOptCluster cluster, RelNode child, List exps, RelDataType rowType) { assert traitSet.containsIfApplicable(HiveRelNode.CONVENTION); - HiveProject hp = new HiveProject(getCluster(), traitSet, input, exps, rowType); - if (this.isSynthetic()) { - hp.setSynthetic(); - } - - return hp; - } - - // TODO: this should come through RelBuilder to the constructor as opposed to - // set method. This requires calcite change - public void setSynthetic() { - this.isSysnthetic = true; - } - - public boolean isSynthetic() { - return isSysnthetic; + return new HiveProject(getCluster(), traitSet, input, exps, rowType); } //required for HiveRelDecorrelator @@ -136,12 +117,6 @@ public boolean isSynthetic() { return shuttle.visit(this); } - @Override - public RelWriter explainTerms(RelWriter pw) { - return super.explainTerms(pw) - .itemIf("synthetic", this.isSysnthetic, pw.getDetailLevel() == SqlExplainLevel.DIGEST_ATTRIBUTES); - } - public List getCorrelationInfos() { return correlationInfos.get(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java index 2c3e4c6e6e3d..05e8b448f626 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java @@ -261,13 +261,8 @@ public RelNode project(ImmutableBitSet fieldsUsed, Set extraFi fieldNames)); // 5. Add Proj on top of TS - HiveProject hp = (HiveProject) relBuilder.push(newHT) + return relBuilder.push(newHT) .project(exprList, new ArrayList(fieldNames)).build(); - - // 6. Set synthetic flag, so that we would push filter below this one - hp.setSynthetic(); - - return hp; } public List getNeededColIndxsFrmReloptHT() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java deleted file mode 100644 index 6eace95a4a28..000000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.optimizer.calcite.rules; - -import org.apache.calcite.adapter.druid.DruidQuery; -import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Filter; -import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.RelFactories; -import org.apache.calcite.rel.core.RelFactories.FilterFactory; -import org.apache.calcite.rel.core.RelFactories.ProjectFactory; -import org.apache.calcite.rel.core.TableScan; -import org.apache.calcite.rel.rules.ProjectMergeRule; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexOver; -import org.apache.calcite.rex.RexUtil; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; - -import java.util.Collections; - -//TODO: Remove this once Calcite FilterProjectTransposeRule can take rule operand -public class HiveFilterProjectTSTransposeRule extends RelOptRule { - - public final static HiveFilterProjectTSTransposeRule INSTANCE = - new HiveFilterProjectTSTransposeRule( - Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class, - HiveRelFactories.HIVE_PROJECT_FACTORY, TableScan.class, ProjectMergeRule.DEFAULT_BLOAT); - - public final static HiveFilterProjectTSTransposeRule INSTANCE_DRUID = - new HiveFilterProjectTSTransposeRule( - Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class, - HiveRelFactories.HIVE_PROJECT_FACTORY, DruidQuery.class, ProjectMergeRule.DEFAULT_BLOAT); - - private final RelFactories.FilterFactory filterFactory; - private final RelFactories.ProjectFactory projectFactory; - private final int bloat; - - private HiveFilterProjectTSTransposeRule(Class filterClass, - FilterFactory filterFactory, Class projectClass, - ProjectFactory projectFactory, Class tsClass, - int bloat) { - super(operand(filterClass, operand(projectClass, operand(tsClass, none())))); - this.filterFactory = filterFactory; - this.projectFactory = projectFactory; - this.bloat = bloat; - } - - @Override - public boolean matches(RelOptRuleCall call) { - final HiveProject projRel = call.rel(1); - - // Assumption: - // 1. This will be run last after PP, Col Pruning in the PreJoinOrder - // optimizations. - // 2. If ProjectRel is not synthetic then PPD would have already pushed - // relevant pieces down and hence no point in running PPD again. - // 3. For synthetic Projects we don't care about non deterministic UDFs - if (!projRel.isSynthetic()) { - return false; - } - - return super.matches(call); - } - - // ~ Methods ---------------------------------------------------------------- - - // implement RelOptRule - public void onMatch(RelOptRuleCall call) { - final Filter filter = call.rel(0); - final Project project = call.rel(1); - - if (RexOver.containsOver(project.getProjects(), null)) { - // In general a filter cannot be pushed below a windowing calculation. - // Applying the filter before the aggregation function changes - // the results of the windowing invocation. - // - // When the filter is on the PARTITION BY expression of the OVER clause - // it can be pushed down. For now we don't support this. - return; - } - - if (RexUtil.containsCorrelation(filter.getCondition())) { - // If there is a correlation condition anywhere in the filter, don't - // push this filter past project since in some cases it can prevent a - // Correlate from being de-correlated. - return; - } - - // convert the filter to one that references the child of the project - RexNode newCondition = HiveRelOptUtil.pushPastProjectUnlessBloat(filter.getCondition(), project, bloat); - if (newCondition == null) { - return; - } - - // Remove cast of BOOLEAN NOT NULL to BOOLEAN or vice versa. Filter accepts - // nullable and not-nullable conditions, but a CAST might get in the way of - // other rewrites. - final RelDataTypeFactory typeFactory = filter.getCluster().getTypeFactory(); - if (RexUtil.isNullabilityCast(typeFactory, newCondition)) { - newCondition = ((RexCall) newCondition).getOperands().get(0); - } - - RelNode newFilterRel = filterFactory == null ? filter.copy(filter.getTraitSet(), - project.getInput(), newCondition) : filterFactory.createFilter(project.getInput(), - newCondition); - - RelNode newProjRel = projectFactory == null ? project.copy(project.getTraitSet(), newFilterRel, - project.getProjects(), project.getRowType()) : projectFactory.createProject(newFilterRel, - Collections.emptyList(), project.getProjects(), project.getRowType().getFieldNames()); - - call.transformTo(newProjRel); - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java index 819231cb54d7..5db95b0ca48f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java @@ -23,7 +23,9 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; +import org.apache.calcite.adapter.druid.DruidQuery; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptPredicateList; import org.apache.calcite.plan.RelOptRuleCall; @@ -35,6 +37,7 @@ import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.rules.FilterProjectTransposeRule; import org.apache.calcite.rel.rules.ProjectMergeRule; import org.apache.calcite.rel.type.RelDataTypeFactory; @@ -59,14 +62,22 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil.RewritablePKFKJoinInfo; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; public class HiveFilterProjectTransposeRule extends FilterProjectTransposeRule { + private static RelOptRuleOperand joinOperand() { + Predicate isRewritable = join -> { + RelMetadataQuery mq = join.getCluster().getMetadataQuery(); + return HiveRelOptUtil.isRewritablePKFKJoin(join, join.getLeft(), join.getRight(), mq).rewritable; + }; + return operandJ(Join.class, null, isRewritable, any()); + } + public static final HiveFilterProjectTransposeRule DETERMINISTIC_WINDOWING_ON_NON_FILTERING_JOIN = new HiveFilterProjectTransposeRule( - operand(Filter.class, operand(Project.class, operand(Join.class, any()))), + operand(Filter.class, operand(Project.class, joinOperand())), HiveRelFactories.HIVE_BUILDER, true, true, ProjectMergeRule.DEFAULT_BLOAT); public static final HiveFilterProjectTransposeRule DETERMINISTIC_WINDOWING = @@ -75,7 +86,7 @@ public class HiveFilterProjectTransposeRule extends FilterProjectTransposeRule { public static final HiveFilterProjectTransposeRule DETERMINISTIC_ON_NON_FILTERING_JOIN = new HiveFilterProjectTransposeRule( - operand(Filter.class, operand(Project.class, operand(Join.class, any()))), + operand(Filter.class, operand(Project.class, joinOperand())), HiveRelFactories.HIVE_BUILDER, true, false, ProjectMergeRule.DEFAULT_BLOAT); public static final HiveFilterProjectTransposeRule DETERMINISTIC = @@ -86,6 +97,20 @@ public class HiveFilterProjectTransposeRule extends FilterProjectTransposeRule { new HiveFilterProjectTransposeRule(Filter.class, HiveProject.class, HiveRelFactories.HIVE_BUILDER, false, false, ProjectMergeRule.DEFAULT_BLOAT); + public static final HiveFilterProjectTransposeRule SCAN = new HiveFilterProjectTransposeRule( + operand(Filter.class, operand(HiveProject.class, operand(HiveTableScan.class, none()))), + HiveRelFactories.HIVE_BUILDER, + false, + false, + ProjectMergeRule.DEFAULT_BLOAT); + + public static final HiveFilterProjectTransposeRule DRUID = new HiveFilterProjectTransposeRule( + operand(Filter.class, operand(HiveProject.class, operand(DruidQuery.class, none()))), + HiveRelFactories.HIVE_BUILDER, + false, + false, + ProjectMergeRule.DEFAULT_BLOAT); + private final boolean onlyDeterministic; private final boolean pushThroughWindowing; @@ -127,16 +152,6 @@ public boolean matches(RelOptRuleCall call) { if (this.onlyDeterministic && !HiveCalciteUtil.isDeterministic(condition)) { return false; } - - if (call.rels.length > 2) { - final Join joinRel = call.rel(2); - RewritablePKFKJoinInfo joinInfo = HiveRelOptUtil.isRewritablePKFKJoin( - joinRel, joinRel.getLeft(), joinRel.getRight(), call.getMetadataQuery()); - if (!joinInfo.rewritable) { - return false; - } - } - return super.matches(call); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java index 3d507f4ebf9a..f6f011fe5823 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java @@ -69,7 +69,6 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveMultiJoin; -import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortExchange; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableFunctionScan; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; @@ -390,10 +389,7 @@ private static RelNode project(DruidQuery dq, ImmutableBitSet fieldsUsed, true)); nameList.add(extraField.getName()); } - - HiveProject hp = (HiveProject) relBuilder.push(dq).project(exprList, nameList).build(); - hp.setSynthetic(); - return hp; + return relBuilder.push(dq).project(exprList, nameList).build(); } private boolean isRexLiteral(final RexNode rexNode) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index eb24496743fb..f1e9da25ebec 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -229,7 +229,6 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFieldTrimmerRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterAggregateTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterJoinRule; -import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterProjectTSTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterProjectTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterSetOpTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterSortPredicates; @@ -1933,7 +1932,7 @@ protected RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataPr // matches FIL-PROJ-TS // Also merge, remove and reduce Project if possible generatePartialProgram(program, true, HepMatchOrder.TOP_DOWN, - HiveFilterProjectTSTransposeRule.INSTANCE, HiveFilterProjectTSTransposeRule.INSTANCE_DRUID, + HiveFilterProjectTransposeRule.SCAN, HiveFilterProjectTransposeRule.DRUID, HiveProjectFilterPullUpConstantsRule.INSTANCE, HiveProjectMergeRule.INSTANCE, ProjectRemoveRule.Config.DEFAULT.toRule(), HiveSortMergeRule.INSTANCE); @@ -2100,7 +2099,7 @@ private RelNode rewriteUsingViews(RelOptPlanner planner, RelNode basePlan, // Unboxing rule planner.addRule(HiveMaterializedViewBoxing.INSTANCE_UNBOXING); // Partition pruner rule - planner.addRule(HiveFilterProjectTSTransposeRule.INSTANCE); + planner.addRule(HiveFilterProjectTransposeRule.SCAN); planner.addRule(new HivePartitionPruneRule(conf)); // Optimize plan @@ -2411,7 +2410,7 @@ private RelNode applyPostJoinOrderingTransform(RelNode basePlan, RelMetadataProv generatePartialProgram(program, false, HepMatchOrder.DEPTH_FIRST, ProjectRemoveRule.Config.DEFAULT.toRule(), new ProjectMergeRule(false, HiveRelFactories.HIVE_BUILDER)); generatePartialProgram(program, true, HepMatchOrder.TOP_DOWN, - HiveFilterProjectTSTransposeRule.INSTANCE, HiveFilterProjectTSTransposeRule.INSTANCE_DRUID, + HiveFilterProjectTransposeRule.SCAN, HiveFilterProjectTransposeRule.DRUID, HiveProjectFilterPullUpConstantsRule.INSTANCE); // 9.2. Introduce exchange operators below join/multijoin operators