planner: trace predicate push down

planner/core/logical_plan_trace_test.go

@@ -86,6 +86,44 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) {
  assertRuleName string
  assertRuleSteps []assertTraceStep
  }{
+ {
+ sql: "select * from t as t1 join t as t2 on t1.a = t2.a where t1.a < 1;",
+ flags: []uint64{flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns},
+ assertRuleName: "predicate_push_down",
+ assertRuleSteps: []assertTraceStep{
+ {
+ assertReason: "",
+ assertAction: "The conditions[lt(test.t.a, 1)] are pushed down across DataSource_1",
+ },
+ {
+ assertReason: "",
+ assertAction: "The conditions[lt(test.t.a, 1)] are pushed down across DataSource_2",
+ },
+ {
+ assertAction: "Selection_4 is removed",
+ assertReason: "The conditions[eq(test.t.a, test.t.a)] in Selection_4 are pushed down",
+ },
+ {
+ assertAction: "Selection_5 is removed",
+ assertReason: "The conditions[lt(test.t.a, 1)] in Selection_5 are pushed down",
+ },
+ },
+ },
+ {
+ sql: "select * from t where a < 1;",
+ flags: []uint64{flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns},
+ assertRuleName: "predicate_push_down",
+ assertRuleSteps: []assertTraceStep{
+ {
+ assertReason: "",
+ assertAction: "The conditions[lt(test.t.a, 1)] are pushed down across DataSource_1",
+ },
+ {
+ assertReason: "The conditions[lt(test.t.a, 1)] in Selection_2 are pushed down",
+ assertAction: "Selection_2 is removed",
+ },
+ },
+ },
  {
  sql: "select * from t as t1 left join t as t2 on t1.a = t2.a order by t1.a limit 10;",
  flags: []uint64{flagPrunColumns, flagBuildKeyInfo, flagPushDownTopN},

planner/core/plan.go

@@ -237,7 +237,7 @@ type LogicalPlan interface {
  // PredicatePushDown pushes down the predicates in the where/on/having clauses as deeply as possible.
  // It will accept a predicate that is an expression slice, and return the expressions that can't be pushed.
  // Because it might change the root if the having clause exists, we need to return a plan that represents a new root.
- PredicatePushDown([]expression.Expression) ([]expression.Expression, LogicalPlan)
+ PredicatePushDown([]expression.Expression, *logicalOptimizeOp) ([]expression.Expression, LogicalPlan)
 
  // PruneColumns prunes the unused columns.
  PruneColumns([]*expression.Column) error

planner/core/rule_predicate_push_down.go

@@ -15,7 +15,9 @@
 package core
 
 import (
+ "bytes"
  "context"
+ "fmt"
 
  "github.com/pingcap/tidb/expression"
  "github.com/pingcap/tidb/kv"
@@ -28,11 +30,11 @@ import (
 type ppdSolver struct{}
 
 func (s *ppdSolver) optimize(ctx context.Context, lp LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) {
- _, p := lp.PredicatePushDown(nil)
+ _, p := lp.PredicatePushDown(nil, opt)
  return p, nil
 }
 
-func addSelection(p LogicalPlan, child LogicalPlan, conditions []expression.Expression, chIdx int) {
+func addSelection(p LogicalPlan, child LogicalPlan, conditions []expression.Expression, chIdx int, opt *logicalOptimizeOp) {
  if len(conditions) == 0 {
  p.Children()[chIdx] = child
  return
@@ -42,6 +44,7 @@ func addSelection(p LogicalPlan, child LogicalPlan, conditions []expression.Expr
  dual := Conds2TableDual(child, conditions)
  if dual != nil {
  p.Children()[chIdx] = dual
+ appendTableDualTraceStep(child, dual, conditions, opt)
  return
  }
 
@@ -53,16 +56,17 @@ func addSelection(p LogicalPlan, child LogicalPlan, conditions []expression.Expr
  selection := LogicalSelection{Conditions: conditions}.Init(p.SCtx(), p.SelectBlockOffset())
  selection.SetChildren(child)
  p.Children()[chIdx] = selection
+ appendAddSelectionTraceStep(p, child, selection, opt)
 }
 
 // PredicatePushDown implements LogicalPlan interface.
-func (p *baseLogicalPlan) PredicatePushDown(predicates []expression.Expression) ([]expression.Expression, LogicalPlan) {
+func (p *baseLogicalPlan) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) {
  if len(p.children) == 0 {
  return predicates, p.self
  }
  child := p.children[0]
- rest, newChild := child.PredicatePushDown(predicates)
- addSelection(p.self, newChild, rest, 0)
+ rest, newChild := child.PredicatePushDown(predicates, opt)
+ addSelection(p.self, newChild, rest, 0, opt)
  return nil, p.self
 }
 
@@ -80,56 +84,61 @@ func splitSetGetVarFunc(filters []expression.Expression) ([]expression.Expressio
 }
 
 // PredicatePushDown implements LogicalPlan PredicatePushDown interface.
-func (p *LogicalSelection) PredicatePushDown(predicates []expression.Expression) ([]expression.Expression, LogicalPlan) {
+func (p *LogicalSelection) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) {
  predicates = DeleteTrueExprs(p, predicates)
  p.Conditions = DeleteTrueExprs(p, p.Conditions)
  var child LogicalPlan
  var retConditions []expression.Expression
+ var originConditions []expression.Expression
  if p.buildByHaving {
- retConditions, child = p.children[0].PredicatePushDown(predicates)
+ retConditions, child = p.children[0].PredicatePushDown(predicates, opt)
  retConditions = append(retConditions, p.Conditions...)
  } else {
  canBePushDown, canNotBePushDown := splitSetGetVarFunc(p.Conditions)
- retConditions, child = p.children[0].PredicatePushDown(append(canBePushDown, predicates...))
+ originConditions = canBePushDown
+ retConditions, child = p.children[0].PredicatePushDown(append(canBePushDown, predicates...), opt)
  retConditions = append(retConditions, canNotBePushDown...)
  }
  if len(retConditions) > 0 {
  p.Conditions = expression.PropagateConstant(p.ctx, retConditions)
  // Return table dual when filter is constant false or null.
  dual := Conds2TableDual(p, p.Conditions)
  if dual != nil {
+ appendTableDualTraceStep(p, dual, p.Conditions, opt)
  return nil, dual
  }
  return nil, p
  }
+ appendSelectionPredicatePushDownTraceStep(p, originConditions, opt)
  return nil, child
 }
 
 // PredicatePushDown implements LogicalPlan PredicatePushDown interface.
-func (p *LogicalUnionScan) PredicatePushDown(predicates []expression.Expression) ([]expression.Expression, LogicalPlan) {
- retainedPredicates, _ := p.children[0].PredicatePushDown(predicates)
+func (p *LogicalUnionScan) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) {
+ retainedPredicates, _ := p.children[0].PredicatePushDown(predicates, opt)
  p.conditions = make([]expression.Expression, 0, len(predicates))
  p.conditions = append(p.conditions, predicates...)
  // The conditions in UnionScan is only used for added rows, so parent Selection should not be removed.
  return retainedPredicates, p
 }
 
 // PredicatePushDown implements LogicalPlan PredicatePushDown interface.
-func (ds *DataSource) PredicatePushDown(predicates []expression.Expression) ([]expression.Expression, LogicalPlan) {
+func (ds *DataSource) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) {
  predicates = expression.PropagateConstant(ds.ctx, predicates)
  predicates = DeleteTrueExprs(ds, predicates)
  ds.allConds = predicates
  ds.pushedDownConds, predicates = expression.PushDownExprs(ds.ctx.GetSessionVars().StmtCtx, predicates, ds.ctx.GetClient(), kv.UnSpecified)
+ appendDataSourcePredicatePushDownTraceStep(ds, opt)
  return predicates, ds
 }
 
 // PredicatePushDown implements LogicalPlan PredicatePushDown interface.
-func (p *LogicalTableDual) PredicatePushDown(predicates []expression.Expression) ([]expression.Expression, LogicalPlan) {
+func (p *LogicalTableDual) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) {
  return predicates, p
 }
 
 // PredicatePushDown implements LogicalPlan PredicatePushDown interface.
-func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression) (ret []expression.Expression, retPlan LogicalPlan) {
+func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) (ret []expression.Expression, retPlan LogicalPlan) {
  simplifyOuterJoin(p, predicates)
  var equalCond []*expression.ScalarFunction
  var leftPushCond, rightPushCond, otherCond, leftCond, rightCond []expression.Expression
@@ -138,6 +147,7 @@ func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression) (ret
  predicates = p.outerJoinPropConst(predicates)
  dual := Conds2TableDual(p, predicates)
  if dual != nil {
+ appendTableDualTraceStep(p, dual, predicates, opt)
  return ret, dual
  }
  // Handle where conditions
@@ -156,6 +166,7 @@ func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression) (ret
  predicates = p.outerJoinPropConst(predicates)
  dual := Conds2TableDual(p, predicates)
  if dual != nil {
+ appendTableDualTraceStep(p, dual, predicates, opt)
  return ret, dual
  }
  // Handle where conditions
@@ -182,6 +193,7 @@ func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression) (ret
  // Return table dual when filter is constant false or null.
  dual := Conds2TableDual(p, tempCond)
  if dual != nil {
+ appendTableDualTraceStep(p, dual, tempCond, opt)
  return ret, dual
  }
  equalCond, leftPushCond, rightPushCond, otherCond = p.extractOnCondition(tempCond, true, true)
@@ -196,6 +208,7 @@ func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression) (ret
  // Return table dual when filter is constant false or null.
  dual := Conds2TableDual(p, predicates)
  if dual != nil {
+ appendTableDualTraceStep(p, dual, predicates, opt)
  return ret, dual
  }
  // `predicates` should only contain left conditions or constant filters.
@@ -212,10 +225,10 @@ func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression) (ret
  }
  leftCond = expression.RemoveDupExprs(p.ctx, leftCond)
  rightCond = expression.RemoveDupExprs(p.ctx, rightCond)
- leftRet, lCh := p.children[0].PredicatePushDown(leftCond)
- rightRet, rCh := p.children[1].PredicatePushDown(rightCond)
- addSelection(p, lCh, leftRet, 0)
- addSelection(p, rCh, rightRet, 1)
+ leftRet, lCh := p.children[0].PredicatePushDown(leftCond, opt)
+ rightRet, rCh := p.children[1].PredicatePushDown(rightCond, opt)
+ addSelection(p, lCh, leftRet, 0, opt)
+ addSelection(p, rCh, rightRet, 1, opt)
  p.updateEQCond()
  buildKeyInfo(p)
  return ret, p.self
@@ -380,12 +393,12 @@ func isNullRejected(ctx sessionctx.Context, schema *expression.Schema, expr expr
 }
 
 // PredicatePushDown implements LogicalPlan PredicatePushDown interface.
-func (p *LogicalProjection) PredicatePushDown(predicates []expression.Expression) (ret []expression.Expression, retPlan LogicalPlan) {
+func (p *LogicalProjection) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) (ret []expression.Expression, retPlan LogicalPlan) {
  canBePushed := make([]expression.Expression, 0, len(predicates))
  canNotBePushed := make([]expression.Expression, 0, len(predicates))
  for _, expr := range p.Exprs {
  if expression.HasAssignSetVarFunc(expr) {
- _, child := p.baseLogicalPlan.PredicatePushDown(nil)
+ _, child := p.baseLogicalPlan.PredicatePushDown(nil, opt)
  return predicates, child
  }
  }
@@ -397,23 +410,23 @@ func (p *LogicalProjection) PredicatePushDown(predicates []expression.Expression
  canNotBePushed = append(canNotBePushed, cond)
  }
  }
- remained, child := p.baseLogicalPlan.PredicatePushDown(canBePushed)
+ remained, child := p.baseLogicalPlan.PredicatePushDown(canBePushed, opt)
  return append(remained, canNotBePushed...), child
 }
 
 // PredicatePushDown implements LogicalPlan PredicatePushDown interface.
-func (p *LogicalUnionAll) PredicatePushDown(predicates []expression.Expression) (ret []expression.Expression, retPlan LogicalPlan) {
+func (p *LogicalUnionAll) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) (ret []expression.Expression, retPlan LogicalPlan) {
  for i, proj := range p.children {
  newExprs := make([]expression.Expression, 0, len(predicates))
  newExprs = append(newExprs, predicates...)
- retCond, newChild := proj.PredicatePushDown(newExprs)
- addSelection(p, newChild, retCond, i)
+ retCond, newChild := proj.PredicatePushDown(newExprs, opt)
+ addSelection(p, newChild, retCond, i, opt)
  }
  return nil, p
 }
 
 // PredicatePushDown implements LogicalPlan PredicatePushDown interface.
-func (la *LogicalAggregation) PredicatePushDown(predicates []expression.Expression) (ret []expression.Expression, retPlan LogicalPlan) {
+func (la *LogicalAggregation) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) (ret []expression.Expression, retPlan LogicalPlan) {
  var condsToPush []expression.Expression
  exprsOriginal := make([]expression.Expression, 0, len(la.AggFuncs))
  for _, fun := range la.AggFuncs {
@@ -447,21 +460,21 @@ func (la *LogicalAggregation) PredicatePushDown(predicates []expression.Expressi
  ret = append(ret, cond)
  }
  }
- la.baseLogicalPlan.PredicatePushDown(condsToPush)
+ la.baseLogicalPlan.PredicatePushDown(condsToPush, opt)
  return ret, la
 }
 
 // PredicatePushDown implements LogicalPlan PredicatePushDown interface.
-func (p *LogicalLimit) PredicatePushDown(predicates []expression.Expression) ([]expression.Expression, LogicalPlan) {
+func (p *LogicalLimit) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) {
  // Limit forbids any condition to push down.
- p.baseLogicalPlan.PredicatePushDown(nil)
+ p.baseLogicalPlan.PredicatePushDown(nil, opt)
  return predicates, p
 }
 
 // PredicatePushDown implements LogicalPlan PredicatePushDown interface.
-func (p *LogicalMaxOneRow) PredicatePushDown(predicates []expression.Expression) ([]expression.Expression, LogicalPlan) {
+func (p *LogicalMaxOneRow) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) {
  // MaxOneRow forbids any condition to push down.
- p.baseLogicalPlan.PredicatePushDown(nil)
+ p.baseLogicalPlan.PredicatePushDown(nil, opt)
  return predicates, p
 }
 
@@ -610,7 +623,7 @@ func (p *LogicalWindow) GetPartitionByCols() []*expression.Column {
 }
 
 // PredicatePushDown implements LogicalPlan PredicatePushDown interface.
-func (p *LogicalWindow) PredicatePushDown(predicates []expression.Expression) ([]expression.Expression, LogicalPlan) {
+func (p *LogicalWindow) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) {
  canBePushed := make([]expression.Expression, 0, len(predicates))
  canNotBePushed := make([]expression.Expression, 0, len(predicates))
  partitionCols := expression.NewSchema(p.GetPartitionByCols()...)
@@ -623,12 +636,12 @@ func (p *LogicalWindow) PredicatePushDown(predicates []expression.Expression) ([
  canNotBePushed = append(canNotBePushed, cond)
  }
  }
- p.baseLogicalPlan.PredicatePushDown(canBePushed)
+ p.baseLogicalPlan.PredicatePushDown(canBePushed, opt)
  return canNotBePushed, p
 }
 
 // PredicatePushDown implements LogicalPlan PredicatePushDown interface.
-func (p *LogicalMemTable) PredicatePushDown(predicates []expression.Expression) ([]expression.Expression, LogicalPlan) {
+func (p *LogicalMemTable) PredicatePushDown(predicates []expression.Expression, opt *logicalOptimizeOp) ([]expression.Expression, LogicalPlan) {
  if p.Extractor != nil {
  predicates = p.Extractor.Extract(p.ctx, p.schema, p.names, predicates)
  }
@@ -638,3 +651,75 @@ func (p *LogicalMemTable) PredicatePushDown(predicates []expression.Expression)
 func (*ppdSolver) name() string {
  return "predicate_push_down"
 }
+
+func appendTableDualTraceStep(replaced LogicalPlan, dual LogicalPlan, conditions []expression.Expression, opt *logicalOptimizeOp) {
+ action := func() string {
+ return fmt.Sprintf("%v_%v is replaced by %v_%v", replaced.TP(), replaced.ID(), dual.TP(), dual.ID())
+ }
+ reason := func() string {
+ buffer := bytes.NewBufferString("The conditions[")
+ for i, cond := range conditions {
+ if i > 0 {
+ buffer.WriteString(",")
+ }
+ buffer.WriteString(cond.String())
+ }
+ buffer.WriteString("] are constant false or null")
+ return buffer.String()
+ }
+ opt.appendStepToCurrent(dual.ID(), dual.TP(), reason, action)
+}
+
+func appendSelectionPredicatePushDownTraceStep(p *LogicalSelection, conditions []expression.Expression, opt *logicalOptimizeOp) {
+ action := func() string {
+ return fmt.Sprintf("%v_%v is removed", p.TP(), p.ID())
+ }
+ reason := func() string {
+ return ""
+ }
+ if len(conditions) > 0 && !p.buildByHaving {
+ reason = func() string {
+ buffer := bytes.NewBufferString("The conditions[")
+ for i, cond := range conditions {
+ if i > 0 {
+ buffer.WriteString(",")
+ }
+ buffer.WriteString(cond.String())
+ }
+ buffer.WriteString(fmt.Sprintf("] in %v_%v are pushed down", p.TP(), p.ID()))
+ return buffer.String()
+ }
+ }
+ opt.appendStepToCurrent(p.ID(), p.TP(), reason, action)
+}
+
+func appendDataSourcePredicatePushDownTraceStep(ds *DataSource, opt *logicalOptimizeOp) {
+ if len(ds.pushedDownConds) < 1 {
+ return
+ }
+ reason := func() string {
+ return ""
+ }
+ action := func() string {
+ buffer := bytes.NewBufferString("The conditions[")
+ for i, cond := range ds.pushedDownConds {
+ if i > 0 {
+ buffer.WriteString(",")
+ }
+ buffer.WriteString(cond.String())
+ }
+ buffer.WriteString(fmt.Sprintf("] are pushed down across %v_%v", ds.TP(), ds.ID()))
+ return buffer.String()
+ }
+ opt.appendStepToCurrent(ds.ID(), ds.TP(), reason, action)
+}
+
+func appendAddSelectionTraceStep(p LogicalPlan, child LogicalPlan, sel *LogicalSelection, opt *logicalOptimizeOp) {
+ reason := func() string {
+ return ""
+ }
+ action := func() string {
+ return fmt.Sprintf("add %v_%v to connect %v_%v and %v_%v", sel.TP(), sel.ID(), p.TP(), p.ID(), child.TP(), child.ID())
+ }
+ opt.appendStepToCurrent(sel.ID(), sel.TP(), reason, action)
+}