From 29c74c7af7a4038e54de504a03a7003df6d7a63e Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 15 Dec 2021 18:07:04 +0800 Subject: [PATCH 1/3] support trace topN Signed-off-by: yisaer --- planner/core/plan.go | 2 +- planner/core/rule_topn_push_down.go | 102 +++++++++++++++++++--------- 2 files changed, 70 insertions(+), 34 deletions(-) diff --git a/planner/core/plan.go b/planner/core/plan.go index 3515f44e91750..bd9d19a7de4e8 100644 --- a/planner/core/plan.go +++ b/planner/core/plan.go @@ -259,7 +259,7 @@ type LogicalPlan interface { BuildKeyInfo(selfSchema *expression.Schema, childSchema []*expression.Schema) // pushDownTopN will push down the topN or limit operator during logical optimization. - pushDownTopN(topN *LogicalTopN) LogicalPlan + pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) LogicalPlan // recursiveDeriveStats derives statistic info between plans. recursiveDeriveStats(colGroups [][]*expression.Column) (*property.StatsInfo, error) diff --git a/planner/core/rule_topn_push_down.go b/planner/core/rule_topn_push_down.go index e6234bbc3f3dc..daecfbd9c4a54 100644 --- a/planner/core/rule_topn_push_down.go +++ b/planner/core/rule_topn_push_down.go @@ -15,7 +15,9 @@ package core import ( + "bytes" "context" + "fmt" "github.com/cznic/mathutil" "github.com/pingcap/tidb/expression" @@ -27,22 +29,22 @@ type pushDownTopNOptimizer struct { } func (s *pushDownTopNOptimizer) optimize(ctx context.Context, p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) { - return p.pushDownTopN(nil), nil + return p.pushDownTopN(nil, opt), nil } -func (s *baseLogicalPlan) pushDownTopN(topN *LogicalTopN) LogicalPlan { +func (s *baseLogicalPlan) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) LogicalPlan { p := s.self for i, child := range p.Children() { - p.Children()[i] = child.pushDownTopN(nil) + p.Children()[i] = child.pushDownTopN(nil, opt) } if topN != nil { - return topN.setChild(p) + return topN.setChild(p, opt) } return p } // setChild set p as topn's child. -func (lt *LogicalTopN) setChild(p LogicalPlan) LogicalPlan { +func (lt *LogicalTopN) setChild(p LogicalPlan, opt *logicalOptimizeOp) LogicalPlan { // Remove this TopN if its child is a TableDual. dual, isDual := p.(*LogicalTableDual) if isDual { @@ -62,37 +64,44 @@ func (lt *LogicalTopN) setChild(p LogicalPlan) LogicalPlan { limitHints: lt.limitHints, }.Init(lt.ctx, lt.blockOffset) limit.SetChildren(p) + appendTopNPushDownTraceStep(limit, p, opt) return limit } // Then lt must be topN. lt.SetChildren(p) + appendTopNPushDownTraceStep(lt, p, opt) return lt } -func (ls *LogicalSort) pushDownTopN(topN *LogicalTopN) LogicalPlan { +func (ls *LogicalSort) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) LogicalPlan { if topN == nil { - return ls.baseLogicalPlan.pushDownTopN(nil) + return ls.baseLogicalPlan.pushDownTopN(nil, opt) } else if topN.isLimit() { topN.ByItems = ls.ByItems - return ls.children[0].pushDownTopN(topN) + // sort pass byItems + opt.appendStepToCurrent(ls.ID(), ls.TP(), "", "") + return ls.children[0].pushDownTopN(topN, opt) } // If a TopN is pushed down, this sort is useless. - return ls.children[0].pushDownTopN(topN) + return ls.children[0].pushDownTopN(topN, opt) } -func (p *LogicalLimit) convertToTopN() *LogicalTopN { - return LogicalTopN{Offset: p.Offset, Count: p.Count, limitHints: p.limitHints}.Init(p.ctx, p.blockOffset) +func (p *LogicalLimit) convertToTopN(opt *logicalOptimizeOp) *LogicalTopN { + topn := LogicalTopN{Offset: p.Offset, Count: p.Count, limitHints: p.limitHints}.Init(p.ctx, p.blockOffset) + opt.appendStepToCurrent(topn.ID(), topn.TP(), "", fmt.Sprintf("%v[%v] converts into %v[%v]", + p.TP(), p.ID(), topn.TP(), topn.ID())) + return topn } -func (p *LogicalLimit) pushDownTopN(topN *LogicalTopN) LogicalPlan { - child := p.children[0].pushDownTopN(p.convertToTopN()) +func (p *LogicalLimit) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) LogicalPlan { + child := p.children[0].pushDownTopN(p.convertToTopN(opt), opt) if topN != nil { - return topN.setChild(child) + return topN.setChild(child, opt) } return child } -func (p *LogicalUnionAll) pushDownTopN(topN *LogicalTopN) LogicalPlan { +func (p *LogicalUnionAll) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) LogicalPlan { for i, child := range p.children { var newTopN *LogicalTopN if topN != nil { @@ -100,19 +109,23 @@ func (p *LogicalUnionAll) pushDownTopN(topN *LogicalTopN) LogicalPlan { for _, by := range topN.ByItems { newTopN.ByItems = append(newTopN.ByItems, &util.ByItems{Expr: by.Expr, Desc: by.Desc}) } + // newTopN to push down Union's child + opt.appendStepToCurrent(newTopN.ID(), newTopN.TP(), "", + fmt.Sprintf("%v[%v] is added and pushed down across %v[%v]", + newTopN.TP(), newTopN.ID(), p.TP(), p.ID())) } - p.children[i] = child.pushDownTopN(newTopN) + p.children[i] = child.pushDownTopN(newTopN, opt) } if topN != nil { - return topN.setChild(p) + return topN.setChild(p, opt) } return p } -func (p *LogicalProjection) pushDownTopN(topN *LogicalTopN) LogicalPlan { +func (p *LogicalProjection) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) LogicalPlan { for _, expr := range p.Exprs { if expression.HasAssignSetVarFunc(expr) { - return p.baseLogicalPlan.pushDownTopN(topN) + return p.baseLogicalPlan.pushDownTopN(topN, opt) } } if topN != nil { @@ -128,28 +141,28 @@ func (p *LogicalProjection) pushDownTopN(topN *LogicalTopN) LogicalPlan { } } } - p.children[0] = p.children[0].pushDownTopN(topN) + p.children[0] = p.children[0].pushDownTopN(topN, opt) return p } -func (p *LogicalLock) pushDownTopN(topN *LogicalTopN) LogicalPlan { +func (p *LogicalLock) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) LogicalPlan { if topN != nil { - p.children[0] = p.children[0].pushDownTopN(topN) + p.children[0] = p.children[0].pushDownTopN(topN, opt) } return p.self } // pushDownTopNToChild will push a topN to one child of join. The idx stands for join child index. 0 is for left child. -func (p *LogicalJoin) pushDownTopNToChild(topN *LogicalTopN, idx int) LogicalPlan { +func (p *LogicalJoin) pushDownTopNToChild(topN *LogicalTopN, idx int, opt *logicalOptimizeOp) LogicalPlan { if topN == nil { - return p.children[idx].pushDownTopN(nil) + return p.children[idx].pushDownTopN(nil, opt) } for _, by := range topN.ByItems { cols := expression.ExtractColumns(by.Expr) for _, col := range cols { if !p.children[idx].Schema().Contains(col) { - return p.children[idx].pushDownTopN(nil) + return p.children[idx].pushDownTopN(nil, opt) } } } @@ -162,24 +175,25 @@ func (p *LogicalJoin) pushDownTopNToChild(topN *LogicalTopN, idx int) LogicalPla for i := range topN.ByItems { newTopN.ByItems[i] = topN.ByItems[i].Clone() } - return p.children[idx].pushDownTopN(newTopN) + appendTopNPushDownJoinTraceStep(p, newTopN, idx, opt) + return p.children[idx].pushDownTopN(newTopN, opt) } -func (p *LogicalJoin) pushDownTopN(topN *LogicalTopN) LogicalPlan { +func (p *LogicalJoin) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) LogicalPlan { switch p.JoinType { case LeftOuterJoin, LeftOuterSemiJoin, AntiLeftOuterSemiJoin: - p.children[0] = p.pushDownTopNToChild(topN, 0) - p.children[1] = p.children[1].pushDownTopN(nil) + p.children[0] = p.pushDownTopNToChild(topN, 0, opt) + p.children[1] = p.children[1].pushDownTopN(nil, opt) case RightOuterJoin: - p.children[1] = p.pushDownTopNToChild(topN, 1) - p.children[0] = p.children[0].pushDownTopN(nil) + p.children[1] = p.pushDownTopNToChild(topN, 1, opt) + p.children[0] = p.children[0].pushDownTopN(nil, opt) default: - return p.baseLogicalPlan.pushDownTopN(topN) + return p.baseLogicalPlan.pushDownTopN(topN, opt) } // The LogicalJoin may be also a LogicalApply. So we must use self to set parents. if topN != nil { - return topN.setChild(p.self) + return topN.setChild(p.self, opt) } return p.self } @@ -187,3 +201,25 @@ func (p *LogicalJoin) pushDownTopN(topN *LogicalTopN) LogicalPlan { func (*pushDownTopNOptimizer) name() string { return "topn_push_down" } + +func appendTopNPushDownTraceStep(parent LogicalPlan, child LogicalPlan, opt *logicalOptimizeOp) { + action := fmt.Sprintf("%v[%v] is added as %v[%v]'s parent", parent.TP(), parent.ID(), child.TP(), child.ID()) + reason := fmt.Sprintf("%v is pushed down", parent.TP()) + opt.appendStepToCurrent(parent.ID(), parent.TP(), reason, action) +} + +func appendTopNPushDownJoinTraceStep(p *LogicalJoin, topN *LogicalTopN, idx int, opt *logicalOptimizeOp) { + action := func() string { + buffer := bytes.NewBufferString(fmt.Sprintf("%v[%v] is added and pushed into %v[%v]'s ", + topN.TP(), topN.ID(), p.TP(), p.ID())) + if idx == 0 { + buffer.WriteString("left ") + } else { + buffer.WriteString("right ") + } + buffer.WriteString("child") + return buffer.String() + }() + reason := fmt.Sprintf("%v[%v]'s joinType is %v", p.TP(), p.ID(), p.JoinType.String()) + opt.appendStepToCurrent(p.ID(), p.TP(), reason, action) +} From 6594b60d82e84d5619f093f108d5341178c80d72 Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 16 Dec 2021 16:39:04 +0800 Subject: [PATCH 2/3] add testcase Signed-off-by: yisaer --- planner/core/logical_plan_trace_test.go | 46 +++++++++++++++++++++++++ planner/core/rule_topn_push_down.go | 41 +++++++++++++++++++--- 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/planner/core/logical_plan_trace_test.go b/planner/core/logical_plan_trace_test.go index f0c6d5718eaae..ff93d94d67850 100644 --- a/planner/core/logical_plan_trace_test.go +++ b/planner/core/logical_plan_trace_test.go @@ -86,6 +86,52 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleName string assertRuleSteps []assertTraceStep }{ + { + sql: "select * from t as t1 left join t as t2 on t1.a = t2.a order by t1.a limit 10;", + flags: []uint64{flagPrunColumns, flagBuildKeyInfo, flagPushDownTopN}, + assertRuleName: "topn_push_down", + assertRuleSteps: []assertTraceStep{ + { + assertAction: "Limit[6] is converted into TopN[7]", + assertReason: "Limit can be converted into TopN", + }, + { + assertAction: "Sort[5] passes ByItems[test.t.a] to TopN[7]", + assertReason: "TopN[7] is Limit originally", + }, + { + assertAction: "TopN[8] is added and pushed into Join[3]'s left table", + assertReason: "Join[3]'s joinType is left outer join, and all ByItems[test.t.a] contained in left table", + }, + { + assertAction: "TopN[8] is added as DataSource[1]'s parent", + assertReason: "TopN is pushed down", + }, + { + assertAction: "TopN[7] is added as Join[3]'s parent", + assertReason: "TopN is pushed down", + }, + }, + }, + { + sql: "select * from t order by a limit 10", + flags: []uint64{flagPrunColumns, flagBuildKeyInfo, flagPushDownTopN}, + assertRuleName: "topn_push_down", + assertRuleSteps: []assertTraceStep{ + { + assertAction: "Limit[4] is converted into TopN[5]", + assertReason: "Limit can be converted into TopN", + }, + { + assertAction: "Sort[3] passes ByItems[test.t.a] to TopN[5]", + assertReason: "TopN[5] is Limit originally", + }, + { + assertAction: "TopN[5] is added as DataSource[1]'s parent", + assertReason: "TopN is pushed down", + }, + }, + }, { sql: "select * from (t t1, t t2, t t3,t t4) union all select * from (t t5, t t6, t t7,t t8)", flags: []uint64{flagBuildKeyInfo, flagPrunColumns, flagDecorrelate, flagPredicatePushDown, flagEliminateOuterJoin, flagJoinReOrder}, diff --git a/planner/core/rule_topn_push_down.go b/planner/core/rule_topn_push_down.go index daecfbd9c4a54..4a2a333eb7b45 100644 --- a/planner/core/rule_topn_push_down.go +++ b/planner/core/rule_topn_push_down.go @@ -78,8 +78,7 @@ func (ls *LogicalSort) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) L return ls.baseLogicalPlan.pushDownTopN(nil, opt) } else if topN.isLimit() { topN.ByItems = ls.ByItems - // sort pass byItems - opt.appendStepToCurrent(ls.ID(), ls.TP(), "", "") + appendSortPassByItemsTraceStep(ls, topN, opt) return ls.children[0].pushDownTopN(topN, opt) } // If a TopN is pushed down, this sort is useless. @@ -88,7 +87,7 @@ func (ls *LogicalSort) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) L func (p *LogicalLimit) convertToTopN(opt *logicalOptimizeOp) *LogicalTopN { topn := LogicalTopN{Offset: p.Offset, Count: p.Count, limitHints: p.limitHints}.Init(p.ctx, p.blockOffset) - opt.appendStepToCurrent(topn.ID(), topn.TP(), "", fmt.Sprintf("%v[%v] converts into %v[%v]", + opt.appendStepToCurrent(topn.ID(), topn.TP(), "Limit can be converted into TopN", fmt.Sprintf("%v[%v] is converted into %v[%v]", p.TP(), p.ID(), topn.TP(), topn.ID())) return topn } @@ -217,9 +216,41 @@ func appendTopNPushDownJoinTraceStep(p *LogicalJoin, topN *LogicalTopN, idx int, } else { buffer.WriteString("right ") } - buffer.WriteString("child") + buffer.WriteString("table") + return buffer.String() + }() + reason := func() string { + buffer := bytes.NewBufferString(fmt.Sprintf("%v[%v]'s joinType is %v, and all ByItems[", p.TP(), p.ID(), p.JoinType.String())) + for i, item := range topN.ByItems { + if i > 0 { + buffer.WriteString(",") + } + buffer.WriteString(item.String()) + } + buffer.WriteString("] contained in ") + if idx == 0 { + buffer.WriteString("left ") + } else { + buffer.WriteString("right ") + } + buffer.WriteString("table") return buffer.String() }() - reason := fmt.Sprintf("%v[%v]'s joinType is %v", p.TP(), p.ID(), p.JoinType.String()) opt.appendStepToCurrent(p.ID(), p.TP(), reason, action) } + +func appendSortPassByItemsTraceStep(sort *LogicalSort, topN *LogicalTopN, opt *logicalOptimizeOp) { + action := func() string { + buffer := bytes.NewBufferString(fmt.Sprintf("%v[%v] passes ByItems[", sort.TP(), sort.ID())) + for i, item := range sort.ByItems { + if i > 0 { + buffer.WriteString(",") + } + buffer.WriteString(item.String()) + } + buffer.WriteString(fmt.Sprintf("] to %v[%v]", topN.TP(), topN.ID())) + return buffer.String() + }() + reason := fmt.Sprintf("%v[%v] is Limit originally", topN.TP(), topN.ID()) + opt.appendStepToCurrent(sort.ID(), sort.TP(), reason, action) +} From 2708108fcd885872b19b33573d33aba3ead7fa1a Mon Sep 17 00:00:00 2001 From: yisaer Date: Mon, 20 Dec 2021 14:04:59 +0800 Subject: [PATCH 3/3] add testcase Signed-off-by: yisaer --- planner/core/logical_plan_trace_test.go | 26 ++++++++++++------------- planner/core/rule_topn_push_down.go | 16 +++++++-------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/planner/core/logical_plan_trace_test.go b/planner/core/logical_plan_trace_test.go index 700b60b026aad..0d4577bc1f107 100644 --- a/planner/core/logical_plan_trace_test.go +++ b/planner/core/logical_plan_trace_test.go @@ -92,23 +92,23 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleName: "topn_push_down", assertRuleSteps: []assertTraceStep{ { - assertAction: "Limit[6] is converted into TopN[7]", - assertReason: "Limit can be converted into TopN", + assertAction: "Limit_6 is converted into TopN_7", + assertReason: "", }, { - assertAction: "Sort[5] passes ByItems[test.t.a] to TopN[7]", - assertReason: "TopN[7] is Limit originally", + assertAction: "Sort_5 passes ByItems[test.t.a] to TopN_7", + assertReason: "TopN_7 is Limit originally", }, { - assertAction: "TopN[8] is added and pushed into Join[3]'s left table", - assertReason: "Join[3]'s joinType is left outer join, and all ByItems[test.t.a] contained in left table", + assertAction: "TopN_8 is added and pushed into Join_3's left table", + assertReason: "Join_3's joinType is left outer join, and all ByItems[test.t.a] contained in left table", }, { - assertAction: "TopN[8] is added as DataSource[1]'s parent", + assertAction: "TopN_8 is added as DataSource_1's parent", assertReason: "TopN is pushed down", }, { - assertAction: "TopN[7] is added as Join[3]'s parent", + assertAction: "TopN_7 is added as Join_3's parent", assertReason: "TopN is pushed down", }, }, @@ -119,15 +119,15 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleName: "topn_push_down", assertRuleSteps: []assertTraceStep{ { - assertAction: "Limit[4] is converted into TopN[5]", - assertReason: "Limit can be converted into TopN", + assertAction: "Limit_4 is converted into TopN_5", + assertReason: "", }, { - assertAction: "Sort[3] passes ByItems[test.t.a] to TopN[5]", - assertReason: "TopN[5] is Limit originally", + assertAction: "Sort_3 passes ByItems[test.t.a] to TopN_5", + assertReason: "TopN_5 is Limit originally", }, { - assertAction: "TopN[5] is added as DataSource[1]'s parent", + assertAction: "TopN_5 is added as DataSource_1's parent", assertReason: "TopN is pushed down", }, }, diff --git a/planner/core/rule_topn_push_down.go b/planner/core/rule_topn_push_down.go index 4a2a333eb7b45..73ed5749bc57c 100644 --- a/planner/core/rule_topn_push_down.go +++ b/planner/core/rule_topn_push_down.go @@ -87,7 +87,7 @@ func (ls *LogicalSort) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp) L func (p *LogicalLimit) convertToTopN(opt *logicalOptimizeOp) *LogicalTopN { topn := LogicalTopN{Offset: p.Offset, Count: p.Count, limitHints: p.limitHints}.Init(p.ctx, p.blockOffset) - opt.appendStepToCurrent(topn.ID(), topn.TP(), "Limit can be converted into TopN", fmt.Sprintf("%v[%v] is converted into %v[%v]", + opt.appendStepToCurrent(topn.ID(), topn.TP(), "", fmt.Sprintf("%v_%v is converted into %v_%v", p.TP(), p.ID(), topn.TP(), topn.ID())) return topn } @@ -110,7 +110,7 @@ func (p *LogicalUnionAll) pushDownTopN(topN *LogicalTopN, opt *logicalOptimizeOp } // newTopN to push down Union's child opt.appendStepToCurrent(newTopN.ID(), newTopN.TP(), "", - fmt.Sprintf("%v[%v] is added and pushed down across %v[%v]", + fmt.Sprintf("%v_%v is added and pushed down across %v_%v", newTopN.TP(), newTopN.ID(), p.TP(), p.ID())) } p.children[i] = child.pushDownTopN(newTopN, opt) @@ -202,14 +202,14 @@ func (*pushDownTopNOptimizer) name() string { } func appendTopNPushDownTraceStep(parent LogicalPlan, child LogicalPlan, opt *logicalOptimizeOp) { - action := fmt.Sprintf("%v[%v] is added as %v[%v]'s parent", parent.TP(), parent.ID(), child.TP(), child.ID()) + action := fmt.Sprintf("%v_%v is added as %v_%v's parent", parent.TP(), parent.ID(), child.TP(), child.ID()) reason := fmt.Sprintf("%v is pushed down", parent.TP()) opt.appendStepToCurrent(parent.ID(), parent.TP(), reason, action) } func appendTopNPushDownJoinTraceStep(p *LogicalJoin, topN *LogicalTopN, idx int, opt *logicalOptimizeOp) { action := func() string { - buffer := bytes.NewBufferString(fmt.Sprintf("%v[%v] is added and pushed into %v[%v]'s ", + buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v is added and pushed into %v_%v's ", topN.TP(), topN.ID(), p.TP(), p.ID())) if idx == 0 { buffer.WriteString("left ") @@ -220,7 +220,7 @@ func appendTopNPushDownJoinTraceStep(p *LogicalJoin, topN *LogicalTopN, idx int, return buffer.String() }() reason := func() string { - buffer := bytes.NewBufferString(fmt.Sprintf("%v[%v]'s joinType is %v, and all ByItems[", p.TP(), p.ID(), p.JoinType.String())) + buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v's joinType is %v, and all ByItems[", p.TP(), p.ID(), p.JoinType.String())) for i, item := range topN.ByItems { if i > 0 { buffer.WriteString(",") @@ -241,16 +241,16 @@ func appendTopNPushDownJoinTraceStep(p *LogicalJoin, topN *LogicalTopN, idx int, func appendSortPassByItemsTraceStep(sort *LogicalSort, topN *LogicalTopN, opt *logicalOptimizeOp) { action := func() string { - buffer := bytes.NewBufferString(fmt.Sprintf("%v[%v] passes ByItems[", sort.TP(), sort.ID())) + buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v passes ByItems[", sort.TP(), sort.ID())) for i, item := range sort.ByItems { if i > 0 { buffer.WriteString(",") } buffer.WriteString(item.String()) } - buffer.WriteString(fmt.Sprintf("] to %v[%v]", topN.TP(), topN.ID())) + buffer.WriteString(fmt.Sprintf("] to %v_%v", topN.TP(), topN.ID())) return buffer.String() }() - reason := fmt.Sprintf("%v[%v] is Limit originally", topN.TP(), topN.ID()) + reason := fmt.Sprintf("%v_%v is Limit originally", topN.TP(), topN.ID()) opt.appendStepToCurrent(sort.ID(), sort.TP(), reason, action) }