From ef2104eeabda20ccc29a6615d705f3193eca06cd Mon Sep 17 00:00:00 2001 From: yisaer Date: Tue, 7 Dec 2021 19:05:06 +0800 Subject: [PATCH 1/9] trace partition Signed-off-by: yisaer --- planner/core/rule_partition_processor.go | 61 ++++++++++++++++++++---- 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index bb57b0fac33da..9323401e0bb59 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -15,6 +15,7 @@ package core import ( + "bytes" "context" "fmt" gomath "math" @@ -58,18 +59,18 @@ const FullRange = -1 type partitionProcessor struct{} func (s *partitionProcessor) optimize(ctx context.Context, lp LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) { - p, err := s.rewriteDataSource(lp) + p, err := s.rewriteDataSource(lp, opt) return p, err } -func (s *partitionProcessor) rewriteDataSource(lp LogicalPlan) (LogicalPlan, error) { +func (s *partitionProcessor) rewriteDataSource(lp LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) { // Assert there will not be sel -> sel in the ast. switch p := lp.(type) { case *DataSource: - return s.prune(p) + return s.prune(p, opt) case *LogicalUnionScan: ds := p.Children()[0] - ds, err := s.prune(ds.(*DataSource)) + ds, err := s.prune(ds.(*DataSource), opt) if err != nil { return nil, err } @@ -94,7 +95,7 @@ func (s *partitionProcessor) rewriteDataSource(lp LogicalPlan) (LogicalPlan, err default: children := lp.Children() for i, child := range children { - newChild, err := s.rewriteDataSource(child) + newChild, err := s.rewriteDataSource(child, opt) if err != nil { return nil, err } @@ -617,7 +618,7 @@ func (s *partitionProcessor) pruneListPartition(ctx sessionctx.Context, tbl tabl return used, nil } -func (s *partitionProcessor) prune(ds *DataSource) (LogicalPlan, error) { +func (s *partitionProcessor) prune(ds *DataSource, opt *logicalOptimizeOp) (LogicalPlan, error) { pi := ds.tableInfo.GetPartitionInfo() if pi == nil { return ds, nil @@ -631,7 +632,7 @@ func (s *partitionProcessor) prune(ds *DataSource) (LogicalPlan, error) { // Try to locate partition directly for hash partition. switch pi.Type { case model.PartitionTypeRange: - return s.processRangePartition(ds, pi) + return s.processRangePartition(ds, pi, opt) case model.PartitionTypeHash: return s.processHashPartition(ds, pi) case model.PartitionTypeList: @@ -872,7 +873,7 @@ func (s *partitionProcessor) pruneRangePartition(ctx sessionctx.Context, pi *mod return result, newConds, nil } -func (s *partitionProcessor) processRangePartition(ds *DataSource, pi *model.PartitionInfo) (LogicalPlan, error) { +func (s *partitionProcessor) processRangePartition(ds *DataSource, pi *model.PartitionInfo, opt *logicalOptimizeOp) (LogicalPlan, error) { used, prunedConds, err := s.pruneRangePartition(ds.ctx, pi, ds.table.(table.PartitionedTable), ds.allConds, ds.TblCols, ds.names, &ds.pushedDownConds) if err != nil { return nil, err @@ -1391,9 +1392,11 @@ func (s *partitionProcessor) checkHintsApplicable(ds *DataSource, partitionSet s appendWarnForUnknownPartitions(ds.ctx, HintReadFromStorage, unknownPartitions) } -func (s *partitionProcessor) makeUnionAllChildren(ds *DataSource, pi *model.PartitionInfo, or partitionRangeOR) (LogicalPlan, error) { +func (s *partitionProcessor) makeUnionAllChildren(ds *DataSource, pi *model.PartitionInfo, or partitionRangeOR, opt *logicalOptimizeOp) (LogicalPlan, error) { + children := make([]LogicalPlan, 0, len(pi.Definitions)) partitionNameSet := make(set.StringSet) + usedDefinations := make(map[int64]model.PartitionDefinition, 0) for _, r := range or { for i := r.start; i < r.end; i++ { // This is for `table partition (p0,p1)` syntax, only union the specified partition if has specified partitions. @@ -1424,6 +1427,7 @@ func (s *partitionProcessor) makeUnionAllChildren(ds *DataSource, pi *model.Part return nil, err } children = append(children, &newDataSource) + usedDefinations[pi.Definitions[i].ID] = pi.Definitions[i] } } s.checkHintsApplicable(ds, partitionNameSet) @@ -1432,15 +1436,18 @@ func (s *partitionProcessor) makeUnionAllChildren(ds *DataSource, pi *model.Part // No result after table pruning. tableDual := LogicalTableDual{RowCount: 0}.Init(ds.SCtx(), ds.blockOffset) tableDual.schema = ds.Schema() + appendMakeUnionAllChildrenTranceStep(ds, tableDual, children, opt) return tableDual, nil } if len(children) == 1 { // No need for the union all. + appendMakeUnionAllChildrenTranceStep(ds, children[0], children, opt) return children[0], nil } unionAll := LogicalPartitionUnionAll{}.Init(ds.SCtx(), ds.blockOffset) unionAll.SetChildren(children...) unionAll.SetSchema(ds.schema.Clone()) + appendMakeUnionAllChildrenTranceStep(ds, unionAll, children, opt) return unionAll, nil } @@ -1568,3 +1575,39 @@ func (p *rangeColumnsPruner) pruneUseBinarySearch(sctx sessionctx.Context, op st } return start, end } + +func appendMakeUnionAllChildrenTranceStep(ds *DataSource, used []model.PartitionDefinition, plan LogicalPlan, children []LogicalPlan, opt *logicalOptimizeOp) { + action := "" + reason := "" + if len(children) == 0 { + action = fmt.Sprintf("Datasource[%v] becomes %s[%v]", ds.ID(), plan.TP(), plan.ID()) + reason = fmt.Sprintf("Datasource[%v] has no available partition table after partition pruning", ds.ID()) + } else if len(children) == 1 { + action = fmt.Sprintf("Datasource[%v] becomes %s[%v]", ds.ID(), plan.TP(), plan.ID()) + reason = fmt.Sprintf("Datasource[%v] has one available partiton table[%s] after partition pruning", ds.ID(), used[0].Name) + } else { + action = func() string { + buffer := bytes.NewBufferString(fmt.Sprintf("Datasource[%v] becomes %s[%v] with children[", ds.ID(), plan.TP(), plan.ID())) + for i, child := range children { + if i > 0 { + buffer.WriteString(",") + } + buffer.WriteString(fmt.Sprintf("%s[%v]", child.TP(), child.ID())) + } + buffer.WriteString("]") + return buffer.String() + }() + reason = func() string { + buffer := bytes.NewBufferString(fmt.Sprintf("Datasource[%v] have multi available partition tables[", ds.ID())) + for i, u := range used { + if i > 0 { + buffer.WriteString(",") + } + buffer.WriteString(fmt.Sprintf("%s", u.Name)) + } + buffer.WriteString("]") + return buffer.String() + }() + } + opt.appendStepToCurrent(ds.ID(), ds.TP(), reason, action) +} From d48281afd4e8bde736ee466b2cfe7a900e29f030 Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 8 Dec 2021 15:39:05 +0800 Subject: [PATCH 2/9] support trace Signed-off-by: yisaer --- planner/core/rule_partition_processor.go | 32 +++++++++++++----------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index 9323401e0bb59..00ab67c6a29f7 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -325,7 +325,7 @@ func (s *partitionProcessor) reconstructTableColNames(ds *DataSource) ([]*types. return names, nil } -func (s *partitionProcessor) processHashPartition(ds *DataSource, pi *model.PartitionInfo) (LogicalPlan, error) { +func (s *partitionProcessor) processHashPartition(ds *DataSource, pi *model.PartitionInfo, opt *logicalOptimizeOp) (LogicalPlan, error) { names, err := s.reconstructTableColNames(ds) if err != nil { return nil, err @@ -335,7 +335,7 @@ func (s *partitionProcessor) processHashPartition(ds *DataSource, pi *model.Part return nil, err } if used != nil { - return s.makeUnionAllChildren(ds, pi, convertToRangeOr(used, pi)) + return s.makeUnionAllChildren(ds, pi, convertToRangeOr(used, pi), opt) } tableDual := LogicalTableDual{RowCount: 0}.Init(ds.SCtx(), ds.blockOffset) tableDual.schema = ds.Schema() @@ -634,13 +634,13 @@ func (s *partitionProcessor) prune(ds *DataSource, opt *logicalOptimizeOp) (Logi case model.PartitionTypeRange: return s.processRangePartition(ds, pi, opt) case model.PartitionTypeHash: - return s.processHashPartition(ds, pi) + return s.processHashPartition(ds, pi, opt) case model.PartitionTypeList: - return s.processListPartition(ds, pi) + return s.processListPartition(ds, pi, opt) } // We haven't implement partition by list and so on. - return s.makeUnionAllChildren(ds, pi, fullRange(len(pi.Definitions))) + return s.makeUnionAllChildren(ds, pi, fullRange(len(pi.Definitions)), opt) } // findByName checks whether object name exists in list. @@ -881,16 +881,16 @@ func (s *partitionProcessor) processRangePartition(ds *DataSource, pi *model.Par if prunedConds != nil { ds.pushedDownConds = prunedConds } - return s.makeUnionAllChildren(ds, pi, used) + return s.makeUnionAllChildren(ds, pi, used, opt) } -func (s *partitionProcessor) processListPartition(ds *DataSource, pi *model.PartitionInfo) (LogicalPlan, error) { +func (s *partitionProcessor) processListPartition(ds *DataSource, pi *model.PartitionInfo, opt *logicalOptimizeOp) (LogicalPlan, error) { used, err := s.pruneListPartition(ds.SCtx(), ds.table, ds.partitionNames, ds.allConds) if err != nil { return nil, err } if used != nil { - return s.makeUnionAllChildren(ds, pi, convertToRangeOr(used, pi)) + return s.makeUnionAllChildren(ds, pi, convertToRangeOr(used, pi), opt) } tableDual := LogicalTableDual{RowCount: 0}.Init(ds.SCtx(), ds.blockOffset) tableDual.schema = ds.Schema() @@ -1396,7 +1396,7 @@ func (s *partitionProcessor) makeUnionAllChildren(ds *DataSource, pi *model.Part children := make([]LogicalPlan, 0, len(pi.Definitions)) partitionNameSet := make(set.StringSet) - usedDefinations := make(map[int64]model.PartitionDefinition, 0) + usedDefinition := make(map[int64]model.PartitionDefinition, 0) for _, r := range or { for i := r.start; i < r.end; i++ { // This is for `table partition (p0,p1)` syntax, only union the specified partition if has specified partitions. @@ -1427,7 +1427,7 @@ func (s *partitionProcessor) makeUnionAllChildren(ds *DataSource, pi *model.Part return nil, err } children = append(children, &newDataSource) - usedDefinations[pi.Definitions[i].ID] = pi.Definitions[i] + usedDefinition[pi.Definitions[i].ID] = pi.Definitions[i] } } s.checkHintsApplicable(ds, partitionNameSet) @@ -1436,18 +1436,18 @@ func (s *partitionProcessor) makeUnionAllChildren(ds *DataSource, pi *model.Part // No result after table pruning. tableDual := LogicalTableDual{RowCount: 0}.Init(ds.SCtx(), ds.blockOffset) tableDual.schema = ds.Schema() - appendMakeUnionAllChildrenTranceStep(ds, tableDual, children, opt) + appendMakeUnionAllChildrenTranceStep(ds, usedDefinition, tableDual, children, opt) return tableDual, nil } if len(children) == 1 { // No need for the union all. - appendMakeUnionAllChildrenTranceStep(ds, children[0], children, opt) + appendMakeUnionAllChildrenTranceStep(ds, usedDefinition, children[0], children, opt) return children[0], nil } unionAll := LogicalPartitionUnionAll{}.Init(ds.SCtx(), ds.blockOffset) unionAll.SetChildren(children...) unionAll.SetSchema(ds.schema.Clone()) - appendMakeUnionAllChildrenTranceStep(ds, unionAll, children, opt) + appendMakeUnionAllChildrenTranceStep(ds, usedDefinition, unionAll, children, opt) return unionAll, nil } @@ -1576,9 +1576,13 @@ func (p *rangeColumnsPruner) pruneUseBinarySearch(sctx sessionctx.Context, op st return start, end } -func appendMakeUnionAllChildrenTranceStep(ds *DataSource, used []model.PartitionDefinition, plan LogicalPlan, children []LogicalPlan, opt *logicalOptimizeOp) { +func appendMakeUnionAllChildrenTranceStep(ds *DataSource, usedMap map[int64]model.PartitionDefinition, plan LogicalPlan, children []LogicalPlan, opt *logicalOptimizeOp) { action := "" reason := "" + var used []model.PartitionDefinition + for _, def := range usedMap { + used = append(used, def) + } if len(children) == 0 { action = fmt.Sprintf("Datasource[%v] becomes %s[%v]", ds.ID(), plan.TP(), plan.ID()) reason = fmt.Sprintf("Datasource[%v] has no available partition table after partition pruning", ds.ID()) From 78b2bc834a8df2c960deb4d5af609eef06bbcb23 Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 8 Dec 2021 17:18:17 +0800 Subject: [PATCH 3/9] add testcase Signed-off-by: yisaer --- planner/core/logical_plan_test.go | 2 +- planner/core/logical_plan_trace_test.go | 33 ++++++++++++++++++++++ planner/core/mock.go | 36 ++++++++++++++++++++++++ planner/core/rule_partition_processor.go | 2 +- 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index a0bb8a8070726..5b65ffbb8ad79 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -59,7 +59,7 @@ type testPlanSuite struct { } func (s *testPlanSuite) SetUpSuite(c *C) { - s.is = infoschema.MockInfoSchema([]*model.TableInfo{MockSignedTable(), MockUnsignedTable(), MockView(), MockNoPKTable()}) + s.is = infoschema.MockInfoSchema([]*model.TableInfo{MockSignedTable(), MockUnsignedTable(), MockView(), MockNoPKTable(), MockPartitionTable()}) s.ctx = MockContext() domain.GetDomain(s.ctx).MockInfoCacheAndLoadInfoSchema(s.is) s.ctx.GetSessionVars().EnableWindowFunction = true diff --git a/planner/core/logical_plan_trace_test.go b/planner/core/logical_plan_trace_test.go index a16f0111677d0..7215a4f6647b4 100644 --- a/planner/core/logical_plan_trace_test.go +++ b/planner/core/logical_plan_trace_test.go @@ -86,6 +86,39 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleName string assertRuleSteps []assertTraceStep }{ + { + sql: "select * from pt1 where ptn > 100;", + flags: []uint64{flagPartitionProcessor, flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns}, + assertRuleName: "partition_processor", + assertRuleSteps: []assertTraceStep{ + { + assertReason: "Datasource[1] has no available partition table after partition pruning", + assertAction: "Datasource[1] becomes TableDual[5]", + }, + }, + }, + { + sql: "select * from pt1 where ptn in (10,20);", + flags: []uint64{flagPartitionProcessor, flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns}, + assertRuleName: "partition_processor", + assertRuleSteps: []assertTraceStep{ + { + assertReason: "Datasource[1] have multi available partition tables[p1,p2] after partition pruning", + assertAction: "Datasource[1] becomes PartitionUnion[7] with children[TableScan[1],TableScan[1]]", + }, + }, + }, + { + sql: "select * from pt1 where ptn < 4;", + flags: []uint64{flagPartitionProcessor, flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns}, + assertRuleName: "partition_processor", + assertRuleSteps: []assertTraceStep{ + { + assertReason: "Datasource[1] has one available partiton table[p1] after partition pruning", + assertAction: "Datasource[1] becomes TableScan[1]", + }, + }, + }, { sql: "select min(distinct a) from t group by a", flags: []uint64{flagBuildKeyInfo, flagEliminateAgg}, diff --git a/planner/core/mock.go b/planner/core/mock.go index 57375118dfd13..0af77bd39b029 100644 --- a/planner/core/mock.go +++ b/planner/core/mock.go @@ -432,3 +432,39 @@ func MockPartitionInfoSchema(definitions []model.PartitionDefinition) infoschema is := infoschema.MockInfoSchema([]*model.TableInfo{tableInfo}) return is } + +func MockPartitionTable() *model.TableInfo { + definitions := []model.PartitionDefinition{ + { + ID: 41, + Name: model.NewCIStr("p1"), + LessThan: []string{"16"}, + }, + { + ID: 42, + Name: model.NewCIStr("p2"), + LessThan: []string{"32"}, + }, + } + tableInfo := MockSignedTable() + tableInfo.Name = model.NewCIStr("pt1") + cols := make([]*model.ColumnInfo, 0, len(tableInfo.Columns)) + cols = append(cols, tableInfo.Columns...) + last := tableInfo.Columns[len(tableInfo.Columns)-1] + cols = append(cols, &model.ColumnInfo{ + State: model.StatePublic, + Offset: last.Offset + 1, + Name: model.NewCIStr("ptn"), + FieldType: newLongType(), + ID: last.ID + 1, + }) + partition := &model.PartitionInfo{ + Type: model.PartitionTypeRange, + Expr: "ptn", + Enable: true, + Definitions: definitions, + } + tableInfo.Columns = cols + tableInfo.Partition = partition + return tableInfo +} diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index 00ab67c6a29f7..9660782a5d0a6 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -1609,7 +1609,7 @@ func appendMakeUnionAllChildrenTranceStep(ds *DataSource, usedMap map[int64]mode } buffer.WriteString(fmt.Sprintf("%s", u.Name)) } - buffer.WriteString("]") + buffer.WriteString("] after partition pruning") return buffer.String() }() } From df3965a2ed23b748f8402f1f4b8981ea1e475171 Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 9 Dec 2021 12:51:28 +0800 Subject: [PATCH 4/9] add comment Signed-off-by: yisaer --- planner/core/mock.go | 1 + 1 file changed, 1 insertion(+) diff --git a/planner/core/mock.go b/planner/core/mock.go index 0af77bd39b029..e4f9afa830c15 100644 --- a/planner/core/mock.go +++ b/planner/core/mock.go @@ -433,6 +433,7 @@ func MockPartitionInfoSchema(definitions []model.PartitionDefinition) infoschema return is } +// MockPartitionTable mocks a partition table for test func MockPartitionTable() *model.TableInfo { definitions := []model.PartitionDefinition{ { From 0668001194879776aaaf4e0a4ed837ee0685611e Mon Sep 17 00:00:00 2001 From: yisaer Date: Mon, 13 Dec 2021 12:39:37 +0800 Subject: [PATCH 5/9] address the comment Signed-off-by: yisaer --- ddl/db_partition_test.go | 12 ++++ planner/core/logical_plan_test.go | 3 +- planner/core/logical_plan_trace_test.go | 44 +++++++++++++ planner/core/mock.go | 84 ++++++++++++++++++++++++- 4 files changed, 140 insertions(+), 3 deletions(-) diff --git a/ddl/db_partition_test.go b/ddl/db_partition_test.go index 647c421ab6b0d..cb012e8b9668d 100644 --- a/ddl/db_partition_test.go +++ b/ddl/db_partition_test.go @@ -1304,6 +1304,18 @@ func (s *testIntegrationSuite3) TestCreateTableWithKeyPartition(c *C) { tk.MustExec(`create table tm2 (a char(5), unique key(a(5))) partition by key() partitions 5;`) } +func (s *testIntegrationSuite5) TestTry(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test;") + tk.MustExec("create table t (x int) partition by hash(x) partitions 4;") + ctx := tk.Se.(sessionctx.Context) + is := domain.GetDomain(ctx).InfoSchema() + tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + c.Assert(err, IsNil) + p := tbl.Meta().Partition + fmt.Println(p) +} + func (s *testIntegrationSuite5) TestAlterTableAddPartition(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test;") diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index 5b65ffbb8ad79..e381da64fcdb6 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -59,7 +59,8 @@ type testPlanSuite struct { } func (s *testPlanSuite) SetUpSuite(c *C) { - s.is = infoschema.MockInfoSchema([]*model.TableInfo{MockSignedTable(), MockUnsignedTable(), MockView(), MockNoPKTable(), MockPartitionTable()}) + s.is = infoschema.MockInfoSchema([]*model.TableInfo{MockSignedTable(), MockUnsignedTable(), MockView(), MockNoPKTable(), + MockRangePartitionTable(), MockHashPartitionTable(), MockListPartitionTable()}) s.ctx = MockContext() domain.GetDomain(s.ctx).MockInfoCacheAndLoadInfoSchema(s.is) s.ctx.GetSessionVars().EnableWindowFunction = true diff --git a/planner/core/logical_plan_trace_test.go b/planner/core/logical_plan_trace_test.go index 7215a4f6647b4..6918105a03310 100644 --- a/planner/core/logical_plan_trace_test.go +++ b/planner/core/logical_plan_trace_test.go @@ -86,6 +86,50 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleName string assertRuleSteps []assertTraceStep }{ + { + sql: "select * from pt3 where ptn > 3;", + flags: []uint64{flagPartitionProcessor, flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns}, + assertRuleName: "partition_processor", + assertRuleSteps: []assertTraceStep{ + { + assertReason: "Datasource[1] have multi available partition tables[p1,p2] after partition pruning", + assertAction: "Datasource[1] becomes PartitionUnion[6] with children[TableScan[1],TableScan[1]]", + }, + }, + }, + { + sql: "select * from pt3 where ptn = 1;", + flags: []uint64{flagPartitionProcessor, flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns}, + assertRuleName: "partition_processor", + assertRuleSteps: []assertTraceStep{ + { + assertReason: "Datasource[1] has one available partiton table[p1] after partition pruning", + assertAction: "Datasource[1] becomes TableScan[1]", + }, + }, + }, + { + sql: "select * from pt2 where ptn in (1,2,3);", + flags: []uint64{flagPartitionProcessor, flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns}, + assertRuleName: "partition_processor", + assertRuleSteps: []assertTraceStep{ + { + assertReason: "Datasource[1] have multi available partition tables[p1,p2] after partition pruning", + assertAction: "Datasource[1] becomes PartitionUnion[7] with children[TableScan[1],TableScan[1]]", + }, + }, + }, + { + sql: "select * from pt2 where ptn = 1;", + flags: []uint64{flagPartitionProcessor, flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns}, + assertRuleName: "partition_processor", + assertRuleSteps: []assertTraceStep{ + { + assertReason: "Datasource[1] has one available partiton table[p2] after partition pruning", + assertAction: "Datasource[1] becomes TableScan[1]", + }, + }, + }, { sql: "select * from pt1 where ptn > 100;", flags: []uint64{flagPartitionProcessor, flagPredicatePushDown, flagBuildKeyInfo, flagPrunColumns}, diff --git a/planner/core/mock.go b/planner/core/mock.go index e4f9afa830c15..4419c6f1c8b7e 100644 --- a/planner/core/mock.go +++ b/planner/core/mock.go @@ -433,8 +433,8 @@ func MockPartitionInfoSchema(definitions []model.PartitionDefinition) infoschema return is } -// MockPartitionTable mocks a partition table for test -func MockPartitionTable() *model.TableInfo { +// MockRangePartitionTable mocks a range partition table for test +func MockRangePartitionTable() *model.TableInfo { definitions := []model.PartitionDefinition{ { ID: 41, @@ -469,3 +469,83 @@ func MockPartitionTable() *model.TableInfo { tableInfo.Partition = partition return tableInfo } + +func MockHashPartitionTable() *model.TableInfo { + definitions := []model.PartitionDefinition{ + { + ID: 51, + Name: model.NewCIStr("p1"), + }, + { + ID: 52, + Name: model.NewCIStr("p2"), + }, + } + tableInfo := MockSignedTable() + tableInfo.Name = model.NewCIStr("pt2") + cols := make([]*model.ColumnInfo, 0, len(tableInfo.Columns)) + cols = append(cols, tableInfo.Columns...) + last := tableInfo.Columns[len(tableInfo.Columns)-1] + cols = append(cols, &model.ColumnInfo{ + State: model.StatePublic, + Offset: last.Offset + 1, + Name: model.NewCIStr("ptn"), + FieldType: newLongType(), + ID: last.ID + 1, + }) + partition := &model.PartitionInfo{ + Type: model.PartitionTypeHash, + Expr: "ptn", + Enable: true, + Definitions: definitions, + Num: 2, + } + tableInfo.Columns = cols + tableInfo.Partition = partition + return tableInfo +} + +func MockListPartitionTable() *model.TableInfo { + definitions := []model.PartitionDefinition{ + { + ID: 61, + Name: model.NewCIStr("p1"), + InValues: [][]string{ + { + "1", + }, + }, + }, + { + ID: 62, + Name: model.NewCIStr("p2"), + InValues: [][]string{ + { + "2", + }, + }, + }, + } + tableInfo := MockSignedTable() + tableInfo.Name = model.NewCIStr("pt3") + cols := make([]*model.ColumnInfo, 0, len(tableInfo.Columns)) + cols = append(cols, tableInfo.Columns...) + last := tableInfo.Columns[len(tableInfo.Columns)-1] + cols = append(cols, &model.ColumnInfo{ + State: model.StatePublic, + Offset: last.Offset + 1, + Name: model.NewCIStr("ptn"), + FieldType: newLongType(), + ID: last.ID + 1, + }) + partition := &model.PartitionInfo{ + Type: model.PartitionTypeList, + Expr: "ptn", + Enable: true, + Definitions: definitions, + Num: 2, + } + tableInfo.Columns = cols + tableInfo.Partition = partition + return tableInfo +} From f286923b77e3f5126ed7b7521f9917d930a144d5 Mon Sep 17 00:00:00 2001 From: yisaer Date: Mon, 13 Dec 2021 18:24:51 +0800 Subject: [PATCH 6/9] address the comment Signed-off-by: yisaer address the comment Signed-off-by: yisaer address the comment Signed-off-by: yisaer address the comment Signed-off-by: yisaer --- ddl/db_partition_test.go | 12 ------------ planner/core/mock.go | 2 ++ planner/core/rule_partition_processor.go | 4 ++-- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/ddl/db_partition_test.go b/ddl/db_partition_test.go index cb012e8b9668d..647c421ab6b0d 100644 --- a/ddl/db_partition_test.go +++ b/ddl/db_partition_test.go @@ -1304,18 +1304,6 @@ func (s *testIntegrationSuite3) TestCreateTableWithKeyPartition(c *C) { tk.MustExec(`create table tm2 (a char(5), unique key(a(5))) partition by key() partitions 5;`) } -func (s *testIntegrationSuite5) TestTry(c *C) { - tk := testkit.NewTestKit(c, s.store) - tk.MustExec("use test;") - tk.MustExec("create table t (x int) partition by hash(x) partitions 4;") - ctx := tk.Se.(sessionctx.Context) - is := domain.GetDomain(ctx).InfoSchema() - tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) - c.Assert(err, IsNil) - p := tbl.Meta().Partition - fmt.Println(p) -} - func (s *testIntegrationSuite5) TestAlterTableAddPartition(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test;") diff --git a/planner/core/mock.go b/planner/core/mock.go index 4419c6f1c8b7e..4161e235244f4 100644 --- a/planner/core/mock.go +++ b/planner/core/mock.go @@ -470,6 +470,7 @@ func MockRangePartitionTable() *model.TableInfo { return tableInfo } +// MockHashPartitionTable mocks a hash partition table for test func MockHashPartitionTable() *model.TableInfo { definitions := []model.PartitionDefinition{ { @@ -505,6 +506,7 @@ func MockHashPartitionTable() *model.TableInfo { return tableInfo } +// MockListPartitionTable mocks a list partition table for test func MockListPartitionTable() *model.TableInfo { definitions := []model.PartitionDefinition{ { diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index e39404dfd300b..f438b46d6d992 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -1396,7 +1396,7 @@ func (s *partitionProcessor) makeUnionAllChildren(ds *DataSource, pi *model.Part children := make([]LogicalPlan, 0, len(pi.Definitions)) partitionNameSet := make(set.StringSet) - usedDefinition := make(map[int64]model.PartitionDefinition, 0) + usedDefinition := make(map[int64]model.PartitionDefinition) for _, r := range or { for i := r.start; i < r.end; i++ { // This is for `table partition (p0,p1)` syntax, only union the specified partition if has specified partitions. @@ -1607,7 +1607,7 @@ func appendMakeUnionAllChildrenTranceStep(ds *DataSource, usedMap map[int64]mode if i > 0 { buffer.WriteString(",") } - buffer.WriteString(fmt.Sprintf("%s", u.Name)) + buffer.WriteString(u.Name.String()) } buffer.WriteString("] after partition pruning") return buffer.String() From 0c08debe72c56bdef1d9fc0bb3c26af9a39cf3db Mon Sep 17 00:00:00 2001 From: yisaer Date: Tue, 14 Dec 2021 20:05:36 +0800 Subject: [PATCH 7/9] address the comment Signed-off-by: yisaer --- planner/core/rule_partition_processor.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index f438b46d6d992..c06de722758ca 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -339,6 +339,7 @@ func (s *partitionProcessor) processHashPartition(ds *DataSource, pi *model.Part } tableDual := LogicalTableDual{RowCount: 0}.Init(ds.SCtx(), ds.blockOffset) tableDual.schema = ds.Schema() + appendTableDualTraceStep(ds, tableDual, opt) return tableDual, nil } @@ -894,6 +895,7 @@ func (s *partitionProcessor) processListPartition(ds *DataSource, pi *model.Part } tableDual := LogicalTableDual{RowCount: 0}.Init(ds.SCtx(), ds.blockOffset) tableDual.schema = ds.Schema() + appendTableDualTraceStep(ds, tableDual, opt) return tableDual, nil } @@ -1615,3 +1617,9 @@ func appendMakeUnionAllChildrenTranceStep(ds *DataSource, usedMap map[int64]mode } opt.appendStepToCurrent(ds.ID(), ds.TP(), reason, action) } + +func appendTableDualTraceStep(ds *DataSource, dual *LogicalTableDual, opt *logicalOptimizeOp) { + action := fmt.Sprintf("DataSource[%v] becomes %v[%v]", ds.ID(), dual.TP(), dual.ID()) + reason := fmt.Sprintf("Datasource[%v] has no available partition table after partition pruning", ds.ID()) + opt.appendStepToCurrent(dual.ID(), dual.TP(), reason, action) +} From 607177083bfacc501eea9552913ef8a900074a5c Mon Sep 17 00:00:00 2001 From: yisaer Date: Tue, 14 Dec 2021 20:17:34 +0800 Subject: [PATCH 8/9] address the comment Signed-off-by: yisaer --- planner/core/rule_partition_processor.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index c06de722758ca..d9fdd841fcba2 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -339,7 +339,7 @@ func (s *partitionProcessor) processHashPartition(ds *DataSource, pi *model.Part } tableDual := LogicalTableDual{RowCount: 0}.Init(ds.SCtx(), ds.blockOffset) tableDual.schema = ds.Schema() - appendTableDualTraceStep(ds, tableDual, opt) + appendNoPartitionChildTraceStep(ds, tableDual, opt) return tableDual, nil } @@ -895,7 +895,7 @@ func (s *partitionProcessor) processListPartition(ds *DataSource, pi *model.Part } tableDual := LogicalTableDual{RowCount: 0}.Init(ds.SCtx(), ds.blockOffset) tableDual.schema = ds.Schema() - appendTableDualTraceStep(ds, tableDual, opt) + appendNoPartitionChildTraceStep(ds, tableDual, opt) return tableDual, nil } @@ -1579,16 +1579,20 @@ func (p *rangeColumnsPruner) pruneUseBinarySearch(sctx sessionctx.Context, op st } func appendMakeUnionAllChildrenTranceStep(ds *DataSource, usedMap map[int64]model.PartitionDefinition, plan LogicalPlan, children []LogicalPlan, opt *logicalOptimizeOp) { + if len(children) == 0 { + appendNoPartitionChildTraceStep(ds, plan, opt) + return + } action := "" reason := "" var used []model.PartitionDefinition for _, def := range usedMap { used = append(used, def) } - if len(children) == 0 { - action = fmt.Sprintf("Datasource[%v] becomes %s[%v]", ds.ID(), plan.TP(), plan.ID()) - reason = fmt.Sprintf("Datasource[%v] has no available partition table after partition pruning", ds.ID()) - } else if len(children) == 1 { + sort.Slice(used, func(i, j int) bool { + return used[i].ID < used[j].ID + }) + if len(children) == 1 { action = fmt.Sprintf("Datasource[%v] becomes %s[%v]", ds.ID(), plan.TP(), plan.ID()) reason = fmt.Sprintf("Datasource[%v] has one available partiton table[%s] after partition pruning", ds.ID(), used[0].Name) } else { @@ -1618,8 +1622,8 @@ func appendMakeUnionAllChildrenTranceStep(ds *DataSource, usedMap map[int64]mode opt.appendStepToCurrent(ds.ID(), ds.TP(), reason, action) } -func appendTableDualTraceStep(ds *DataSource, dual *LogicalTableDual, opt *logicalOptimizeOp) { - action := fmt.Sprintf("DataSource[%v] becomes %v[%v]", ds.ID(), dual.TP(), dual.ID()) +func appendNoPartitionChildTraceStep(ds *DataSource, dual LogicalPlan, opt *logicalOptimizeOp) { + action := fmt.Sprintf("Datasource[%v] becomes %v[%v]", ds.ID(), dual.TP(), dual.ID()) reason := fmt.Sprintf("Datasource[%v] has no available partition table after partition pruning", ds.ID()) opt.appendStepToCurrent(dual.ID(), dual.TP(), reason, action) } From f6422684947af5ac62734a300ce57b5f84e83505 Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 15 Dec 2021 18:21:42 +0800 Subject: [PATCH 9/9] address the comment Signed-off-by: yisaer --- planner/core/logical_plan_trace_test.go | 14 +++++++------- planner/core/rule_partition_processor.go | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/planner/core/logical_plan_trace_test.go b/planner/core/logical_plan_trace_test.go index 8cdc2176ade04..58348bd7712de 100644 --- a/planner/core/logical_plan_trace_test.go +++ b/planner/core/logical_plan_trace_test.go @@ -92,7 +92,7 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleName: "partition_processor", assertRuleSteps: []assertTraceStep{ { - assertReason: "Datasource[1] have multi available partition tables[p1,p2] after partition pruning", + assertReason: "Datasource[1] has multiple needed partitions[p1,p2] after pruning", assertAction: "Datasource[1] becomes PartitionUnion[6] with children[TableScan[1],TableScan[1]]", }, }, @@ -103,7 +103,7 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleName: "partition_processor", assertRuleSteps: []assertTraceStep{ { - assertReason: "Datasource[1] has one available partiton table[p1] after partition pruning", + assertReason: "Datasource[1] has one needed partition[p1] after pruning", assertAction: "Datasource[1] becomes TableScan[1]", }, }, @@ -114,7 +114,7 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleName: "partition_processor", assertRuleSteps: []assertTraceStep{ { - assertReason: "Datasource[1] have multi available partition tables[p1,p2] after partition pruning", + assertReason: "Datasource[1] has multiple needed partitions[p1,p2] after pruning", assertAction: "Datasource[1] becomes PartitionUnion[7] with children[TableScan[1],TableScan[1]]", }, }, @@ -125,7 +125,7 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleName: "partition_processor", assertRuleSteps: []assertTraceStep{ { - assertReason: "Datasource[1] has one available partiton table[p2] after partition pruning", + assertReason: "Datasource[1] has one needed partition[p2] after pruning", assertAction: "Datasource[1] becomes TableScan[1]", }, }, @@ -136,7 +136,7 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleName: "partition_processor", assertRuleSteps: []assertTraceStep{ { - assertReason: "Datasource[1] has no available partition table after partition pruning", + assertReason: "Datasource[1] doesn't have needed partition table after pruning", assertAction: "Datasource[1] becomes TableDual[5]", }, }, @@ -147,7 +147,7 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleName: "partition_processor", assertRuleSteps: []assertTraceStep{ { - assertReason: "Datasource[1] have multi available partition tables[p1,p2] after partition pruning", + assertReason: "Datasource[1] has multiple needed partitions[p1,p2] after pruning", assertAction: "Datasource[1] becomes PartitionUnion[7] with children[TableScan[1],TableScan[1]]", }, }, @@ -158,7 +158,7 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) { assertRuleName: "partition_processor", assertRuleSteps: []assertTraceStep{ { - assertReason: "Datasource[1] has one available partiton table[p1] after partition pruning", + assertReason: "Datasource[1] has one needed partition[p1] after pruning", assertAction: "Datasource[1] becomes TableScan[1]", }, }, diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index d9fdd841fcba2..58e32d31220e9 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -850,7 +850,7 @@ func (s *partitionProcessor) pruneRangePartition(ctx sessionctx.Context, pi *mod if condsToBePruned == nil { return result, nil, nil } - // remove useless predicates after partition pruning + // remove useless predicates after pruning newConds := make([]expression.Expression, 0, len(*condsToBePruned)) for _, cond := range *condsToBePruned { if dataForPrune, ok := pruner.extractDataForPrune(ctx, cond); ok { @@ -1594,7 +1594,7 @@ func appendMakeUnionAllChildrenTranceStep(ds *DataSource, usedMap map[int64]mode }) if len(children) == 1 { action = fmt.Sprintf("Datasource[%v] becomes %s[%v]", ds.ID(), plan.TP(), plan.ID()) - reason = fmt.Sprintf("Datasource[%v] has one available partiton table[%s] after partition pruning", ds.ID(), used[0].Name) + reason = fmt.Sprintf("Datasource[%v] has one needed partition[%s] after pruning", ds.ID(), used[0].Name) } else { action = func() string { buffer := bytes.NewBufferString(fmt.Sprintf("Datasource[%v] becomes %s[%v] with children[", ds.ID(), plan.TP(), plan.ID())) @@ -1608,14 +1608,14 @@ func appendMakeUnionAllChildrenTranceStep(ds *DataSource, usedMap map[int64]mode return buffer.String() }() reason = func() string { - buffer := bytes.NewBufferString(fmt.Sprintf("Datasource[%v] have multi available partition tables[", ds.ID())) + buffer := bytes.NewBufferString(fmt.Sprintf("Datasource[%v] has multiple needed partitions[", ds.ID())) for i, u := range used { if i > 0 { buffer.WriteString(",") } buffer.WriteString(u.Name.String()) } - buffer.WriteString("] after partition pruning") + buffer.WriteString("] after pruning") return buffer.String() }() } @@ -1624,6 +1624,6 @@ func appendMakeUnionAllChildrenTranceStep(ds *DataSource, usedMap map[int64]mode func appendNoPartitionChildTraceStep(ds *DataSource, dual LogicalPlan, opt *logicalOptimizeOp) { action := fmt.Sprintf("Datasource[%v] becomes %v[%v]", ds.ID(), dual.TP(), dual.ID()) - reason := fmt.Sprintf("Datasource[%v] has no available partition table after partition pruning", ds.ID()) + reason := fmt.Sprintf("Datasource[%v] doesn't have needed partition table after pruning", ds.ID()) opt.appendStepToCurrent(dual.ID(), dual.TP(), reason, action) }