From b9bd5a7d7e4dd00ee730d4cc32ca1de6a905708f Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Wed, 9 Mar 2022 01:33:48 +0100 Subject: [PATCH] *: add explicit partition pruning to index joins (#32007) (#32093) close pingcap/tidb#32007 --- ddl/db_partition_test.go | 4 +- executor/builder.go | 80 +++++++++++--------------- executor/distsql.go | 2 +- planner/core/exhaust_physical_plans.go | 1 + planner/core/partition_pruner_test.go | 24 +++++++- 5 files changed, 62 insertions(+), 49 deletions(-) diff --git a/ddl/db_partition_test.go b/ddl/db_partition_test.go index 4003551968d67..2b62322c2ca00 100644 --- a/ddl/db_partition_test.go +++ b/ddl/db_partition_test.go @@ -1296,7 +1296,7 @@ func TestAlterTableTruncatePartitionByList(t *testing.T) { tk.MustExec(`insert into t values (1),(3),(5),(null)`) oldTbl := tk.GetTableByName("test", "t") tk.MustExec(`alter table t truncate partition p1`) - tk.MustQuery("select * from t").Check(testkit.Rows("1", "5", "")) + tk.MustQuery("select * from t").Sort().Check(testkit.Rows("1", "5", "")) tbl := tk.GetTableByName("test", "t") require.NotNil(t, tbl.Meta().Partition) part := tbl.Meta().Partition @@ -1329,7 +1329,7 @@ func TestAlterTableTruncatePartitionByListColumns(t *testing.T) { tk.MustExec(`insert into t values (1,'a'),(3,'a'),(5,'a'),(null,null)`) oldTbl := tk.GetTableByName("test", "t") tk.MustExec(`alter table t truncate partition p1`) - tk.MustQuery("select * from t").Check(testkit.Rows("1 a", "5 a", " ")) + tk.MustQuery("select * from t").Sort().Check(testkit.Rows("1 a", "5 a", " ")) tbl := tk.GetTableByName("test", "t") require.NotNil(t, tbl.Meta().Partition) part := tbl.Meta().Partition diff --git a/executor/builder.go b/executor/builder.go index 30ecf028e9dae..483e4078d2afa 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -3896,26 +3896,37 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte return nil, err } tbInfo := e.table.Meta() - if v.IsCommonHandle { - if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().UseDynamicPartitionPrune() { + if v.IsCommonHandle { kvRanges, err := buildKvRangesForIndexJoin(e.ctx, getPhysicalTableID(e.table), -1, lookUpContents, indexRanges, keyOff2IdxOff, cwc, memTracker, interruptSignal) if err != nil { return nil, err } return builder.buildTableReaderFromKvRanges(ctx, e, kvRanges) } - - tbl, _ := builder.is.TableByID(tbInfo.ID) - pt := tbl.(table.PartitionedTable) - pe, err := tbl.(interface { - PartitionExpr() (*tables.PartitionExpr, error) - }).PartitionExpr() - if err != nil { - return nil, err - } - var kvRanges []kv.KeyRange + handles, _ := dedupHandles(lookUpContents) + return builder.buildTableReaderFromHandles(ctx, e, handles, canReorderHandles) + } + tbl, _ := builder.is.TableByID(tbInfo.ID) + pt := tbl.(table.PartitionedTable) + pe, err := tbl.(interface { + PartitionExpr() (*tables.PartitionExpr, error) + }).PartitionExpr() + if err != nil { + return nil, err + } + partitionInfo := &v.PartitionInfo + usedPartitionList, err := partitionPruning(e.ctx, pt, partitionInfo.PruningConds, partitionInfo.PartitionNames, partitionInfo.Columns, partitionInfo.ColumnNames) + if err != nil { + return nil, err + } + usedPartitions := make(map[int64]table.PhysicalTable, len(usedPartitionList)) + for _, p := range usedPartitionList { + usedPartitions[p.GetPhysicalID()] = p + } + var kvRanges []kv.KeyRange + if v.IsCommonHandle { if len(lookUpContents) > 0 && keyColumnsIncludeAllPartitionColumns(lookUpContents[0].keyCols, pe) { - // In this case we can use dynamic partition pruning. locateKey := make([]types.Datum, e.Schema().Len()) kvRanges = make([]kv.KeyRange, 0, len(lookUpContents)) for _, content := range lookUpContents { @@ -3927,6 +3938,9 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte return nil, err } pid := p.GetPhysicalID() + if _, ok := usedPartitions[pid]; !ok { + continue + } tmp, err := buildKvRangesForIndexJoin(e.ctx, pid, -1, []*indexJoinLookUpContent{content}, indexRanges, keyOff2IdxOff, cwc, nil, interruptSignal) if err != nil { return nil, err @@ -3934,15 +3948,9 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte kvRanges = append(kvRanges, tmp...) } } else { - partitionInfo := &v.PartitionInfo - partitions, err := partitionPruning(e.ctx, pt, partitionInfo.PruningConds, partitionInfo.PartitionNames, partitionInfo.Columns, partitionInfo.ColumnNames) - if err != nil { - return nil, err - } - kvRanges = make([]kv.KeyRange, 0, len(partitions)*len(lookUpContents)) - for _, p := range partitions { - pid := p.GetPhysicalID() - tmp, err := buildKvRangesForIndexJoin(e.ctx, pid, -1, lookUpContents, indexRanges, keyOff2IdxOff, cwc, memTracker, interruptSignal) + kvRanges = make([]kv.KeyRange, 0, len(usedPartitions)*len(lookUpContents)) + for _, p := range usedPartitionList { + tmp, err := buildKvRangesForIndexJoin(e.ctx, p.GetPhysicalID(), -1, lookUpContents, indexRanges, keyOff2IdxOff, cwc, memTracker, interruptSignal) if err != nil { return nil, err } @@ -3953,22 +3961,7 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte } handles, lookUpContents := dedupHandles(lookUpContents) - if tbInfo.GetPartitionInfo() == nil { - return builder.buildTableReaderFromHandles(ctx, e, handles, canReorderHandles) - } - if !builder.ctx.GetSessionVars().UseDynamicPartitionPrune() { - return builder.buildTableReaderFromHandles(ctx, e, handles, canReorderHandles) - } - tbl, _ := builder.is.TableByID(tbInfo.ID) - pt := tbl.(table.PartitionedTable) - pe, err := tbl.(interface { - PartitionExpr() (*tables.PartitionExpr, error) - }).PartitionExpr() - if err != nil { - return nil, err - } - var kvRanges []kv.KeyRange if len(lookUpContents) > 0 && keyColumnsIncludeAllPartitionColumns(lookUpContents[0].keyCols, pe) { locateKey := make([]types.Datum, e.Schema().Len()) kvRanges = make([]kv.KeyRange, 0, len(lookUpContents)) @@ -3981,19 +3974,16 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte return nil, err } pid := p.GetPhysicalID() + if _, ok := usedPartitions[pid]; !ok { + continue + } handle := kv.IntHandle(content.keys[0].GetInt64()) tmp := distsql.TableHandlesToKVRanges(pid, []kv.Handle{handle}) kvRanges = append(kvRanges, tmp...) } } else { - partitionInfo := &v.PartitionInfo - partitions, err := partitionPruning(e.ctx, pt, partitionInfo.PruningConds, partitionInfo.PartitionNames, partitionInfo.Columns, partitionInfo.ColumnNames) - if err != nil { - return nil, err - } - for _, p := range partitions { - pid := p.GetPhysicalID() - tmp := distsql.TableHandlesToKVRanges(pid, handles) + for _, p := range usedPartitionList { + tmp := distsql.TableHandlesToKVRanges(p.GetPhysicalID(), handles) kvRanges = append(kvRanges, tmp...) } } diff --git a/executor/distsql.go b/executor/distsql.go index 4558066f6719c..a43afbd72438b 100644 --- a/executor/distsql.go +++ b/executor/distsql.go @@ -346,7 +346,7 @@ type IndexLookUpExecutor struct { partitionTableMode bool // if this executor is accessing a partition table prunedPartitions []table.PhysicalTable // partition tables need to access partitionRangeMap map[int64][]*ranger.Range - partitionKVRanges [][]kv.KeyRange // kvRanges of each partition table + partitionKVRanges [][]kv.KeyRange // kvRanges of each prunedPartitions // All fields above are immutable. diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 80c506051a735..192cb4374a3c9 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -988,6 +988,7 @@ func (p *LogicalJoin) constructInnerTableScanTask( Columns: ds.TblCols, ColumnNames: ds.names, } + ts.PartitionInfo = copTask.partitionInfo selStats := ts.stats.Scale(selectivity) ts.addPushedDownSelection(copTask, selStats) t := copTask.convertToRootTask(ds.ctx) diff --git a/planner/core/partition_pruner_test.go b/planner/core/partition_pruner_test.go index 8de01732dc0cf..ac4673efaa1ff 100644 --- a/planner/core/partition_pruner_test.go +++ b/planner/core/partition_pruner_test.go @@ -566,7 +566,7 @@ func TestIssue22898(t *testing.T) { tk.MustExec("CREATE TABLE NT_RP3763 (COL1 TINYINT(8) SIGNED COMMENT \"NUMERIC NO INDEX\" DEFAULT 41,COL2 VARCHAR(20),COL3 DATETIME,COL4 BIGINT,COL5 FLOAT) PARTITION BY RANGE (COL1 * COL3) (PARTITION P0 VALUES LESS THAN (0),PARTITION P1 VALUES LESS THAN (10),PARTITION P2 VALUES LESS THAN (20),PARTITION P3 VALUES LESS THAN (30),PARTITION P4 VALUES LESS THAN (40),PARTITION P5 VALUES LESS THAN (50),PARTITION PMX VALUES LESS THAN MAXVALUE);") tk.MustExec("insert into NT_RP3763 (COL1,COL2,COL3,COL4,COL5) values(-82,\"夐齏醕皆磹漋甓崘潮嵙燷渏艂朼洛炷鉢儝鱈肇\",\"5748\\-06\\-26\\ 20:48:49\",-3133527360541070260,-2.624880003397658e+38);") tk.MustExec("insert into NT_RP3763 (COL1,COL2,COL3,COL4,COL5) values(48,\"簖鹩筈匹眜赖泽騈爷詵赺玡婙Ɇ郝鮙廛賙疼舢\",\"7228\\-12\\-13\\ 02:59:54\",-6181009269190017937,2.7731105531290494e+38);") - tk.MustQuery("select * from `NT_RP3763` where `COL1` in (10, 48, -82);").Check(testkit.Rows("-82 夐齏醕皆磹漋甓崘潮嵙燷渏艂朼洛炷鉢儝鱈肇 5748-06-26 20:48:49 -3133527360541070260 -262488000000000000000000000000000000000", "48 簖鹩筈匹眜赖泽騈爷詵赺玡婙Ɇ郝鮙廛賙疼舢 7228-12-13 02:59:54 -6181009269190017937 277311060000000000000000000000000000000")) + tk.MustQuery("select * from `NT_RP3763` where `COL1` in (10, 48, -82);").Sort().Check(testkit.Rows("-82 夐齏醕皆磹漋甓崘潮嵙燷渏艂朼洛炷鉢儝鱈肇 5748-06-26 20:48:49 -3133527360541070260 -262488000000000000000000000000000000000", "48 簖鹩筈匹眜赖泽騈爷詵赺玡婙Ɇ郝鮙廛賙疼舢 7228-12-13 02:59:54 -6181009269190017937 277311060000000000000000000000000000000")) tk.MustQuery("select * from `NT_RP3763` where `COL1` in (48);").Check(testkit.Rows("48 簖鹩筈匹眜赖泽騈爷詵赺玡婙Ɇ郝鮙廛賙疼舢 7228-12-13 02:59:54 -6181009269190017937 277311060000000000000000000000000000000")) } @@ -696,3 +696,25 @@ func TestHashPartitionPruning(t *testing.T) { tk.MustExec("insert into t(col1, col3) values(0, 3522101843073676459);") tk.MustQuery("SELECT col1, COL3 FROM t WHERE COL1 IN (0,14158354938390,0) AND COL3 IN (3522101843073676459,-2846203247576845955,838395691793635638);").Check(testkit.Rows("0 3522101843073676459")) } + +func TestIssue32007(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + tk := testkit.NewTestKit(t, store) + tk.MustExec("create database Issue32007") + tk.MustExec("USE Issue32007") + tk.MustExec("create table t1 (a int, b tinyint, primary key (a)) partition by range (a) (" + + "partition p0 values less than (5)," + + "partition p1 values less than (20)," + + "partition p2 values less than (30)," + + "partition p3 values less than (40)," + + "partition p4 values less than MAXVALUE)") + tk.MustExec("insert into t1 values (0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7), (10, 10), (11, 11), (12, 12), (13, 13), (14, 14), (15, 15), (20, 20), (21, 21), (22, 22), (23, 23), (24, 24), (25, 25), (30, 30), (31, 31), (32, 32), (33, 33), (34, 34), (35, 35), (36, 36), (40, 40), (50, 50), (80, 80), (90, 90), (100, 100)") + tk.MustExec("create table t3 (a int, b mediumint, primary key (a))") + tk.MustExec("insert into t3 values (0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7), (8, 8), (9, 9), (10, 10), (11, 11), (12, 12), (13, 13), (14, 14), (15, 15), (16, 16), (17, 17), (18, 18), (19, 19), (20, 20), (21, 21), (22, 22), (23, 23)") + + tk.MustExec("set @@tidb_partition_prune_mode='static'") + tk.MustQuery("select * from t3 where t3.a <> ALL (select t1.a from t1 partition (p0)) order by t3.a").Sort().Check(testkit.Rows("10 10", "11 11", "12 12", "13 13", "14 14", "15 15", "16 16", "17 17", "18 18", "19 19", "20 20", "21 21", "22 22", "23 23", "5 5", "6 6", "7 7", "8 8", "9 9")) + tk.MustExec("set @@tidb_partition_prune_mode='dynamic'") + tk.MustQuery("select * from t3 where t3.a <> ALL (select t1.a from t1 partition (p0)) order by t3.a").Sort().Check(testkit.Rows("10 10", "11 11", "12 12", "13 13", "14 14", "15 15", "16 16", "17 17", "18 18", "19 19", "20 20", "21 21", "22 22", "23 23", "5 5", "6 6", "7 7", "8 8", "9 9")) +}