Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

*: re-implement partition pruning for better performance (#14679) #15628

Merged
merged 2 commits into from
Mar 26, 2020
Merged

*: re-implement partition pruning for better performance (#14679) #15628

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
16f8a36
*: re-implement partition pruning for better performance (#14679)
tiancaiamao Feb 14, 2020
5863162
Merge branch 'release-3.0' into 3.0-partition-pruning
sre-bot Mar 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,698 changes: 771 additions & 927 deletions cmd/explaintest/r/partition_pruning.result
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 0 additions & 2 deletions expression/builtin_time_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
package expression

import (
"fmt"
"math"
"strings"
"time"
Expand Down Expand Up @@ -1723,7 +1722,6 @@ func (s *testEvaluatorSuite) TestUnixTimestamp(c *C) {
}

for _, test := range tests {
fmt.Printf("Begin Test %v\n", test)
expr := s.datumsToConstants([]types.Datum{test.input})
expr[0].GetType().Decimal = test.inputDecimal
resetStmtContext(s.ctx)
Expand Down
229 changes: 197 additions & 32 deletions planner/core/partition_pruning_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/pingcap/parser/model"
"github.com/pingcap/tidb/ddl"
"github.com/pingcap/tidb/expression"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/util/mock"
)

Expand All @@ -29,12 +30,6 @@ type testPartitionPruningSuite struct {
}

func (s *testPartitionPruningSuite) TestCanBePrune(c *C) {
p := parser.New()
stmt, err := p.ParseOneStmt("create table t (d datetime not null)", "", "")
c.Assert(err, IsNil)
tblInfo, err := ddl.BuildTableInfoFromAST(stmt.(*ast.CreateTableStmt))
c.Assert(err, IsNil)

// For the following case:
// CREATE TABLE t1 ( recdate DATETIME NOT NULL )
// PARTITION BY RANGE( TO_DAYS(recdate) ) (
Expand All @@ -43,21 +38,18 @@ func (s *testPartitionPruningSuite) TestCanBePrune(c *C) {
// );
// SELECT * FROM t1 WHERE recdate < '2007-03-08 00:00:00';
// SELECT * FROM t1 WHERE recdate > '2018-03-08 00:00:00';

ctx := mock.NewContext()
columns := expression.ColumnInfos2ColumnsWithDBName(ctx, model.NewCIStr("t"), tblInfo.Name, tblInfo.Columns)
schema := expression.NewSchema(columns...)
partitionExpr, err := expression.ParseSimpleExprsWithSchema(ctx, "to_days(d) < to_days('2007-03-08') and to_days(d) >= to_days('2007-03-07')", schema)
c.Assert(err, IsNil)
queryExpr, err := expression.ParseSimpleExprsWithSchema(ctx, "d < '2000-03-08 00:00:00'", schema)
c.Assert(err, IsNil)
succ, err := s.canBePruned(ctx, nil, partitionExpr[0], queryExpr)
tc := prepareTestCtx(c,
"create table t (d datetime not null)",
"to_days(d)",
)
partitionExpr := tc.expr("to_days(d) < to_days('2007-03-08') and to_days(d) >= to_days('2007-03-07')")
queryExpr := tc.expr("d < '2000-03-08 00:00:00'")
succ, err := s.canBePruned(tc.sctx, nil, partitionExpr[0], queryExpr)
c.Assert(err, IsNil)
c.Assert(succ, IsTrue)

queryExpr, err = expression.ParseSimpleExprsWithSchema(ctx, "d > '2018-03-08 00:00:00'", schema)
c.Assert(err, IsNil)
succ, err = s.canBePruned(ctx, nil, partitionExpr[0], queryExpr)
queryExpr = tc.expr("d > '2018-03-08 00:00:00'")
succ, err = s.canBePruned(tc.sctx, nil, partitionExpr[0], queryExpr)
c.Assert(err, IsNil)
c.Assert(succ, IsTrue)

Expand All @@ -72,24 +64,19 @@ func (s *testPartitionPruningSuite) TestCanBePrune(c *C) {
// PARTITION p2 VALUES LESS THAN (UNIX_TIMESTAMP('2010-01-01 00:00:00')),
// PARTITION p3 VALUES LESS THAN (MAXVALUE)
// );
stmt, err = p.ParseOneStmt("create table t (report_updated timestamp)", "", "")
c.Assert(err, IsNil)
tblInfo, err = ddl.BuildTableInfoFromAST(stmt.(*ast.CreateTableStmt))
c.Assert(err, IsNil)
columns = expression.ColumnInfos2ColumnsWithDBName(ctx, model.NewCIStr("t"), tblInfo.Name, tblInfo.Columns)
schema = expression.NewSchema(columns...)
tc = prepareTestCtx(c,
"create table t (report_updated timestamp)",
"unix_timestamp(report_updated)",
)

partitionExpr, err = expression.ParseSimpleExprsWithSchema(ctx, "unix_timestamp(report_updated) < unix_timestamp('2008-04-01') and unix_timestamp(report_updated) >= unix_timestamp('2008-01-01')", schema)
c.Assert(err, IsNil)
queryExpr, err = expression.ParseSimpleExprsWithSchema(ctx, "report_updated > '2008-05-01 00:00:00'", schema)
c.Assert(err, IsNil)
succ, err = s.canBePruned(ctx, nil, partitionExpr[0], queryExpr)
partitionExpr = tc.expr("unix_timestamp(report_updated) < unix_timestamp('2008-04-01') and unix_timestamp(report_updated) >= unix_timestamp('2008-01-01')")
queryExpr = tc.expr("report_updated > '2008-05-01 00:00:00'")
succ, err = s.canBePruned(tc.sctx, nil, partitionExpr[0], queryExpr)
c.Assert(err, IsNil)
c.Assert(succ, IsTrue)

queryExpr, err = expression.ParseSimpleExprsWithSchema(ctx, "report_updated > unix_timestamp('2008-05-01 00:00:00')", schema)
c.Assert(err, IsNil)
succ, err = s.canBePruned(ctx, nil, partitionExpr[0], queryExpr)
queryExpr = tc.expr("report_updated > unix_timestamp('2008-05-01 00:00:00')")
succ, err = s.canBePruned(tc.sctx, nil, partitionExpr[0], queryExpr)
c.Assert(err, IsNil)
_ = succ
// c.Assert(succ, IsTrue)
Expand All @@ -98,3 +85,181 @@ func (s *testPartitionPruningSuite) TestCanBePrune(c *C) {
// Because unix_timestamp('2008-05-01 00:00:00') is fold to constant int 1564761600, and compare it with timestamp (report_updated)
// need to convert 1564761600 to a timestamp, during that step, an error happen and the result is set to <nil>
}

func (s *testPartitionPruningSuite) TestPruneUseBinarySearch(c *C) {
lessThan := lessThanData{data: []int64{4, 7, 11, 14, 17, 0}, maxvalue: true}
cases := []struct {
input dataForPrune
result partitionRange
}{
{dataForPrune{ast.EQ, 66}, partitionRange{5, 6}},
{dataForPrune{ast.EQ, 14}, partitionRange{4, 5}},
{dataForPrune{ast.EQ, 10}, partitionRange{2, 3}},
{dataForPrune{ast.EQ, 3}, partitionRange{0, 1}},
{dataForPrune{ast.LT, 66}, partitionRange{0, 6}},
{dataForPrune{ast.LT, 14}, partitionRange{0, 4}},
{dataForPrune{ast.LT, 10}, partitionRange{0, 3}},
{dataForPrune{ast.LT, 3}, partitionRange{0, 1}},
{dataForPrune{ast.GE, 66}, partitionRange{5, 6}},
{dataForPrune{ast.GE, 14}, partitionRange{4, 6}},
{dataForPrune{ast.GE, 10}, partitionRange{2, 6}},
{dataForPrune{ast.GE, 3}, partitionRange{0, 6}},
{dataForPrune{ast.GT, 66}, partitionRange{5, 6}},
{dataForPrune{ast.GT, 14}, partitionRange{4, 6}},
{dataForPrune{ast.GT, 10}, partitionRange{3, 6}},
{dataForPrune{ast.GT, 3}, partitionRange{1, 6}},
{dataForPrune{ast.GT, 2}, partitionRange{0, 6}},
{dataForPrune{ast.LE, 66}, partitionRange{0, 6}},
{dataForPrune{ast.LE, 14}, partitionRange{0, 5}},
{dataForPrune{ast.LE, 10}, partitionRange{0, 3}},
{dataForPrune{ast.LE, 3}, partitionRange{0, 1}},
{dataForPrune{ast.IsNull, 0}, partitionRange{0, 1}},
{dataForPrune{"illegal", 0}, partitionRange{0, 6}},
}

for i, ca := range cases {
start, end := pruneUseBinarySearch(lessThan, ca.input, false)
c.Assert(ca.result.start, Equals, start, Commentf("fail = %d", i))
c.Assert(ca.result.end, Equals, end, Commentf("fail = %d", i))
}
}

type testCtx struct {
c *C
sctx sessionctx.Context
schema *expression.Schema
columns []*expression.Column
// names types.NameSlice
lessThan lessThanData
}

func prepareTestCtx(c *C, createTable string, partitionExpr string) *testCtx {
p := parser.New()
stmt, err := p.ParseOneStmt(createTable, "", "")
c.Assert(err, IsNil)
sctx := mock.NewContext()
tblInfo, err := ddl.BuildTableInfoFromAST(stmt.(*ast.CreateTableStmt))
c.Assert(err, IsNil)
columns := expression.ColumnInfos2ColumnsWithDBName(sctx, model.NewCIStr("t"), tblInfo.Name, tblInfo.Columns)
schema := expression.NewSchema(columns...)
return &testCtx{
c: c,
sctx: sctx,
schema: schema,
columns: columns,
// names: names,
}
}

func (tc *testCtx) expr(expr string) []expression.Expression {
res, err := expression.ParseSimpleExprsWithSchema(tc.sctx, expr, tc.schema)
tc.c.Assert(err, IsNil)
return res
}

func (s *testPartitionPruningSuite) TestPartitionRangeForExpr(c *C) {
tc := prepareTestCtx(c,
"create table t (a int)",
"a",
)
lessThan := lessThanData{data: []int64{4, 7, 11, 14, 17, 0}, maxvalue: true}
cases := []struct {
input string
result partitionRangeOR
}{
{"a > 3", partitionRangeOR{{1, 6}}},
{"a < 3", partitionRangeOR{{0, 1}}},
{"a >= 11", partitionRangeOR{{3, 6}}},
{"a > 11", partitionRangeOR{{3, 6}}},
{"a < 11", partitionRangeOR{{0, 3}}},
{"a = 16", partitionRangeOR{{4, 5}}},
{"a > 66", partitionRangeOR{{5, 6}}},
{"a > 2 and a < 10", partitionRangeOR{{0, 3}}},
{"a < 2 or a >= 15", partitionRangeOR{{0, 1}, {4, 6}}},
{"a is null", partitionRangeOR{{0, 1}}},
{"12 > a", partitionRangeOR{{0, 4}}},
{"4 <= a", partitionRangeOR{{1, 6}}},
}

for _, ca := range cases {
expr, err := expression.ParseSimpleExprsWithSchema(tc.sctx, ca.input, tc.schema)
c.Assert(err, IsNil)
result := fullRange(lessThan.length())
result = partitionRangeForExpr(tc.sctx, expr[0], lessThan, tc.columns[0], nil, result)
c.Assert(equalPartitionRangeOR(ca.result, result), IsTrue, Commentf("unexpected:", ca.input))
}
}

func equalPartitionRangeOR(x, y partitionRangeOR) bool {
if len(x) != len(y) {
return false
}
for i := 0; i < len(x); i++ {
if x[i] != y[i] {
return false
}
}
return true
}

func (s *testPartitionPruningSuite) TestPartitionRangeOperation(c *C) {
testIntersectionRange := []struct {
input1 partitionRangeOR
input2 partitionRange
result partitionRangeOR
}{
{input1: partitionRangeOR{{0, 3}, {6, 12}},
input2: partitionRange{4, 7},
result: partitionRangeOR{{6, 7}}},
{input1: partitionRangeOR{{0, 5}},
input2: partitionRange{6, 7},
result: partitionRangeOR{}},
{input1: partitionRangeOR{{0, 4}, {6, 7}, {8, 11}},
input2: partitionRange{3, 9},
result: partitionRangeOR{{3, 4}, {6, 7}, {8, 9}}},
}
for i, ca := range testIntersectionRange {
result := ca.input1.intersectionRange(ca.input2.start, ca.input2.end)
c.Assert(equalPartitionRangeOR(ca.result, result), IsTrue, Commentf("failed %d", i))
}

testIntersection := []struct {
input1 partitionRangeOR
input2 partitionRangeOR
result partitionRangeOR
}{
{input1: partitionRangeOR{{0, 3}, {6, 12}},
input2: partitionRangeOR{{4, 7}},
result: partitionRangeOR{{6, 7}}},
{input1: partitionRangeOR{{4, 7}},
input2: partitionRangeOR{{0, 3}, {6, 12}},
result: partitionRangeOR{{6, 7}}},
{input1: partitionRangeOR{{4, 7}, {8, 10}},
input2: partitionRangeOR{{0, 5}, {6, 12}},
result: partitionRangeOR{{4, 5}, {6, 7}, {8, 10}}},
}
for i, ca := range testIntersection {
result := ca.input1.intersection(ca.input2)
c.Assert(equalPartitionRangeOR(ca.result, result), IsTrue, Commentf("failed %d", i))
}

testUnion := []struct {
input1 partitionRangeOR
input2 partitionRangeOR
result partitionRangeOR
}{
{input1: partitionRangeOR{{0, 1}, {2, 7}},
input2: partitionRangeOR{{3, 5}},
result: partitionRangeOR{{0, 1}, {2, 7}}},
{input1: partitionRangeOR{{2, 7}},
input2: partitionRangeOR{{0, 3}, {4, 12}},
result: partitionRangeOR{{0, 12}}},
{input1: partitionRangeOR{{4, 7}, {8, 10}},
input2: partitionRangeOR{{0, 5}},
result: partitionRangeOR{{0, 7}, {8, 10}}},
}
for i, ca := range testUnion {
result := ca.input1.union(ca.input2)
c.Assert(equalPartitionRangeOR(ca.result, result), IsTrue, Commentf("failed %d", i))
}
}
Loading