From 3dd7fa739d1844111a770b4e3b68a4815864012b Mon Sep 17 00:00:00 2001 From: Kenan Yao Date: Mon, 12 Nov 2018 12:45:25 +0800 Subject: [PATCH 1/2] plan: use `Column.UniqueID` in `conditionChecker` of ranger (#8236) --- planner/core/rule_partition_processor.go | 2 +- statistics/selectivity.go | 2 +- table/tables/partition.go | 15 +++++++----- util/ranger/checker.go | 8 ++----- util/ranger/detacher.go | 18 ++++++--------- util/ranger/ranger_test.go | 29 +++++++++++++++++++++++- 6 files changed, 48 insertions(+), 26 deletions(-) diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index 4b8a90334fc0a..b6a7e6f1dc7b2 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -135,7 +135,7 @@ func (s *partitionProcessor) canBePrune(ctx sessionctx.Context, col *expression. conds = expression.PropagateConstant(ctx, conds) // Calculate the column range to prune. - accessConds := ranger.ExtractAccessConditionsForColumn(conds, col.ColName) + accessConds := ranger.ExtractAccessConditionsForColumn(conds, col.UniqueID) r, err := ranger.BuildColumnRange(accessConds, ctx.GetSessionVars().StmtCtx, col.RetType) if err != nil { return false, errors.Trace(err) diff --git a/statistics/selectivity.go b/statistics/selectivity.go index 8dfebb03fb20e..578f85dc79331 100644 --- a/statistics/selectivity.go +++ b/statistics/selectivity.go @@ -244,7 +244,7 @@ func getMaskAndRanges(ctx sessionctx.Context, exprs []expression.Expression, ran var accessConds, remainedConds []expression.Expression switch rangeType { case ranger.ColumnRangeType: - accessConds = ranger.ExtractAccessConditionsForColumn(exprs, cols[0].ColName) + accessConds = ranger.ExtractAccessConditionsForColumn(exprs, cols[0].UniqueID) ranges, err = ranger.BuildColumnRange(accessConds, sc, cols[0].RetType) case ranger.IndexRangeType: var res *ranger.DetachRangeResult diff --git a/table/tables/partition.go b/table/tables/partition.go index 3d01959a7cfa1..576f4f8876205 100644 --- a/table/tables/partition.go +++ b/table/tables/partition.go @@ -115,6 +115,9 @@ func generatePartitionExpr(tblInfo *model.TableInfo) (*PartitionExpr, error) { partitionPruneExprs := make([]expression.Expression, 0, len(pi.Definitions)) locateExprs := make([]expression.Expression, 0, len(pi.Definitions)) var buf bytes.Buffer + dbName := model.NewCIStr(ctx.GetSessionVars().CurrentDB) + columns := expression.ColumnInfos2ColumnsWithDBName(ctx, dbName, tblInfo.Name, tblInfo.Columns) + schema := expression.NewSchema(columns...) for i := 0; i < len(pi.Definitions); i++ { if strings.EqualFold(pi.Definitions[i].LessThan[0], "MAXVALUE") { // Expr less than maxvalue is always true. @@ -122,13 +125,13 @@ func generatePartitionExpr(tblInfo *model.TableInfo) (*PartitionExpr, error) { } else { fmt.Fprintf(&buf, "((%s) < (%s))", pi.Expr, pi.Definitions[i].LessThan[0]) } - expr, err := expression.ParseSimpleExprWithTableInfo(ctx, buf.String(), tblInfo) + exprs, err := expression.ParseSimpleExprsWithSchema(ctx, buf.String(), schema) if err != nil { // If it got an error here, ddl may hang forever, so this error log is important. log.Error("wrong table partition expression:", errors.ErrorStack(err), buf.String()) return nil, errors.Trace(err) } - locateExprs = append(locateExprs, expr) + locateExprs = append(locateExprs, exprs[0]) if i > 0 { fmt.Fprintf(&buf, " and ((%s) >= (%s))", pi.Expr, pi.Definitions[i-1].LessThan[0]) @@ -137,8 +140,8 @@ func generatePartitionExpr(tblInfo *model.TableInfo) (*PartitionExpr, error) { fmt.Fprintf(&buf, " or ((%s) is null)", pi.Expr) // Extracts the column of the partition expression, it will be used by partition prunning. - if tmp, err1 := expression.ParseSimpleExprWithTableInfo(ctx, pi.Expr, tblInfo); err1 == nil { - if col, ok := tmp.(*expression.Column); ok { + if tmps, err1 := expression.ParseSimpleExprsWithSchema(ctx, pi.Expr, schema); err1 == nil { + if col, ok := tmps[0].(*expression.Column); ok { column = col } } @@ -147,13 +150,13 @@ func generatePartitionExpr(tblInfo *model.TableInfo) (*PartitionExpr, error) { } } - expr, err = expression.ParseSimpleExprWithTableInfo(ctx, buf.String(), tblInfo) + exprs, err = expression.ParseSimpleExprsWithSchema(ctx, buf.String(), schema) if err != nil { // If it got an error here, ddl may hang forever, so this error log is important. log.Error("wrong table partition expression:", errors.ErrorStack(err), buf.String()) return nil, errors.Trace(err) } - partitionPruneExprs = append(partitionPruneExprs, expr) + partitionPruneExprs = append(partitionPruneExprs, exprs[0]) buf.Reset() } return &PartitionExpr{ diff --git a/util/ranger/checker.go b/util/ranger/checker.go index 74ea582ca94de..c99f94c5e54ff 100644 --- a/util/ranger/checker.go +++ b/util/ranger/checker.go @@ -15,14 +15,13 @@ package ranger import ( "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/model" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/types" ) // conditionChecker checks if this condition can be pushed to index planner. type conditionChecker struct { - colName model.CIStr + colUniqueID int64 shouldReserve bool // check if a access condition should be reserved in filter conditions. length int } @@ -135,8 +134,5 @@ func (c *conditionChecker) checkColumn(expr expression.Expression) bool { if !ok { return false } - if c.colName.L != "" { - return c.colName.L == col.ColName.L - } - return true + return c.colUniqueID == col.UniqueID } diff --git a/util/ranger/detacher.go b/util/ranger/detacher.go index 5144450349936..774942cc7b573 100644 --- a/util/ranger/detacher.go +++ b/util/ranger/detacher.go @@ -16,7 +16,6 @@ package ranger import ( "github.com/pingcap/errors" "github.com/pingcap/parser/ast" - "github.com/pingcap/parser/model" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/types" @@ -174,7 +173,7 @@ func detachCNFCondAndBuildRangeForIndex(sctx sessionctx.Context, conditions []ex return res, nil } checker := &conditionChecker{ - colName: cols[eqOrInCount].ColName, + colUniqueID: cols[eqOrInCount].UniqueID, length: lengths[eqOrInCount], shouldReserve: lengths[eqOrInCount] != types.UnspecifiedLength, } @@ -257,7 +256,7 @@ func detachDNFCondAndBuildRangeForIndex(sctx sessionctx.Context, condition *expr cols []*expression.Column, newTpSlice []*types.FieldType, lengths []int) ([]*Range, []expression.Expression, bool, error) { sc := sctx.GetSessionVars().StmtCtx firstColumnChecker := &conditionChecker{ - colName: cols[0].ColName, + colUniqueID: cols[0].UniqueID, shouldReserve: lengths[0] != types.UnspecifiedLength, length: lengths[0], } @@ -376,13 +375,10 @@ func removeAccessConditions(conditions, accessConds []expression.Expression) []e } // ExtractAccessConditionsForColumn detaches the access conditions used for range calculation. -func ExtractAccessConditionsForColumn(conds []expression.Expression, colName model.CIStr) []expression.Expression { - if colName.L == "" { - return nil - } +func ExtractAccessConditionsForColumn(conds []expression.Expression, uniqueID int64) []expression.Expression { checker := conditionChecker{ - colName: colName, - length: types.UnspecifiedLength, + colUniqueID: uniqueID, + length: types.UnspecifiedLength, } accessConds := make([]expression.Expression, 0, 8) return expression.Filter(accessConds, conds, checker.check) @@ -392,8 +388,8 @@ func ExtractAccessConditionsForColumn(conds []expression.Expression, colName mod // calculating the table range. func DetachCondsForTableRange(sctx sessionctx.Context, conds []expression.Expression, col *expression.Column) (accessContditions, otherConditions []expression.Expression) { checker := &conditionChecker{ - colName: col.ColName, - length: types.UnspecifiedLength, + colUniqueID: col.UniqueID, + length: types.UnspecifiedLength, } return detachColumnCNFConditions(sctx, conds, checker) } diff --git a/util/ranger/ranger_test.go b/util/ranger/ranger_test.go index bf5d49ae8fbc9..b06065ac58ccb 100644 --- a/util/ranger/ranger_test.go +++ b/util/ranger/ranger_test.go @@ -954,7 +954,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { } col := expression.ColInfo2Col(sel.Schema().Columns, ds.TableInfo().Columns[tt.colPos]) c.Assert(col, NotNil) - conds = ranger.ExtractAccessConditionsForColumn(conds, col.ColName) + conds = ranger.ExtractAccessConditionsForColumn(conds, col.UniqueID) c.Assert(fmt.Sprintf("%s", conds), Equals, tt.accessConds, Commentf("wrong access conditions for expr: %s", tt.exprStr)) result, err := ranger.BuildColumnRange(conds, new(stmtctx.StatementContext), col.RetType) c.Assert(err, IsNil) @@ -962,3 +962,30 @@ func (s *testRangerSuite) TestColumnRange(c *C) { c.Assert(got, Equals, tt.resultStr, Commentf("different for expr %s, col: %v", tt.exprStr, col)) } } + +func (s *testRangerSuite) TestIndexRangeElimininatedProjection(c *C) { + defer testleak.AfterTest(c)() + dom, store, err := newDomainStoreWithBootstrap(c) + defer func() { + dom.Close() + store.Close() + }() + c.Assert(err, IsNil) + testKit := testkit.NewTestKit(c, store) + testKit.MustExec("use test") + testKit.MustExec("drop table if exists t") + testKit.MustExec("create table t(a int not null, b int not null, primary key(a,b))") + testKit.MustExec("insert into t values(1,2)") + testKit.MustExec("analyze table t") + testKit.MustQuery("explain select * from (select * from t union all select ifnull(a,b), b from t) sub where a > 0").Check(testkit.Rows( + "Union_11 2.00 root ", + "├─IndexReader_17 1.00 root index:IndexScan_16", + "│ └─IndexScan_16 1.00 cop table:t, index:a, b, range:(0 +inf,+inf +inf], keep order:false", + "└─IndexReader_23 1.00 root index:IndexScan_22", + " └─IndexScan_22 1.00 cop table:t, index:a, b, range:(0 +inf,+inf +inf], keep order:false", + )) + testKit.MustQuery("select * from (select * from t union all select ifnull(a,b), b from t) sub where a > 0").Check(testkit.Rows( + "1 2", + "1 2", + )) +} From b6b609bb6ed6686b7c006a3cd210bda8a7e624b3 Mon Sep 17 00:00:00 2001 From: xuhuaiyu <391585975@qq.com> Date: Thu, 7 Mar 2019 14:33:55 +0800 Subject: [PATCH 2/2] make ci happy --- util/ranger/ranger_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/util/ranger/ranger_test.go b/util/ranger/ranger_test.go index b06065ac58ccb..4a060c81e88e0 100644 --- a/util/ranger/ranger_test.go +++ b/util/ranger/ranger_test.go @@ -978,11 +978,13 @@ func (s *testRangerSuite) TestIndexRangeElimininatedProjection(c *C) { testKit.MustExec("insert into t values(1,2)") testKit.MustExec("analyze table t") testKit.MustQuery("explain select * from (select * from t union all select ifnull(a,b), b from t) sub where a > 0").Check(testkit.Rows( - "Union_11 2.00 root ", - "├─IndexReader_17 1.00 root index:IndexScan_16", - "│ └─IndexScan_16 1.00 cop table:t, index:a, b, range:(0 +inf,+inf +inf], keep order:false", - "└─IndexReader_23 1.00 root index:IndexScan_22", - " └─IndexScan_22 1.00 cop table:t, index:a, b, range:(0 +inf,+inf +inf], keep order:false", + "Union_11 1.80 root ", + "├─IndexReader_14 1.00 root index:IndexScan_13", + "│ └─IndexScan_13 1.00 cop table:t, index:a, b, range:(0 +inf,+inf +inf], keep order:false", + "└─Projection_16 0.80 root ifnull(test.t.a, test.t.b), test.t.b", + " └─TableReader_19 0.80 root data:Selection_18", + " └─Selection_18 0.80 cop gt(ifnull(test.t.a, test.t.b), 0)", + " └─TableScan_17 1.00 cop table:t, range:[-inf,+inf], keep order:false", )) testKit.MustQuery("select * from (select * from t union all select ifnull(a,b), b from t) sub where a > 0").Check(testkit.Rows( "1 2",