diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 7fecffb556e0d..17bcdca69e7b1 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -1256,8 +1256,8 @@ func (cwc *ColWithCmpFuncManager) BuildRangesByRow(ctx sessionctx.Context, row c } exprs = append(exprs, newExpr) // nozero } - // TODO: We already limit range mem usage when buildTemplateRange for inner table of IndexJoin in optimizer phase, - // so we don't need and shouldn't limit range mem usage when we refill inner ranges during the execution phase. + // We already limit range mem usage when buildTemplateRange for inner table of IndexJoin in optimizer phase, so we + // don't need and shouldn't limit range mem usage when we refill inner ranges during the execution phase. ranges, _, _, err := ranger.BuildColumnRange(exprs, ctx, cwc.TargetCol.RetType, cwc.colLength, 0) if err != nil { return nil, err @@ -1398,7 +1398,7 @@ loopOtherConds: // It's clearly that the column c cannot be used to access data. So we need to remove it and reset the IdxOff2KeyOff to // [0 -1 -1]. // So that we can use t1.a=t2.a and t1.b > t2.b-10 and t1.b < t2.b+10 to build ranges then access data. -func (ijHelper *indexJoinBuildHelper) removeUselessEqAndInFunc(idxCols []*expression.Column, notKeyEqAndIn []expression.Expression, _ []*expression.Column) (usefulEqAndIn, uselessOnes []expression.Expression) { +func (ijHelper *indexJoinBuildHelper) removeUselessEqAndInFunc(idxCols []*expression.Column, notKeyEqAndIn []expression.Expression) (usefulEqAndIn, uselessOnes []expression.Expression) { ijHelper.curPossibleUsedKeys = make([]*expression.Column, 0, len(idxCols)) for idxColPos, notKeyColPos := 0, 0; idxColPos < len(idxCols); idxColPos++ { if ijHelper.curIdxOff2KeyOff[idxColPos] != -1 { @@ -1421,7 +1421,7 @@ func (ijHelper *indexJoinBuildHelper) removeUselessEqAndInFunc(idxCols []*expres } type mutableIndexJoinRange struct { - ranges []*ranger.Range + ranges ranger.Ranges buildHelper *indexJoinBuildHelper path *util.AccessPath @@ -1429,7 +1429,7 @@ type mutableIndexJoinRange struct { outerJoinKeys []*expression.Column } -func (mr *mutableIndexJoinRange) Range() []*ranger.Range { +func (mr *mutableIndexJoinRange) Range() ranger.Ranges { return mr.ranges } @@ -1465,6 +1465,18 @@ func (ijHelper *indexJoinBuildHelper) createMutableIndexJoinRange(relatedExprs [ return ranger.Ranges(ranges) } +func (ijHelper *indexJoinBuildHelper) updateByTemplateRangeResult(tempRangeRes *templateRangeResult, + accesses, remained []expression.Expression) (lastColPos int, newAccesses, newRemained []expression.Expression) { + lastColPos = tempRangeRes.keyCntInRange + tempRangeRes.eqAndInCntInRange + ijHelper.curPossibleUsedKeys = ijHelper.curPossibleUsedKeys[:tempRangeRes.keyCntInRange] + for i := lastColPos; i < len(ijHelper.curIdxOff2KeyOff); i++ { + ijHelper.curIdxOff2KeyOff[i] = -1 + } + newAccesses = accesses[:tempRangeRes.eqAndInCntInRange] + newRemained = ranger.AppendConditionsIfNotExist(remained, accesses[tempRangeRes.eqAndInCntInRange:]) + return +} + func (ijHelper *indexJoinBuildHelper) analyzeLookUpFilters(path *util.AccessPath, innerPlan *DataSource, innerJoinKeys []*expression.Column, outerJoinKeys []*expression.Column, rebuildMode bool) (emptyRange bool, err error) { if len(path.IdxCols) == 0 { return false, nil @@ -1476,37 +1488,40 @@ func (ijHelper *indexJoinBuildHelper) analyzeLookUpFilters(path *util.AccessPath return true, nil } var remainedEqAndIn []expression.Expression - notKeyEqAndIn, remainedEqAndIn = ijHelper.removeUselessEqAndInFunc(path.IdxCols, notKeyEqAndIn, outerJoinKeys) + notKeyEqAndIn, remainedEqAndIn = ijHelper.removeUselessEqAndInFunc(path.IdxCols, notKeyEqAndIn) matchedKeyCnt := len(ijHelper.curPossibleUsedKeys) // If no join key is matched while join keys actually are not empty. We don't choose index join for now. if matchedKeyCnt <= 0 && len(innerJoinKeys) > 0 { return false, nil } accesses = append(accesses, notKeyEqAndIn...) - remained = append(remained, remainedEqAndIn...) + remained = ranger.AppendConditionsIfNotExist(remained, remainedEqAndIn) lastColPos := matchedKeyCnt + len(notKeyEqAndIn) // There should be some equal conditions. But we don't need that there must be some join key in accesses here. // A more strict check is applied later. if lastColPos <= 0 { return false, nil } + rangeMaxSize := ijHelper.join.ctx.GetSessionVars().RangeMaxSize + if rebuildMode { + // When rebuilding ranges for plan cache, we don't restrict range mem limit. + rangeMaxSize = 0 + } // If all the index columns are covered by eq/in conditions, we don't need to consider other conditions anymore. if lastColPos == len(path.IdxCols) { - // If there's join key matched index column. Then choose hash join is always a better idea. + // If there's no join key matching index column, then choosing hash join is always a better idea. // e.g. select * from t1, t2 where t2.a=1 and t2.b=1. And t2 has index(a, b). // If we don't have the following check, TiDB will build index join for this case. if matchedKeyCnt <= 0 { return false, nil } remained = append(remained, rangeFilterCandidates...) - ranges, emptyRange, err := ijHelper.buildTemplateRange(matchedKeyCnt, notKeyEqAndIn, nil, false) - if err != nil { - return false, err - } - if emptyRange { - return true, nil + tempRangeRes := ijHelper.buildTemplateRange(matchedKeyCnt, notKeyEqAndIn, nil, false, rangeMaxSize) + if tempRangeRes.err != nil || tempRangeRes.emptyRange || tempRangeRes.keyCntInRange <= 0 { + return tempRangeRes.emptyRange, tempRangeRes.err } - mutableRange := ijHelper.createMutableIndexJoinRange(accesses, ranges, path, innerJoinKeys, outerJoinKeys) + lastColPos, accesses, remained = ijHelper.updateByTemplateRangeResult(tempRangeRes, accesses, remained) + mutableRange := ijHelper.createMutableIndexJoinRange(accesses, tempRangeRes.ranges, path, innerJoinKeys, outerJoinKeys) ijHelper.updateBestChoice(mutableRange, path, accesses, remained, nil, lastColPos, rebuildMode) return false, nil } @@ -1519,52 +1534,64 @@ func (ijHelper *indexJoinBuildHelper) analyzeLookUpFilters(path *util.AccessPath lastColAccess := ijHelper.buildLastColManager(lastPossibleCol, innerPlan, lastColManager) // If the column manager holds no expression, then we fallback to find whether there're useful normal filters if len(lastColAccess) == 0 { - // If there's join key matched index column. Then choose hash join is always a better idea. + // If there's no join key matching index column, then choosing hash join is always a better idea. // e.g. select * from t1, t2 where t2.a=1 and t2.b=1 and t2.c > 10 and t2.c < 20. And t2 has index(a, b, c). // If we don't have the following check, TiDB will build index join for this case. if matchedKeyCnt <= 0 { return false, nil } colAccesses, colRemained := ranger.DetachCondsForColumn(ijHelper.join.ctx, rangeFilterCandidates, lastPossibleCol) - var ranges, nextColRange []*ranger.Range + var nextColRange []*ranger.Range var err error if len(colAccesses) > 0 { - // TODO: restrict the mem usage of column ranges - nextColRange, _, _, err = ranger.BuildColumnRange(colAccesses, ijHelper.join.ctx, lastPossibleCol.RetType, path.IdxColLens[lastColPos], 0) + var colRemained2 []expression.Expression + nextColRange, colAccesses, colRemained2, err = ranger.BuildColumnRange(colAccesses, ijHelper.join.ctx, lastPossibleCol.RetType, path.IdxColLens[lastColPos], rangeMaxSize) if err != nil { return false, err } + if len(colRemained2) > 0 { + colRemained = append(colRemained, colRemained2...) + nextColRange = nil + } } - ranges, emptyRange, err = ijHelper.buildTemplateRange(matchedKeyCnt, notKeyEqAndIn, nextColRange, false) - if err != nil { - return false, err - } - if emptyRange { - return true, nil + tempRangeRes := ijHelper.buildTemplateRange(matchedKeyCnt, notKeyEqAndIn, nextColRange, false, rangeMaxSize) + if tempRangeRes.err != nil || tempRangeRes.emptyRange || tempRangeRes.keyCntInRange <= 0 { + return tempRangeRes.emptyRange, tempRangeRes.err } + lastColPos, accesses, remained = ijHelper.updateByTemplateRangeResult(tempRangeRes, accesses, remained) + // update accesses and remained by colAccesses and colRemained. remained = append(remained, colRemained...) - if path.IdxColLens[lastColPos] != types.UnspecifiedLength { - remained = append(remained, colAccesses...) - } - accesses = append(accesses, colAccesses...) - if len(colAccesses) > 0 { + if tempRangeRes.nextColInRange { + if path.IdxColLens[lastColPos] != types.UnspecifiedLength { + remained = append(remained, colAccesses...) + } + accesses = append(accesses, colAccesses...) lastColPos = lastColPos + 1 + } else { + remained = append(remained, colAccesses...) } - mutableRange := ijHelper.createMutableIndexJoinRange(accesses, ranges, path, innerJoinKeys, outerJoinKeys) + mutableRange := ijHelper.createMutableIndexJoinRange(accesses, tempRangeRes.ranges, path, innerJoinKeys, outerJoinKeys) ijHelper.updateBestChoice(mutableRange, path, accesses, remained, nil, lastColPos, rebuildMode) return false, nil } - accesses = append(accesses, lastColAccess...) - remained = append(remained, rangeFilterCandidates...) - ranges, emptyRange, err := ijHelper.buildTemplateRange(matchedKeyCnt, notKeyEqAndIn, nil, true) - if err != nil { - return false, err + tempRangeRes := ijHelper.buildTemplateRange(matchedKeyCnt, notKeyEqAndIn, nil, true, rangeMaxSize) + if tempRangeRes.err != nil || tempRangeRes.emptyRange { + return tempRangeRes.emptyRange, tempRangeRes.err } - if emptyRange { - return true, nil + lastColPos, accesses, remained = ijHelper.updateByTemplateRangeResult(tempRangeRes, accesses, remained) + + remained = append(remained, rangeFilterCandidates...) + if tempRangeRes.extraColInRange { + accesses = append(accesses, lastColAccess...) + lastColPos = lastColPos + 1 + } else { + if tempRangeRes.keyCntInRange <= 0 { + return false, nil + } + lastColManager = nil } - mutableRange := ijHelper.createMutableIndexJoinRange(accesses, ranges, path, innerJoinKeys, outerJoinKeys) - ijHelper.updateBestChoice(mutableRange, path, accesses, remained, lastColManager, lastColPos+1, rebuildMode) + mutableRange := ijHelper.createMutableIndexJoinRange(accesses, tempRangeRes.ranges, path, innerJoinKeys, outerJoinKeys) + ijHelper.updateBestChoice(mutableRange, path, accesses, remained, lastColManager, lastColPos, rebuildMode) return false, nil } @@ -1603,78 +1630,115 @@ func (ijHelper *indexJoinBuildHelper) updateBestChoice(ranges ranger.MutableRang ijHelper.lastColManager = lastColManager } -func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAndInFuncs []expression.Expression, nextColRange []*ranger.Range, haveExtraCol bool) (ranges []*ranger.Range, emptyRange bool, err error) { - pointLength := matchedKeyCnt + len(eqAndInFuncs) - //nolint:gosimple // false positive unnecessary nil check - if nextColRange != nil { - for _, colRan := range nextColRange { - // The range's exclude status is the same with last col's. - ran := &ranger.Range{ - LowVal: make([]types.Datum, pointLength, pointLength+1), - HighVal: make([]types.Datum, pointLength, pointLength+1), - LowExclude: colRan.LowExclude, - HighExclude: colRan.HighExclude, - Collators: make([]collate.Collator, pointLength, pointLength+1), - } - ran.LowVal = append(ran.LowVal, colRan.LowVal[0]) - ran.HighVal = append(ran.HighVal, colRan.HighVal[0]) - ranges = append(ranges, ran) - } - } else if haveExtraCol { - // Reserve a position for the last col. - ranges = append(ranges, &ranger.Range{ - LowVal: make([]types.Datum, pointLength+1), - HighVal: make([]types.Datum, pointLength+1), - Collators: make([]collate.Collator, pointLength+1), - }) - } else { - ranges = append(ranges, &ranger.Range{ - LowVal: make([]types.Datum, pointLength), - HighVal: make([]types.Datum, pointLength), - Collators: make([]collate.Collator, pointLength), - }) +type templateRangeResult struct { + ranges ranger.Ranges + emptyRange bool + keyCntInRange int + eqAndInCntInRange int + nextColInRange bool + extraColInRange bool + err error +} + +// appendTailTemplateRange appends empty datum for each range in originRanges. +// rangeMaxSize is the max memory limit for ranges. O indicates no memory limit. +// If the second return value is true, it means that the estimated memory after appending datums to originRanges exceeds +// rangeMaxSize and the function rejects appending datums to originRanges. +func appendTailTemplateRange(originRanges ranger.Ranges, rangeMaxSize int64) (ranger.Ranges, bool) { + if rangeMaxSize > 0 && originRanges.MemUsage()+(types.EmptyDatumSize*2+16)*int64(len(originRanges)) > rangeMaxSize { + return originRanges, true } - sc := ijHelper.join.ctx.GetSessionVars().StmtCtx - for i, j := 0, 0; j < len(eqAndInFuncs); i++ { - // This position is occupied by join key. - if ijHelper.curIdxOff2KeyOff[i] != -1 { - continue - } - exprs := []expression.Expression{eqAndInFuncs[j]} - // TODO: restrict the mem usage of column ranges - oneColumnRan, _, _, err := ranger.BuildColumnRange(exprs, ijHelper.join.ctx, ijHelper.curNotUsedIndexCols[j].RetType, ijHelper.curNotUsedColLens[j], 0) - if err != nil { - return nil, false, err - } - if len(oneColumnRan) == 0 { - return nil, true, nil - } - if sc.MemTracker != nil { - sc.MemTracker.Consume(2 * types.EstimatedMemUsage(oneColumnRan[0].LowVal, len(oneColumnRan))) - } - for _, ran := range ranges { - ran.LowVal[i] = oneColumnRan[0].LowVal[0] - ran.HighVal[i] = oneColumnRan[0].HighVal[0] - ran.Collators[i] = oneColumnRan[0].Collators[0] + for _, ran := range originRanges { + ran.LowVal = append(ran.LowVal, types.Datum{}) + ran.HighVal = append(ran.HighVal, types.Datum{}) + ran.Collators = append(ran.Collators, nil) + } + return originRanges, false +} + +func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAndInFuncs []expression.Expression, nextColRange []*ranger.Range, + haveExtraCol bool, rangeMaxSize int64) (res *templateRangeResult) { + res = &templateRangeResult{} + ctx := ijHelper.join.ctx + sc := ctx.GetSessionVars().StmtCtx + defer func() { + if sc.MemTracker != nil && res != nil && len(res.ranges) > 0 { + sc.MemTracker.Consume(2 * types.EstimatedMemUsage(res.ranges[0].LowVal, len(res.ranges))) } - curRangeLen := len(ranges) - for ranIdx := 1; ranIdx < len(oneColumnRan); ranIdx++ { - newRanges := make([]*ranger.Range, 0, curRangeLen) - for oldRangeIdx := 0; oldRangeIdx < curRangeLen; oldRangeIdx++ { - newRange := ranges[oldRangeIdx].Clone() - newRange.LowVal[i] = oneColumnRan[ranIdx].LowVal[0] - newRange.HighVal[i] = oneColumnRan[ranIdx].HighVal[0] - newRange.Collators[i] = oneColumnRan[0].Collators[0] - newRanges = append(newRanges, newRange) + }() + pointLength := matchedKeyCnt + len(eqAndInFuncs) + ranges := ranger.Ranges{&ranger.Range{}} + for i, j := 0, 0; i+j < pointLength; { + if ijHelper.curIdxOff2KeyOff[i+j] != -1 { + // This position is occupied by join key. + var fallback bool + ranges, fallback = appendTailTemplateRange(ranges, rangeMaxSize) + if fallback { + ctx.GetSessionVars().StmtCtx.RecordRangeFallback(rangeMaxSize) + res.ranges = ranges + res.keyCntInRange = i + res.eqAndInCntInRange = j + return + } + i++ + } else { + exprs := []expression.Expression{eqAndInFuncs[j]} + oneColumnRan, _, remained, err := ranger.BuildColumnRange(exprs, ijHelper.join.ctx, ijHelper.curNotUsedIndexCols[j].RetType, ijHelper.curNotUsedColLens[j], rangeMaxSize) + if err != nil { + return &templateRangeResult{err: err} + } + if len(oneColumnRan) == 0 { + return &templateRangeResult{emptyRange: true} + } + if sc.MemTracker != nil { + sc.MemTracker.Consume(2 * types.EstimatedMemUsage(oneColumnRan[0].LowVal, len(oneColumnRan))) + } + if len(remained) > 0 { + res.ranges = ranges + res.keyCntInRange = i + res.eqAndInCntInRange = j + return } - if sc.MemTracker != nil && len(newRanges) != 0 { - sc.MemTracker.Consume(2 * types.EstimatedMemUsage(newRanges[0].LowVal, len(newRanges))) + var fallback bool + ranges, fallback = ranger.AppendRanges2PointRanges(ranges, oneColumnRan, rangeMaxSize) + if fallback { + ctx.GetSessionVars().StmtCtx.RecordRangeFallback(rangeMaxSize) + res.ranges = ranges + res.keyCntInRange = i + res.eqAndInCntInRange = j + return } - ranges = append(ranges, newRanges...) + j++ } - j++ } - return ranges, false, nil + if len(nextColRange) > 0 { + var fallback bool + ranges, fallback = ranger.AppendRanges2PointRanges(ranges, nextColRange, rangeMaxSize) + if fallback { + ctx.GetSessionVars().StmtCtx.RecordRangeFallback(rangeMaxSize) + } + res.ranges = ranges + res.keyCntInRange = matchedKeyCnt + res.eqAndInCntInRange = len(eqAndInFuncs) + res.nextColInRange = !fallback + return + } + if haveExtraCol { + var fallback bool + ranges, fallback = appendTailTemplateRange(ranges, rangeMaxSize) + if fallback { + ctx.GetSessionVars().StmtCtx.RecordRangeFallback(rangeMaxSize) + } + res.ranges = ranges + res.keyCntInRange = matchedKeyCnt + res.eqAndInCntInRange = len(eqAndInFuncs) + res.extraColInRange = !fallback + return + } + res.ranges = ranges + res.keyCntInRange = matchedKeyCnt + res.eqAndInCntInRange = len(eqAndInFuncs) + return } func filterIndexJoinBySessionVars(sc sessionctx.Context, indexJoins []PhysicalPlan) []PhysicalPlan { diff --git a/planner/core/exhaust_physical_plans_test.go b/planner/core/exhaust_physical_plans_test.go index 2232d27e40621..e5734ebb1e834 100644 --- a/planner/core/exhaust_physical_plans_test.go +++ b/planner/core/exhaust_physical_plans_test.go @@ -46,7 +46,15 @@ func rewriteSimpleExpr(ctx sessionctx.Context, str string, schema *expression.Sc return filters, nil } -func TestIndexJoinAnalyzeLookUpFilters(t *testing.T) { +type indexJoinContext struct { + dataSourceNode *DataSource + dsNames types.NameSlice + path *util.AccessPath + joinNode *LogicalJoin + joinColNames types.NameSlice +} + +func prepareForAnalyzeLookUpFilters() *indexJoinContext { ctx := MockContext() ctx.GetSessionVars().PlanID = -1 @@ -101,6 +109,10 @@ func TestIndexJoinAnalyzeLookUpFilters(t *testing.T) { }) dataSourceNode.schema = dsSchema dataSourceNode.stats = &property.StatsInfo{StatsVersion: statistics.PseudoVersion} + path := &util.AccessPath{ + IdxCols: append(make([]*expression.Column, 0, 5), dsSchema.Columns...), + IdxColLens: []int{types.UnspecifiedLength, types.UnspecifiedLength, 2, types.UnspecifiedLength, 2}, + } outerChildSchema := expression.NewSchema() var outerChildNames types.NameSlice outerChildSchema.Append(&expression.Column{ @@ -140,22 +152,65 @@ func TestIndexJoinAnalyzeLookUpFilters(t *testing.T) { DBName: model.NewCIStr("test"), }) joinNode.SetSchema(expression.MergeSchema(dsSchema, outerChildSchema)) - path := &util.AccessPath{ - IdxCols: append(make([]*expression.Column, 0, 5), dsSchema.Columns...), - IdxColLens: []int{types.UnspecifiedLength, types.UnspecifiedLength, 2, types.UnspecifiedLength, 2}, - } joinColNames := append(dsNames.Shallow(), outerChildNames...) + return &indexJoinContext{ + dataSourceNode: dataSourceNode, + dsNames: dsNames, + path: path, + joinNode: joinNode, + joinColNames: joinColNames, + } +} - tests := []struct { - innerKeys []*expression.Column - pushedDownConds string - otherConds string - ranges string - idxOff2KeyOff string - accesses string - remained string - compareFilters string - }{ +type indexJoinTestCase struct { + // input + innerKeys []*expression.Column + pushedDownConds string + otherConds string + rangeMaxSize int64 + rebuildMode bool + + // expected output + ranges string + idxOff2KeyOff string + accesses string + remained string + compareFilters string +} + +func testAnalyzeLookUpFilters(t *testing.T, testCtx *indexJoinContext, testCase *indexJoinTestCase, msgAndArgs ...interface{}) *indexJoinBuildHelper { + ctx := testCtx.dataSourceNode.ctx + ctx.GetSessionVars().RangeMaxSize = testCase.rangeMaxSize + dataSourceNode := testCtx.dataSourceNode + joinNode := testCtx.joinNode + pushed, err := rewriteSimpleExpr(ctx, testCase.pushedDownConds, dataSourceNode.schema, testCtx.dsNames) + require.NoError(t, err) + dataSourceNode.pushedDownConds = pushed + others, err := rewriteSimpleExpr(ctx, testCase.otherConds, joinNode.schema, testCtx.joinColNames) + require.NoError(t, err) + joinNode.OtherConditions = others + helper := &indexJoinBuildHelper{join: joinNode, lastColManager: nil, innerPlan: dataSourceNode} + _, err = helper.analyzeLookUpFilters(testCtx.path, dataSourceNode, testCase.innerKeys, testCase.innerKeys, testCase.rebuildMode) + if helper.chosenRanges == nil { + helper.chosenRanges = ranger.Ranges{} + } + require.NoError(t, err) + if testCase.rebuildMode { + require.Equal(t, testCase.ranges, fmt.Sprintf("%v", helper.chosenRanges.Range()), msgAndArgs) + } else { + require.Equal(t, testCase.accesses, fmt.Sprintf("%v", helper.chosenAccess), msgAndArgs) + require.Equal(t, testCase.ranges, fmt.Sprintf("%v", helper.chosenRanges.Range()), msgAndArgs) + require.Equal(t, testCase.idxOff2KeyOff, fmt.Sprintf("%v", helper.idxOff2KeyOff), msgAndArgs) + require.Equal(t, testCase.remained, fmt.Sprintf("%v", helper.chosenRemained), msgAndArgs) + require.Equal(t, testCase.compareFilters, fmt.Sprintf("%v", helper.lastColManager), msgAndArgs) + } + return helper +} + +func TestIndexJoinAnalyzeLookUpFilters(t *testing.T) { + indexJoinCtx := prepareForAnalyzeLookUpFilters() + dsSchema := indexJoinCtx.dataSourceNode.schema + tests := []indexJoinTestCase{ // Join key not continuous and no pushed filter to match. { innerKeys: []*expression.Column{dsSchema.Columns[0], dsSchema.Columns[2]}, @@ -237,7 +292,7 @@ func TestIndexJoinAnalyzeLookUpFilters(t *testing.T) { innerKeys: []*expression.Column{dsSchema.Columns[1]}, pushedDownConds: "a in (1, 2, 3) and c in ('a', 'b', 'c')", otherConds: "", - ranges: "[[1 NULL \"a\",1 NULL \"a\"] [2 NULL \"a\",2 NULL \"a\"] [3 NULL \"a\",3 NULL \"a\"] [1 NULL \"b\",1 NULL \"b\"] [2 NULL \"b\",2 NULL \"b\"] [3 NULL \"b\",3 NULL \"b\"] [1 NULL \"c\",1 NULL \"c\"] [2 NULL \"c\",2 NULL \"c\"] [3 NULL \"c\",3 NULL \"c\"]]", + ranges: "[[1 NULL \"a\",1 NULL \"a\"] [1 NULL \"b\",1 NULL \"b\"] [1 NULL \"c\",1 NULL \"c\"] [2 NULL \"a\",2 NULL \"a\"] [2 NULL \"b\",2 NULL \"b\"] [2 NULL \"c\",2 NULL \"c\"] [3 NULL \"a\",3 NULL \"a\"] [3 NULL \"b\",3 NULL \"b\"] [3 NULL \"c\",3 NULL \"c\"]]", idxOff2KeyOff: "[-1 0 -1 -1 -1]", accesses: "[in(Column#1, 1, 2, 3) in(Column#3, a, b, c)]", remained: "[in(Column#3, a, b, c)]", @@ -248,7 +303,7 @@ func TestIndexJoinAnalyzeLookUpFilters(t *testing.T) { innerKeys: []*expression.Column{dsSchema.Columns[1]}, pushedDownConds: "a in (1, 2, 3) and c in ('a', 'b', 'c')", otherConds: "d > h and d < h + 100", - ranges: "[[1 NULL \"a\" NULL,1 NULL \"a\" NULL] [2 NULL \"a\" NULL,2 NULL \"a\" NULL] [3 NULL \"a\" NULL,3 NULL \"a\" NULL] [1 NULL \"b\" NULL,1 NULL \"b\" NULL] [2 NULL \"b\" NULL,2 NULL \"b\" NULL] [3 NULL \"b\" NULL,3 NULL \"b\" NULL] [1 NULL \"c\" NULL,1 NULL \"c\" NULL] [2 NULL \"c\" NULL,2 NULL \"c\" NULL] [3 NULL \"c\" NULL,3 NULL \"c\" NULL]]", + ranges: "[[1 NULL \"a\" NULL,1 NULL \"a\" NULL] [1 NULL \"b\" NULL,1 NULL \"b\" NULL] [1 NULL \"c\" NULL,1 NULL \"c\" NULL] [2 NULL \"a\" NULL,2 NULL \"a\" NULL] [2 NULL \"b\" NULL,2 NULL \"b\" NULL] [2 NULL \"c\" NULL,2 NULL \"c\" NULL] [3 NULL \"a\" NULL,3 NULL \"a\" NULL] [3 NULL \"b\" NULL,3 NULL \"b\" NULL] [3 NULL \"c\" NULL,3 NULL \"c\" NULL]]", idxOff2KeyOff: "[-1 0 -1 -1 -1]", accesses: "[in(Column#1, 1, 2, 3) in(Column#3, a, b, c) gt(Column#4, Column#9) lt(Column#4, plus(Column#9, 100))]", remained: "[in(Column#3, a, b, c)]", @@ -277,22 +332,150 @@ func TestIndexJoinAnalyzeLookUpFilters(t *testing.T) { }, } for i, tt := range tests { - pushed, err := rewriteSimpleExpr(ctx, tt.pushedDownConds, dsSchema, dsNames) - require.NoError(t, err) - dataSourceNode.pushedDownConds = pushed - others, err := rewriteSimpleExpr(ctx, tt.otherConds, joinNode.schema, joinColNames) - require.NoError(t, err) - joinNode.OtherConditions = others - helper := &indexJoinBuildHelper{join: joinNode, lastColManager: nil, innerPlan: dataSourceNode} - _, err = helper.analyzeLookUpFilters(path, dataSourceNode, tt.innerKeys, tt.innerKeys, false) - if helper.chosenRanges == nil { - helper.chosenRanges = ranger.Ranges{} + testAnalyzeLookUpFilters(t, indexJoinCtx, &tt, fmt.Sprintf("test case: %v", i)) + } +} + +func checkRangeFallbackAndReset(t *testing.T, ctx sessionctx.Context, expectedRangeFallback bool) { + require.Equal(t, expectedRangeFallback, ctx.GetSessionVars().StmtCtx.RangeFallback) + ctx.GetSessionVars().StmtCtx.RangeFallback = false +} + +func TestRangeFallbackForAnalyzeLookUpFilters(t *testing.T) { + ijCtx := prepareForAnalyzeLookUpFilters() + ctx := ijCtx.dataSourceNode.ctx + dsSchema := ijCtx.dataSourceNode.schema + + type testOutput struct { + ranges string + idxOff2KeyOff string + accesses string + remained string + compareFilters string + } + + tests := []struct { + innerKeys []*expression.Column + pushedDownConds string + otherConds string + outputs []testOutput + }{ + { + innerKeys: []*expression.Column{dsSchema.Columns[1], dsSchema.Columns[3]}, + pushedDownConds: "a in (1, 3) and c in ('aaa', 'bbb')", + otherConds: "", + outputs: []testOutput{ + { + ranges: "[[1 NULL \"aa\" NULL,1 NULL \"aa\" NULL] [1 NULL \"bb\" NULL,1 NULL \"bb\" NULL] [3 NULL \"aa\" NULL,3 NULL \"aa\" NULL] [3 NULL \"bb\" NULL,3 NULL \"bb\" NULL]]", + idxOff2KeyOff: "[-1 0 -1 1 -1]", + accesses: "[in(Column#1, 1, 3) in(Column#3, aaa, bbb)]", + remained: "[in(Column#3, aaa, bbb)]", + compareFilters: "", + }, + { + ranges: "[[1 NULL \"aa\",1 NULL \"aa\"] [1 NULL \"bb\",1 NULL \"bb\"] [3 NULL \"aa\",3 NULL \"aa\"] [3 NULL \"bb\",3 NULL \"bb\"]]", + idxOff2KeyOff: "[-1 0 -1 -1 -1]", + accesses: "[in(Column#1, 1, 3) in(Column#3, aaa, bbb)]", + remained: "[in(Column#3, aaa, bbb)]", + compareFilters: "", + }, + { + ranges: "[[1 NULL,1 NULL] [3 NULL,3 NULL]]", + idxOff2KeyOff: "[-1 0 -1 -1 -1]", + accesses: "[in(Column#1, 1, 3)]", + remained: "[in(Column#3, aaa, bbb)]", + compareFilters: "", + }, + { + ranges: "[]", + idxOff2KeyOff: "[]", + accesses: "[]", + remained: "[]", + compareFilters: "", + }, + }, + }, + { + // test haveExtraCol + innerKeys: []*expression.Column{dsSchema.Columns[0]}, + pushedDownConds: "b in (1, 3, 5)", + otherConds: "c > g and c < concat(g, 'aaa')", + outputs: []testOutput{ + { + ranges: "[[NULL 1 NULL,NULL 1 NULL] [NULL 3 NULL,NULL 3 NULL] [NULL 5 NULL,NULL 5 NULL]]", + idxOff2KeyOff: "[0 -1 -1 -1 -1]", + accesses: "[in(Column#2, 1, 3, 5) gt(Column#3, Column#8) lt(Column#3, concat(Column#8, aaa))]", + remained: "[]", + compareFilters: "gt(Column#3, Column#8) lt(Column#3, concat(Column#8, aaa))", + }, + { + ranges: "[[NULL 1,NULL 1] [NULL 3,NULL 3] [NULL 5,NULL 5]]", + idxOff2KeyOff: "[0 -1 -1 -1 -1]", + accesses: "[in(Column#2, 1, 3, 5)]", + remained: "[]", + compareFilters: "", + }, + { + ranges: "[[NULL,NULL]]", + idxOff2KeyOff: "[0 -1 -1 -1 -1]", + accesses: "[]", + remained: "[in(Column#2, 1, 3, 5)]", + compareFilters: "", + }, + }, + }, + { + // test nextColRange + innerKeys: []*expression.Column{dsSchema.Columns[1]}, + pushedDownConds: "a in (1, 3) and c > 'aaa' and c < 'bbb'", + otherConds: "", + outputs: []testOutput{ + { + ranges: "[[1 NULL \"aa\",1 NULL \"bb\"] [3 NULL \"aa\",3 NULL \"bb\"]]", + idxOff2KeyOff: "[-1 0 -1 -1 -1]", + accesses: "[in(Column#1, 1, 3) gt(Column#3, aaa) lt(Column#3, bbb)]", + remained: "[gt(Column#3, aaa) lt(Column#3, bbb)]", + compareFilters: "", + }, + { + ranges: "[[1 NULL,1 NULL] [3 NULL,3 NULL]]", + idxOff2KeyOff: "[-1 0 -1 -1 -1]", + accesses: "[in(Column#1, 1, 3)]", + remained: "[gt(Column#3, aaa) lt(Column#3, bbb)]", + compareFilters: "", + }, + }, + }, + } + for _, tt := range tests { + ijCase := &indexJoinTestCase{ + innerKeys: tt.innerKeys, + pushedDownConds: tt.pushedDownConds, + otherConds: tt.otherConds, + rangeMaxSize: 0, } - require.NoError(t, err) - require.Equal(t, tt.accesses, fmt.Sprintf("%v", helper.chosenAccess)) - require.Equal(t, tt.ranges, fmt.Sprintf("%v", helper.chosenRanges.Range()), "test case: ", i) - require.Equal(t, tt.idxOff2KeyOff, fmt.Sprintf("%v", helper.idxOff2KeyOff)) - require.Equal(t, tt.remained, fmt.Sprintf("%v", helper.chosenRemained)) - require.Equal(t, tt.compareFilters, fmt.Sprintf("%v", helper.lastColManager)) + for i, res := range tt.outputs { + ijCase.ranges = res.ranges + ijCase.idxOff2KeyOff = res.idxOff2KeyOff + ijCase.accesses = res.accesses + ijCase.remained = res.remained + ijCase.compareFilters = res.compareFilters + ijHelper := testAnalyzeLookUpFilters(t, ijCtx, ijCase) + checkRangeFallbackAndReset(t, ctx, i > 0) + ijCase.rangeMaxSize = ijHelper.chosenRanges.Range().MemUsage() - 1 + } + } + + // test that building ranges doesn't have mem limit under rebuild mode + ijCase := &indexJoinTestCase{ + innerKeys: []*expression.Column{dsSchema.Columns[0], dsSchema.Columns[2]}, + pushedDownConds: "b in (1, 3) and d in (2, 4)", + otherConds: "", + rangeMaxSize: 1, + rebuildMode: true, + ranges: "[[NULL 1 NULL 2,NULL 1 NULL 2] [NULL 1 NULL 4,NULL 1 NULL 4] [NULL 3 NULL 2,NULL 3 NULL 2] [NULL 3 NULL 4,NULL 3 NULL 4]]", } + ijHelper := testAnalyzeLookUpFilters(t, ijCtx, ijCase) + checkRangeFallbackAndReset(t, ctx, false) + require.Greater(t, ijHelper.chosenRanges.Range().MemUsage(), ijCase.rangeMaxSize) } diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index f7eaa1c29d592..3b3e4d9201cd3 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -7585,3 +7585,80 @@ func TestExplainAnalyzeDMLCommit(t *testing.T) { require.NoError(t, err) tk.MustQuery("select * from t").Check(testkit.Rows()) } + +func TestIndexJoinRangeFallback(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1, t2") + tk.MustExec("create table t1(a int, b int, c varchar(10), d varchar(10), index idx_a_b_c_d(a, b, c(2), d(2)))") + tk.MustExec("create table t2(e int, f int, g varchar(10), h varchar(10))") + + var input []string + var output []struct { + SQL string + Plan []string + Warn []string + } + integrationSuiteData := core.GetIntegrationSuiteData() + integrationSuiteData.LoadTestCases(t, &input, &output) + for i, tt := range input { + setStmt := strings.HasPrefix(tt, "set") + testdata.OnRecord(func() { + output[i].SQL = tt + if !setStmt { + output[i].Plan = testdata.ConvertRowsToStrings(tk.MustQuery(tt).Rows()) + output[i].Warn = testdata.ConvertSQLWarnToStrings(tk.Session().GetSessionVars().StmtCtx.GetWarnings()) + } + }) + if setStmt { + tk.MustExec(tt) + } else { + tk.MustQuery(tt).Check(testkit.Rows(output[i].Plan...)) + require.Equal(t, output[i].Warn, testdata.ConvertSQLWarnToStrings(tk.Session().GetSessionVars().StmtCtx.GetWarnings())) + } + } +} + +func TestPlanCacheForIndexJoinRangeFallback(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec(`set @@tidb_enable_prepared_plan_cache=1`) + tk.MustExec("set @@tidb_enable_collect_execution_info=0") + tk.MustExec("use test") + tk.MustExec("drop table if exists t1, t2") + tk.MustExec("create table t1(a int, b varchar(10), c varchar(10), index idx_a_b(a, b))") + tk.MustExec("create table t2(d int)") + tk.MustExec("set @@tidb_opt_range_max_size=1275") + // 1275 is enough for [? a,? a], [? b,? b], [? c,? c] but is not enough for [? aaaaaa,? aaaaaa], [? bbbbbb,? bbbbbb], [? cccccc,? cccccc]. + rows := tk.MustQuery("explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.d where t1.b in ('a', 'b', 'c')").Rows() + require.True(t, strings.Contains(rows[6][4].(string), "range: decided by [eq(test.t1.a, test.t2.d) in(test.t1.b, a, b, c)]")) + tk.MustQuery("show warnings").Check(testkit.Rows()) + rows = tk.MustQuery("explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.d where t1.b in ('aaaaaa', 'bbbbbb', 'cccccc');").Rows() + require.True(t, strings.Contains(rows[6][4].(string), "range: decided by [eq(test.t1.a, test.t2.d)]")) + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 Memory capacity of 1275 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen")) + + tk.MustExec("prepare stmt1 from 'select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.d where t1.b in (?, ?, ?)'") + tk.MustExec("set @a='a', @b='b', @c='c'") + tk.MustExec("execute stmt1 using @a, @b, @c") + tk.MustQuery("show warnings").Check(testkit.Rows()) + tk.MustExec("set @a='aaaaaa', @b='bbbbbb', @c='cccccc'") + tk.MustExec("execute stmt1 using @a, @b, @c") + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1")) + tk.MustExec("execute stmt1 using @a, @b, @c") + tkProcess := tk.Session().ShowProcess() + ps := []*util.ProcessInfo{tkProcess} + tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps}) + rows = tk.MustQuery(fmt.Sprintf("explain for connection %d", tkProcess.ID)).Rows() + // We don't limit range mem usage when rebuilding index join ranges for the cached plan. So [? aaaaaa,? aaaaaa], [? bbbbbb,? bbbbbb], [? cccccc,? cccccc] can be built. + // TODO: use the right range `range: decided by [eq(test.t1.a, test.t2.d) in(test.t1.b, aaaaaa, bbbbbb, cccccc)]` after https://github.com/pingcap/tidb/issues/38269 is fixed. + require.True(t, strings.Contains(rows[6][4].(string), "range: decided by [eq(test.t1.a, test.t2.d) in(test.t1.b, a, b, c)]")) + + // Test the plan with range fallback would not be put into cache. + tk.MustExec("prepare stmt2 from 'select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.d where t1.b in (?, ?, ?, ?, ?)'") + tk.MustExec("set @a='a', @b='b', @c='c', @d='d', @e='e'") + tk.MustExec("execute stmt2 using @a, @b, @c, @d, @e") + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 Memory capacity of 1275 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen")) + tk.MustExec("execute stmt2 using @a, @b, @c, @d, @e") + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0")) +} diff --git a/planner/core/testdata/integration_suite_in.json b/planner/core/testdata/integration_suite_in.json index 63a5ed8b9c36d..940c67c7f0f47 100644 --- a/planner/core/testdata/integration_suite_in.json +++ b/planner/core/testdata/integration_suite_in.json @@ -990,5 +990,26 @@ // But fine grained shuffle doesn't support group by for now. "explain format = 'brief' select row_number() over w1, count(c2) from t1 group by c1 having c1 > 10 window w1 as (partition by c1 order by c2);" ] + }, + { + "name": "TestIndexJoinRangeFallback", + "cases": [ + "set @@tidb_opt_range_max_size = 0", + "explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.b = t2.e and t1.d = t2.g where t1.a in (1, 3) and t1.c in ('aaa', 'bbb')", + "set @@tidb_opt_range_max_size = 2900", + "explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.b = t2.e and t1.d = t2.g where t1.a in (1, 3) and t1.c in ('aaa', 'bbb')", + "set @@tidb_opt_range_max_size = 2300", + "explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.b = t2.e and t1.d = t2.g where t1.a in (1, 3) and t1.c in ('aaa', 'bbb')", + "set @@tidb_opt_range_max_size = 700", + "explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.b = t2.e and t1.d = t2.g where t1.a in (1, 3) and t1.c in ('aaa', 'bbb')", + "set @@tidb_opt_range_max_size = 0", + "explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.e where t1.b > 1 and t1.b < 10", + "set @@tidb_opt_range_max_size = 300", + "explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.e where t1.b > 1 and t1.b < 10", + "set @@tidb_opt_range_max_size = 0", + "explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.e where t1.b > t2.f and t1.b < t2.f + 10", + "set @@tidb_opt_range_max_size = 300", + "explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.e where t1.b > t2.f and t1.b < t2.f + 10" + ] } ] diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index 4a2c2b7cdddaf..a42fe6c2f03e8 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -7390,5 +7390,177 @@ ] } ] + }, + { + "Name": "TestIndexJoinRangeFallback", + "Cases": [ + { + "SQL": "set @@tidb_opt_range_max_size = 0", + "Plan": null, + "Warn": null + }, + { + "SQL": "explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.b = t2.e and t1.d = t2.g where t1.a in (1, 3) and t1.c in ('aaa', 'bbb')", + "Plan": [ + "IndexJoin 0.50 root inner join, inner:IndexLookUp, outer key:test.t2.e, test.t2.g, inner key:test.t1.b, test.t1.d, equal cond:eq(test.t2.e, test.t1.b), eq(test.t2.g, test.t1.d)", + "├─TableReader(Build) 9980.01 root data:Selection", + "│ └─Selection 9980.01 cop[tikv] not(isnull(test.t2.e)), not(isnull(test.t2.g))", + "│ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", + "└─IndexLookUp(Probe) 0.00 root ", + " ├─Selection(Build) 0.03 cop[tikv] not(isnull(test.t1.b))", + " │ └─IndexRangeScan 0.03 cop[tikv] table:t1, index:idx_a_b_c_d(a, b, c, d) range: decided by [eq(test.t1.b, test.t2.e) eq(test.t1.d, test.t2.g) in(test.t1.a, 1, 3) in(test.t1.c, aaa, bbb)], keep order:false, stats:pseudo", + " └─Selection(Probe) 0.00 cop[tikv] in(test.t1.c, \"aaa\", \"bbb\"), not(isnull(test.t1.d))", + " └─TableRowIDScan 0.03 cop[tikv] table:t1 keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "set @@tidb_opt_range_max_size = 2900", + "Plan": null, + "Warn": null + }, + { + "SQL": "explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.b = t2.e and t1.d = t2.g where t1.a in (1, 3) and t1.c in ('aaa', 'bbb')", + "Plan": [ + "IndexJoin 0.50 root inner join, inner:IndexLookUp, outer key:test.t2.e, inner key:test.t1.b, equal cond:eq(test.t2.e, test.t1.b), eq(test.t2.g, test.t1.d)", + "├─TableReader(Build) 9980.01 root data:Selection", + "│ └─Selection 9980.01 cop[tikv] not(isnull(test.t2.e)), not(isnull(test.t2.g))", + "│ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", + "└─IndexLookUp(Probe) 0.00 root ", + " ├─Selection(Build) 0.03 cop[tikv] not(isnull(test.t1.b))", + " │ └─IndexRangeScan 0.03 cop[tikv] table:t1, index:idx_a_b_c_d(a, b, c, d) range: decided by [eq(test.t1.b, test.t2.e) in(test.t1.a, 1, 3) in(test.t1.c, aaa, bbb)], keep order:false, stats:pseudo", + " └─Selection(Probe) 0.00 cop[tikv] in(test.t1.c, \"aaa\", \"bbb\"), not(isnull(test.t1.d))", + " └─TableRowIDScan 0.03 cop[tikv] table:t1 keep order:false, stats:pseudo" + ], + "Warn": [ + "Memory capacity of 2900 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen" + ] + }, + { + "SQL": "set @@tidb_opt_range_max_size = 2300", + "Plan": null, + "Warn": null + }, + { + "SQL": "explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.b = t2.e and t1.d = t2.g where t1.a in (1, 3) and t1.c in ('aaa', 'bbb')", + "Plan": [ + "IndexJoin 0.50 root inner join, inner:IndexLookUp, outer key:test.t2.e, inner key:test.t1.b, equal cond:eq(test.t2.e, test.t1.b), eq(test.t2.g, test.t1.d)", + "├─TableReader(Build) 9980.01 root data:Selection", + "│ └─Selection 9980.01 cop[tikv] not(isnull(test.t2.e)), not(isnull(test.t2.g))", + "│ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", + "└─IndexLookUp(Probe) 0.00 root ", + " ├─Selection(Build) 0.03 cop[tikv] not(isnull(test.t1.b))", + " │ └─IndexRangeScan 0.03 cop[tikv] table:t1, index:idx_a_b_c_d(a, b, c, d) range: decided by [eq(test.t1.b, test.t2.e) in(test.t1.a, 1, 3)], keep order:false, stats:pseudo", + " └─Selection(Probe) 0.00 cop[tikv] in(test.t1.c, \"aaa\", \"bbb\"), not(isnull(test.t1.d))", + " └─TableRowIDScan 0.03 cop[tikv] table:t1 keep order:false, stats:pseudo" + ], + "Warn": [ + "Memory capacity of 2300 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen" + ] + }, + { + "SQL": "set @@tidb_opt_range_max_size = 700", + "Plan": null, + "Warn": null + }, + { + "SQL": "explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.b = t2.e and t1.d = t2.g where t1.a in (1, 3) and t1.c in ('aaa', 'bbb')", + "Plan": [ + "HashJoin 0.05 root inner join, equal:[eq(test.t1.b, test.t2.e) eq(test.t1.d, test.t2.g)]", + "├─IndexLookUp(Build) 0.04 root ", + "│ ├─Selection(Build) 19.98 cop[tikv] not(isnull(test.t1.b))", + "│ │ └─IndexRangeScan 20.00 cop[tikv] table:t1, index:idx_a_b_c_d(a, b, c, d) range:[1,1], [3,3], keep order:false, stats:pseudo", + "│ └─Selection(Probe) 0.04 cop[tikv] in(test.t1.c, \"aaa\", \"bbb\"), not(isnull(test.t1.d))", + "│ └─TableRowIDScan 19.98 cop[tikv] table:t1 keep order:false, stats:pseudo", + "└─TableReader(Probe) 9980.01 root data:Selection", + " └─Selection 9980.01 cop[tikv] not(isnull(test.t2.e)), not(isnull(test.t2.g))", + " └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo" + ], + "Warn": [ + "Memory capacity of 700 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen", + "[planner:1815]Optimizer Hint /*+ INL_JOIN(t1) */ or /*+ TIDB_INLJ(t1) */ is inapplicable" + ] + }, + { + "SQL": "set @@tidb_opt_range_max_size = 0", + "Plan": null, + "Warn": null + }, + { + "SQL": "explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.e where t1.b > 1 and t1.b < 10", + "Plan": [ + "IndexJoin 312.19 root inner join, inner:IndexLookUp, outer key:test.t2.e, inner key:test.t1.a, equal cond:eq(test.t2.e, test.t1.a)", + "├─TableReader(Build) 9990.00 root data:Selection", + "│ └─Selection 9990.00 cop[tikv] not(isnull(test.t2.e))", + "│ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", + "└─IndexLookUp(Probe) 0.03 root ", + " ├─Selection(Build) 0.03 cop[tikv] not(isnull(test.t1.a))", + " │ └─IndexRangeScan 0.03 cop[tikv] table:t1, index:idx_a_b_c_d(a, b, c, d) range: decided by [eq(test.t1.a, test.t2.e) gt(test.t1.b, 1) lt(test.t1.b, 10)], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 0.03 cop[tikv] table:t1 keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "set @@tidb_opt_range_max_size = 300", + "Plan": null, + "Warn": null + }, + { + "SQL": "explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.e where t1.b > 1 and t1.b < 10", + "Plan": [ + "IndexJoin 312.19 root inner join, inner:IndexLookUp, outer key:test.t2.e, inner key:test.t1.a, equal cond:eq(test.t2.e, test.t1.a)", + "├─TableReader(Build) 9990.00 root data:Selection", + "│ └─Selection 9990.00 cop[tikv] not(isnull(test.t2.e))", + "│ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", + "└─IndexLookUp(Probe) 0.03 root ", + " ├─Selection(Build) 0.03 cop[tikv] gt(test.t1.b, 1), lt(test.t1.b, 10), not(isnull(test.t1.a))", + " │ └─IndexRangeScan 1.25 cop[tikv] table:t1, index:idx_a_b_c_d(a, b, c, d) range: decided by [eq(test.t1.a, test.t2.e)], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 0.03 cop[tikv] table:t1 keep order:false, stats:pseudo" + ], + "Warn": [ + "Memory capacity of 300 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen" + ] + }, + { + "SQL": "set @@tidb_opt_range_max_size = 0", + "Plan": null, + "Warn": null + }, + { + "SQL": "explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.e where t1.b > t2.f and t1.b < t2.f + 10", + "Plan": [ + "IndexJoin 12475.01 root inner join, inner:IndexLookUp, outer key:test.t2.e, inner key:test.t1.a, equal cond:eq(test.t2.e, test.t1.a), other cond:gt(test.t1.b, test.t2.f), lt(test.t1.b, plus(test.t2.f, 10))", + "├─TableReader(Build) 9980.01 root data:Selection", + "│ └─Selection 9980.01 cop[tikv] not(isnull(test.t2.e)), not(isnull(test.t2.f))", + "│ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", + "└─IndexLookUp(Probe) 1.25 root ", + " ├─Selection(Build) 1.25 cop[tikv] not(isnull(test.t1.a)), not(isnull(test.t1.b))", + " │ └─IndexRangeScan 1.25 cop[tikv] table:t1, index:idx_a_b_c_d(a, b, c, d) range: decided by [eq(test.t1.a, test.t2.e) gt(test.t1.b, test.t2.f) lt(test.t1.b, plus(test.t2.f, 10))], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 1.25 cop[tikv] table:t1 keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "set @@tidb_opt_range_max_size = 300", + "Plan": null, + "Warn": null + }, + { + "SQL": "explain format='brief' select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.e where t1.b > t2.f and t1.b < t2.f + 10", + "Plan": [ + "IndexJoin 12475.01 root inner join, inner:IndexLookUp, outer key:test.t2.e, inner key:test.t1.a, equal cond:eq(test.t2.e, test.t1.a), other cond:gt(test.t1.b, test.t2.f), lt(test.t1.b, plus(test.t2.f, 10))", + "├─TableReader(Build) 9980.01 root data:Selection", + "│ └─Selection 9980.01 cop[tikv] not(isnull(test.t2.e)), not(isnull(test.t2.f))", + "│ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", + "└─IndexLookUp(Probe) 1.25 root ", + " ├─Selection(Build) 1.25 cop[tikv] not(isnull(test.t1.a)), not(isnull(test.t1.b))", + " │ └─IndexRangeScan 1.25 cop[tikv] table:t1, index:idx_a_b_c_d(a, b, c, d) range: decided by [eq(test.t1.a, test.t2.e)], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 1.25 cop[tikv] table:t1 keep order:false, stats:pseudo" + ], + "Warn": [ + "Memory capacity of 300 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen" + ] + } + ] } ] diff --git a/util/ranger/detacher.go b/util/ranger/detacher.go index 5c5154b673f20..cd47c50d3cce0 100644 --- a/util/ranger/detacher.go +++ b/util/ranger/detacher.go @@ -370,12 +370,12 @@ func (d *rangeDetacher) detachCNFCondAndBuildRangeForIndex(conditions []expressi return &DetachRangeResult{}, nil } if len(tailRes.AccessConds) > 0 { - newRanges, rangeFallback := appendRanges2PointRanges(pointRanges, tailRes.Ranges, d.rangeMaxSize) + newRanges, rangeFallback := AppendRanges2PointRanges(pointRanges, tailRes.Ranges, d.rangeMaxSize) if rangeFallback { d.sctx.GetSessionVars().StmtCtx.RecordRangeFallback(d.rangeMaxSize) res.RemainedConds = append(res.RemainedConds, tailRes.AccessConds...) - // Some conditions may be in both tailRes.AccessConds and tailRes.RemainedConds so we call appendConditionsIfNotExist here. - res.RemainedConds = appendConditionsIfNotExist(res.RemainedConds, tailRes.RemainedConds) + // Some conditions may be in both tailRes.AccessConds and tailRes.RemainedConds so we call AppendConditionsIfNotExist here. + res.RemainedConds = AppendConditionsIfNotExist(res.RemainedConds, tailRes.RemainedConds) return res, nil } res.Ranges = newRanges @@ -405,7 +405,7 @@ func (d *rangeDetacher) detachCNFCondAndBuildRangeForIndex(conditions []expressi // [10, 10] [30, 30] exceeds range mem limit, we add `a = 10 or a = 30` back to RemainedConds, which is actually // unnecessary because `(a = 10 and b = 20) or (a = 30 and b = 40)` is already in RemainedConds. // TODO: we will optimize it later. - res.RemainedConds = appendConditionsIfNotExist(res.RemainedConds, remainedConds) + res.RemainedConds = AppendConditionsIfNotExist(res.RemainedConds, remainedConds) res.Ranges = ranges return res, nil } @@ -904,7 +904,8 @@ func removeConditions(conditions, condsToRemove []expression.Expression) []expre return filterConds } -func appendConditionsIfNotExist(conditions, condsToAppend []expression.Expression) []expression.Expression { +// AppendConditionsIfNotExist appends conditions if they are absent. +func AppendConditionsIfNotExist(conditions, condsToAppend []expression.Expression) []expression.Expression { shouldAppend := make([]expression.Expression, 0, len(condsToAppend)) for _, cond := range condsToAppend { if !expression.Contains(conditions, cond) { diff --git a/util/ranger/ranger.go b/util/ranger/ranger.go index aca78c386078e..19976ee061a9f 100644 --- a/util/ranger/ranger.go +++ b/util/ranger/ranger.go @@ -334,11 +334,34 @@ func estimateMemUsageForAppendRanges2PointRanges(pointRanges Ranges, ranges Rang return (EmptyRangeSize+collatorSize)*len1*len2 + getRangesTotalDatumSize(pointRanges)*len2 + getRangesTotalDatumSize(ranges)*len1 } -// appendRanges2PointRanges appends additional ranges to point ranges. +// appendRange2PointRange appends suffixRange to pointRange. +func appendRange2PointRange(pointRange, suffixRange *Range) *Range { + lowVal := make([]types.Datum, 0, len(pointRange.LowVal)+len(suffixRange.LowVal)) + lowVal = append(lowVal, pointRange.LowVal...) + lowVal = append(lowVal, suffixRange.LowVal...) + + highVal := make([]types.Datum, 0, len(pointRange.HighVal)+len(suffixRange.HighVal)) + highVal = append(highVal, pointRange.HighVal...) + highVal = append(highVal, suffixRange.HighVal...) + + collators := make([]collate.Collator, 0, len(pointRange.Collators)+len(suffixRange.Collators)) + collators = append(collators, pointRange.Collators...) + collators = append(collators, suffixRange.Collators...) + + return &Range{ + LowVal: lowVal, + LowExclude: suffixRange.LowExclude, + HighVal: highVal, + HighExclude: suffixRange.HighExclude, + Collators: collators, + } +} + +// AppendRanges2PointRanges appends additional ranges to point ranges. // rangeMaxSize is the max memory limit for ranges. O indicates no memory limit. // If the second return value is true, it means that the estimated memory after appending additional ranges to point ranges // exceeds rangeMaxSize and the function rejects appending additional ranges to point ranges. -func appendRanges2PointRanges(pointRanges Ranges, ranges Ranges, rangeMaxSize int64) (Ranges, bool) { +func AppendRanges2PointRanges(pointRanges Ranges, ranges Ranges, rangeMaxSize int64) (Ranges, bool) { if len(ranges) == 0 { return pointRanges, false } @@ -349,26 +372,7 @@ func appendRanges2PointRanges(pointRanges Ranges, ranges Ranges, rangeMaxSize in newRanges := make(Ranges, 0, len(pointRanges)*len(ranges)) for _, pointRange := range pointRanges { for _, r := range ranges { - lowVal := make([]types.Datum, 0, len(pointRange.LowVal)+len(r.LowVal)) - lowVal = append(lowVal, pointRange.LowVal...) - lowVal = append(lowVal, r.LowVal...) - - highVal := make([]types.Datum, 0, len(pointRange.HighVal)+len(r.HighVal)) - highVal = append(highVal, pointRange.HighVal...) - highVal = append(highVal, r.HighVal...) - - collators := make([]collate.Collator, 0, len(pointRange.Collators)+len(r.Collators)) - collators = append(collators, pointRange.Collators...) - collators = append(collators, r.Collators...) - - newRange := &Range{ - LowVal: lowVal, - LowExclude: r.LowExclude, - HighVal: highVal, - HighExclude: r.HighExclude, - Collators: collators, - } - newRanges = append(newRanges, newRange) + newRanges = append(newRanges, appendRange2PointRange(pointRange, r)) } } return newRanges, false diff --git a/util/ranger/types.go b/util/ranger/types.go index 7a440a3eb68aa..d0beaa1c19a1d 100644 --- a/util/ranger/types.go +++ b/util/ranger/types.go @@ -33,7 +33,7 @@ import ( // It's mainly designed for plan-cache, since some ranges in a cached plan have to be rebuild when reusing. type MutableRanges interface { // Range returns the underlying range values. - Range() []*Range + Range() Ranges // Rebuild rebuilds the underlying ranges again. Rebuild() error } @@ -42,7 +42,7 @@ type MutableRanges interface { type Ranges []*Range // Range returns the range array. -func (rs Ranges) Range() []*Range { +func (rs Ranges) Range() Ranges { return rs } @@ -231,12 +231,12 @@ const EmptyRangeSize = int64(unsafe.Sizeof(Range{})) // MemUsage gets the memory usage of range. func (ran *Range) MemUsage() (sum int64) { // 16 is the size of Collator interface. - sum = EmptyRangeSize + int64(cap(ran.LowVal))*types.EmptyDatumSize + int64(cap(ran.HighVal))*types.EmptyDatumSize + int64(cap(ran.Collators))*16 + sum = EmptyRangeSize + int64(len(ran.Collators))*16 for _, val := range ran.LowVal { - sum += val.MemUsage() - types.EmptyDatumSize + sum += val.MemUsage() } for _, val := range ran.HighVal { - sum += val.MemUsage() - types.EmptyDatumSize + sum += val.MemUsage() } // We ignore size of collator currently. return sum