diff --git a/pkg/planner/cardinality/testdata/cardinality_suite_out.json b/pkg/planner/cardinality/testdata/cardinality_suite_out.json index 66c20ff191e25..2216f049f4470 100644 --- a/pkg/planner/cardinality/testdata/cardinality_suite_out.json +++ b/pkg/planner/cardinality/testdata/cardinality_suite_out.json @@ -1483,12 +1483,24 @@ "expr": "((a < 5)) or ((a > 10 and true))", "row_count": 6 }, + { + "table_name": "t", + "type": "Index Stats-Range", + "expr": "((a < 5))", + "row_count": 6 + }, { "table_name": "t", "type": "Index Stats-Range", "expr": "((a < 5)) or ((a > 10 and true))", "row_count": 6 }, + { + "table_name": "t", + "type": "Index Stats-Range", + "expr": "((a > 10 and true))", + "row_count": 1 + }, { "table_name": "t", "type": "Table Stats-Expression-CNF", diff --git a/pkg/planner/core/casetest/hint/testdata/integration_suite_out.json b/pkg/planner/core/casetest/hint/testdata/integration_suite_out.json index 70ebc16d86c26..9ef9992c1e00d 100644 --- a/pkg/planner/core/casetest/hint/testdata/integration_suite_out.json +++ b/pkg/planner/core/casetest/hint/testdata/integration_suite_out.json @@ -128,12 +128,12 @@ "Plan": [ "PartitionUnion 33.00 root ", "├─IndexMerge 11.00 root type: union", - "│ ├─TableRangeScan(Build) 1.00 cop[tikv] table:t, partition:p0 range:[1,1], keep order:false, stats:pseudo", "│ ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, partition:p0, index:b(b) range:[2,2], keep order:false, stats:pseudo", + "│ ├─TableRangeScan(Build) 1.00 cop[tikv] table:t, partition:p0 range:[1,1], keep order:false, stats:pseudo", "│ └─TableRowIDScan(Probe) 11.00 cop[tikv] table:t, partition:p0 keep order:false, stats:pseudo", "├─IndexMerge 11.00 root type: union", - "│ ├─TableRangeScan(Build) 1.00 cop[tikv] table:t, partition:p1 range:[1,1], keep order:false, stats:pseudo", "│ ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, partition:p1, index:b(b) range:[2,2], keep order:false, stats:pseudo", + "│ ├─TableRangeScan(Build) 1.00 cop[tikv] table:t, partition:p1 range:[1,1], keep order:false, stats:pseudo", "│ └─TableRowIDScan(Probe) 11.00 cop[tikv] table:t, partition:p1 keep order:false, stats:pseudo", "└─TableReader 11.00 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 11.00 mpp[tiflash] ExchangeType: PassThrough", diff --git a/pkg/planner/core/find_best_task.go b/pkg/planner/core/find_best_task.go index b85c2f457b698..e4a1056bfde2f 100644 --- a/pkg/planner/core/find_best_task.go +++ b/pkg/planner/core/find_best_task.go @@ -945,7 +945,7 @@ func matchPropForIndexMergeAlternatives(ds *logicalop.DataSource, path *util.Acc // if matchIdxes greater than 1, we should sort this match alternative path by its CountAfterAccess. alternatives := oneORBranch slices.SortStableFunc(matchIdxes, func(a, b int) int { - res := cmpAlternativesByRowCount(alternatives[a], alternatives[b]) + res := cmpAlternatives(ds.SCtx().GetSessionVars())(alternatives[a], alternatives[b]) if res != 0 { return res } @@ -1033,27 +1033,6 @@ func matchPropForIndexMergeAlternatives(ds *logicalop.DataSource, path *util.Acc break } } - pushDownCtx := util.GetPushDownCtx(ds.SCtx()) - for _, path := range determinedIndexPartialPaths { - // If any partial path contains table filters, we need to keep the whole DNF filter in the Selection. - if len(path.TableFilters) > 0 { - if !expression.CanExprsPushDown(pushDownCtx, path.TableFilters, kv.TiKV) { - // if this table filters can't be pushed down, all of them should be kept in the table side, cleaning the lookup side here. - path.TableFilters = nil - } - shouldKeepCurrentFilter = true - } - // If any partial path's index filter cannot be pushed to TiKV, we should keep the whole DNF filter. - if len(path.IndexFilters) != 0 && !expression.CanExprsPushDown(pushDownCtx, path.IndexFilters, kv.TiKV) { - shouldKeepCurrentFilter = true - // Clear IndexFilter, the whole filter will be put in indexMergePath.TableFilters. - path.IndexFilters = nil - } - } - // Keep this filter as a part of table filters for safety if it has any parameter. - if expression.MaybeOverOptimized4PlanCache(ds.SCtx().GetExprCtx(), []expression.Expression{path.IndexMergeORSourceFilter}) { - shouldKeepCurrentFilter = true - } if shouldKeepCurrentFilter { // add the cnf expression back as table filer. indexMergePath.TableFilters = append(indexMergePath.TableFilters, path.IndexMergeORSourceFilter) diff --git a/pkg/planner/core/indexmerge_path.go b/pkg/planner/core/indexmerge_path.go index 14633582328a1..64e2174ef86dc 100644 --- a/pkg/planner/core/indexmerge_path.go +++ b/pkg/planner/core/indexmerge_path.go @@ -16,7 +16,6 @@ package core import ( "cmp" - "math" "slices" "strings" @@ -81,15 +80,20 @@ func generateIndexMergePath(ds *logicalop.DataSource) error { } regularPathCount := len(ds.PossibleAccessPaths) + + // Now we have 3 entry functions to generate IndexMerge paths: + // 1. Generate AND type IndexMerge for non-MV indexes and all OR type IndexMerge. var err error - if warningMsg, err = generateIndexMerge4NormalIndex(ds, regularPathCount, indexMergeConds); err != nil { + if warningMsg, err = generateOtherIndexMerge(ds, regularPathCount, indexMergeConds); err != nil { return err } - if err := generateIndexMerge4MVIndex(ds, regularPathCount, indexMergeConds); err != nil { + // 2. Generate AND type IndexMerge for MV indexes. Tt can only use one index in an IndexMerge path. + if err := generateANDIndexMerge4MVIndex(ds, regularPathCount, indexMergeConds); err != nil { return err } oldIndexMergeCount := len(ds.PossibleAccessPaths) - if err := generateIndexMerge4ComposedIndex(ds, regularPathCount, indexMergeConds); err != nil { + // 3. Generate AND type IndexMerge for MV indexes. It can use multiple MV and non-MV indexes in an IndexMerge path. + if err := generateANDIndexMerge4ComposedIndex(ds, regularPathCount, indexMergeConds); err != nil { return err } @@ -148,14 +152,7 @@ func generateNormalIndexPartialPath( needSelection = true } } - itemPaths := accessPathsForConds(ds, pushedDownCNFItems, []*util.AccessPath{candidatePath}) - if len(itemPaths) != 1 { - // for this dnf item, we couldn't generate an index merge partial path. - // (1 member of (a)) or (3 member of (b)) or d=1; if one dnf item like d=1 here could walk index path, - // the entire index merge is not valid anymore. - return nil, false - } - partialPath := buildIndexMergePartialPath(itemPaths) + partialPath := accessPathsForConds(ds, pushedDownCNFItems, candidatePath) if partialPath == nil { // for this dnf item, we couldn't generate an index merge partial path. // (1 member of (a)) or (3 member of (b)) or d=1; if one dnf item like d=1 here could walk index path, @@ -163,128 +160,9 @@ func generateNormalIndexPartialPath( return nil, false } - // identify whether all pushedDownCNFItems are fully used. - // If any partial path contains table filters, we need to keep the whole DNF filter in the Selection. - if len(partialPath.TableFilters) > 0 { - needSelection = true - partialPath.TableFilters = nil - } - // If any partial path's index filter cannot be pushed to TiKV, we should keep the whole DNF filter. - if len(partialPath.IndexFilters) != 0 && !expression.CanExprsPushDown(pushDownCtx, partialPath.IndexFilters, kv.TiKV) { - needSelection = true - // Clear IndexFilter, the whole filter will be put in indexMergePath.TableFilters. - partialPath.IndexFilters = nil - } - // Keep this filter as a part of table filters for safety if it has any parameter. - if expression.MaybeOverOptimized4PlanCache(ds.SCtx().GetExprCtx(), cnfItems) { - needSelection = true - } - return partialPath, needSelection } -// getIndexMergeOrPath generates all possible IndexMergeOrPaths. -// For index merge union case, the order property from its partial -// path can be kept and multi-way merged and output. So we don't -// generate a concrete index merge path out, but an un-determined -// alternatives set index merge path instead. -// -// `create table t (a int, b int, c int, key a(a), key b(b), key ac(a, c), key bc(b, c))` -// `explain format='verbose' select * from t where a=1 or b=1 order by c` -// -// like the case here: -// normal index merge OR path should be: -// for a=1, it has two partial alternative paths: [a, ac] -// for b=1, it has two partial alternative paths: [b, bc] -// and the index merge path: -// -// indexMergePath: { -// PartialIndexPaths: empty // 1D array here, currently is not decided yet. -// PartialAlternativeIndexPaths: [[a, ac], [b, bc]] // 2D array here, each for one DNF item choices. -// } -func generateIndexMergeOrPaths(ds *logicalop.DataSource, filters []expression.Expression) error { - usedIndexCount := len(ds.PossibleAccessPaths) - pushDownCtx := util.GetPushDownCtx(ds.SCtx()) - for k, cond := range filters { - sf, ok := cond.(*expression.ScalarFunction) - if !ok || sf.FuncName.L != ast.LogicOr { - continue - } - // shouldKeepCurrentFilter means the partial paths can't cover the current filter completely, so we must add - // the current filter into a Selection after partial paths. - shouldKeepCurrentFilter := false - var partialAlternativePaths = make([][]*util.AccessPath, 0, usedIndexCount) - dnfItems := expression.FlattenDNFConditions(sf) - for _, item := range dnfItems { - cnfItems := expression.SplitCNFItems(item) - - pushedDownCNFItems := make([]expression.Expression, 0, len(cnfItems)) - for _, cnfItem := range cnfItems { - if expression.CanExprsPushDown(pushDownCtx, []expression.Expression{cnfItem}, kv.TiKV) { - pushedDownCNFItems = append(pushedDownCNFItems, cnfItem) - } else { - shouldKeepCurrentFilter = true - } - } - - itemPaths := accessPathsForConds(ds, pushedDownCNFItems, ds.PossibleAccessPaths[:usedIndexCount]) - if len(itemPaths) == 0 { - partialAlternativePaths = nil - break - } - // we don't prune other possible index merge path here. - // keep all the possible index merge partial paths here to let the property choose. - partialAlternativePaths = append(partialAlternativePaths, itemPaths) - } - if len(partialAlternativePaths) <= 1 { - continue - } - // in this loop we do two things. - // 1: If all the partialPaths use the same index, we will not use the indexMerge. - // 2: Compute a theoretical best countAfterAccess(pick its accessConds) for every alternative path(s). - indexMap := make(map[int64]struct{}, 1) - accessConds := make([]expression.Expression, 0, len(partialAlternativePaths)) - for _, oneAlternativeSet := range partialAlternativePaths { - // 1: mark used map. - for _, oneAlternativePath := range oneAlternativeSet { - if oneAlternativePath.IsTablePath() { - // table path - indexMap[-1] = struct{}{} - } else { - // index path - indexMap[oneAlternativePath.Index.ID] = struct{}{} - } - } - // 2.1: trade off on countAfterAccess. - minCountAfterAccessPath := buildIndexMergePartialPath(oneAlternativeSet) - indexCondsForP := minCountAfterAccessPath.AccessConds[:] - indexCondsForP = append(indexCondsForP, minCountAfterAccessPath.IndexFilters...) - if len(indexCondsForP) > 0 { - accessConds = append(accessConds, expression.ComposeCNFCondition(ds.SCtx().GetExprCtx(), indexCondsForP...)) - } - } - if len(indexMap) == 1 { - continue - } - // 2.2 get the theoretical whole count after access for index merge. - accessDNF := expression.ComposeDNFCondition(ds.SCtx().GetExprCtx(), accessConds...) - sel, _, err := cardinality.Selectivity(ds.SCtx(), ds.TableStats.HistColl, []expression.Expression{accessDNF}, nil) - if err != nil { - logutil.BgLogger().Debug("something wrong happened, use the default selectivity", zap.Error(err)) - sel = cost.SelectionFactor - } - - possiblePath := buildIndexMergeOrPath(filters, partialAlternativePaths, k, shouldKeepCurrentFilter) - if possiblePath == nil { - return nil - } - possiblePath.CountAfterAccess = sel * ds.TableStats.RowCount - // only after all partial path is determined, can the countAfterAccess be done, delay it to converging. - ds.PossibleAccessPaths = append(ds.PossibleAccessPaths, possiblePath) - } - return nil -} - // isInIndexMergeHints returns true if the input index name is not excluded by the IndexMerge hints, which means either // (1) there's no IndexMerge hint, (2) there's IndexMerge hint but no specified index names, or (3) the input index // name is specified in the IndexMerge hints. @@ -334,136 +212,66 @@ func isSpecifiedInIndexMergeHints(ds *logicalop.DataSource, name string) bool { return false } -// accessPathsForConds generates all possible index paths for conditions. +// accessPathsForConds generates an AccessPath for given candidate access path and filters. func accessPathsForConds( ds *logicalop.DataSource, conditions []expression.Expression, - candidatePaths []*util.AccessPath, -) []*util.AccessPath { - var results = make([]*util.AccessPath, 0, len(candidatePaths)) - for _, path := range candidatePaths { - newPath := &util.AccessPath{} - if path.IsTablePath() { - if !isInIndexMergeHints(ds, "primary") { - continue - } - if ds.TableInfo.IsCommonHandle { - newPath.IsCommonHandlePath = true - newPath.Index = path.Index - } else { - newPath.IsIntHandlePath = true - } - err := deriveTablePathStats(ds, newPath, conditions, true) - if err != nil { - logutil.BgLogger().Debug("can not derive statistics of a path", zap.Error(err)) - continue - } - var unsignedIntHandle bool - if newPath.IsIntHandlePath && ds.TableInfo.PKIsHandle { - if pkColInfo := ds.TableInfo.GetPkColInfo(); pkColInfo != nil { - unsignedIntHandle = mysql.HasUnsignedFlag(pkColInfo.GetFlag()) - } - } - // If the newPath contains a full range, ignore it. - if ranger.HasFullRange(newPath.Ranges, unsignedIntHandle) { - continue - } - // If we have point or empty range, just remove other possible paths. - if len(newPath.Ranges) == 0 || newPath.OnlyPointRange(ds.SCtx().GetSessionVars().StmtCtx.TypeCtx()) { - if len(results) == 0 { - results = append(results, newPath) - } else { - results[0] = newPath - results = results[:1] - } - break - } - } else { + path *util.AccessPath, +) *util.AccessPath { + newPath := &util.AccessPath{} + if path.IsTablePath() { + if !isInIndexMergeHints(ds, "primary") { + return nil + } + if ds.TableInfo.IsCommonHandle { + newPath.IsCommonHandlePath = true newPath.Index = path.Index - if !isInIndexMergeHints(ds, newPath.Index.Name.L) { - continue - } - err := fillIndexPath(ds, newPath, conditions) - if err != nil { - logutil.BgLogger().Debug("can not derive statistics of a path", zap.Error(err)) - continue - } - deriveIndexPathStats(ds, newPath, conditions, true) - // If the newPath contains a full range, ignore it. - if ranger.HasFullRange(newPath.Ranges, false) { - continue - } - // If we have empty range, or point range on unique index, just remove other possible paths. - if len(newPath.Ranges) == 0 || (newPath.OnlyPointRange(ds.SCtx().GetSessionVars().StmtCtx.TypeCtx()) && newPath.Index.Unique) { - if len(results) == 0 { - results = append(results, newPath) - } else { - results[0] = newPath - results = results[:1] - } - break + } else { + newPath.IsIntHandlePath = true + } + err := deriveTablePathStats(ds, newPath, conditions, true) + if err != nil { + logutil.BgLogger().Debug("can not derive statistics of a path", zap.Error(err)) + return nil + } + var unsignedIntHandle bool + if newPath.IsIntHandlePath && ds.TableInfo.PKIsHandle { + if pkColInfo := ds.TableInfo.GetPkColInfo(); pkColInfo != nil { + unsignedIntHandle = mysql.HasUnsignedFlag(pkColInfo.GetFlag()) } } - results = append(results, newPath) - } - return results -} - -// buildIndexMergePartialPath chooses the best index path from all possible paths. -// Now we choose the index with minimal estimate row count. -func buildIndexMergePartialPath(indexAccessPaths []*util.AccessPath) *util.AccessPath { - if len(indexAccessPaths) == 1 { - return indexAccessPaths[0] - } - - minEstRowIndex := 0 - minEstRow := math.MaxFloat64 - for i := 0; i < len(indexAccessPaths); i++ { - rc := indexAccessPaths[i].CountAfterAccess - if len(indexAccessPaths[i].IndexFilters) > 0 { - rc = indexAccessPaths[i].CountAfterIndex + // If the newPath contains a full range, ignore it. + if ranger.HasFullRange(newPath.Ranges, unsignedIntHandle) { + return nil + } + } else { + newPath.Index = path.Index + if !isInIndexMergeHints(ds, newPath.Index.Name.L) { + return nil } - if rc < minEstRow { - minEstRowIndex = i - minEstRow = rc + err := fillIndexPath(ds, newPath, conditions) + if err != nil { + logutil.BgLogger().Debug("can not derive statistics of a path", zap.Error(err)) + return nil + } + deriveIndexPathStats(ds, newPath, conditions, true) + // If the newPath contains a full range, ignore it. + if ranger.HasFullRange(newPath.Ranges, false) { + return nil } } - return indexAccessPaths[minEstRowIndex] -} - -// buildIndexMergeOrPath generates one possible IndexMergePath. -func buildIndexMergeOrPath( - filters []expression.Expression, - partialAlternativePaths [][]*util.AccessPath, - current int, - shouldKeepCurrentFilter bool, -) *util.AccessPath { - tmp := make([][][]*util.AccessPath, len(partialAlternativePaths)) - for i, orBranch := range partialAlternativePaths { - tmp[i] = make([][]*util.AccessPath, len(orBranch)) - for j, alternative := range orBranch { - tmp[i][j] = []*util.AccessPath{alternative} - } - } - indexMergePath := &util.AccessPath{PartialAlternativeIndexPaths: tmp} - indexMergePath.TableFilters = append(indexMergePath.TableFilters, filters[:current]...) - indexMergePath.TableFilters = append(indexMergePath.TableFilters, filters[current+1:]...) - // since shouldKeepCurrentFilter may be changed in alternative paths converging, kept the filer expression anyway here. - indexMergePath.KeepIndexMergeORSourceFilter = shouldKeepCurrentFilter - // this filter will be merged into indexPath's table filters when converging. - indexMergePath.IndexMergeORSourceFilter = filters[current] - return indexMergePath + return newPath } func generateNormalIndexPartialPath4And(ds *logicalop.DataSource, normalPathCnt int, usedAccessMap map[string]expression.Expression) []*util.AccessPath { - if res := generateIndexMergeAndPaths(ds, normalPathCnt, usedAccessMap); res != nil { + if res := generateANDIndexMerge4NormalIndex(ds, normalPathCnt, usedAccessMap); res != nil { return res.PartialIndexPaths } return nil } -// generateIndexMergeAndPaths generates IndexMerge paths for `AND` (a.k.a. intersection type IndexMerge) -func generateIndexMergeAndPaths(ds *logicalop.DataSource, normalPathCnt int, usedAccessMap map[string]expression.Expression) *util.AccessPath { +// generateANDIndexMerge4NormalIndex generates IndexMerge paths for `AND` (a.k.a. intersection type IndexMerge) +func generateANDIndexMerge4NormalIndex(ds *logicalop.DataSource, normalPathCnt int, usedAccessMap map[string]expression.Expression) *util.AccessPath { // For now, we only consider intersection type IndexMerge when the index names are specified in the hints. if !indexMergeHintsHasSpecifiedIdx(ds) { return nil @@ -704,7 +512,9 @@ func generateMVIndexMergePartialPaths4And(ds *logicalop.DataSource, normalPathCn return mvAndPartialPath, usedAccessCondsMap, nil } -func generateIndexMerge4NormalIndex(ds *logicalop.DataSource, regularPathCount int, indexMergeConds []expression.Expression) (string, error) { +// generateOtherIndexMerge is the entry point for generateORIndexMerge() and generateANDIndexMerge4NormalIndex(), plus +// some extra logic to keep some specific behaviors the same as before. +func generateOtherIndexMerge(ds *logicalop.DataSource, regularPathCount int, indexMergeConds []expression.Expression) (string, error) { isPossibleIdxMerge := len(indexMergeConds) > 0 && // have corresponding access conditions, and len(ds.PossibleAccessPaths) > 1 // have multiple index paths if !isPossibleIdxMerge { @@ -744,77 +554,44 @@ func generateIndexMerge4NormalIndex(ds *logicalop.DataSource, regularPathCount i } } - if !needConsiderIndexMerge { - return "IndexMerge is inapplicable or disabled. ", nil // IndexMerge is inapplicable - } - // 1. Generate possible IndexMerge paths for `OR`. - err := generateIndexMergeOrPaths(ds, indexMergeConds) + err := generateORIndexMerge(ds, indexMergeConds) if err != nil { return "", err } // 2. Generate possible IndexMerge paths for `AND`. - indexMergeAndPath := generateIndexMergeAndPaths(ds, regularPathCount, nil) + indexMergeAndPath := generateANDIndexMerge4NormalIndex(ds, regularPathCount, nil) if indexMergeAndPath != nil { ds.PossibleAccessPaths = append(ds.PossibleAccessPaths, indexMergeAndPath) } - return "", nil -} - -// generateIndexMergeOnDNF4MVIndex generates IndexMerge paths for MVIndex upon DNF filters. -/* - select * from t where ((1 member of (a) and b=1) or (2 member of (a) and b=2)) and (c > 10) - IndexMerge(OR) - IndexRangeScan(a, b, [1 1, 1 1]) - IndexRangeScan(a, b, [2 2, 2 2]) - Selection(c > 10) - TableRowIdScan(t) - Two limitations now: - 1). all filters in the DNF have to be used as access-filters: ((1 member of (a)) or (2 member of (a)) or b > 10) cannot be used to access the MVIndex. - 2). cannot support json_contains: (json_contains(a, '[1, 2]') or json_contains(a, '[3, 4]')) is not supported since a single IndexMerge cannot represent this SQL. -*/ -func generateIndexMergeOnDNF4MVIndex(ds *logicalop.DataSource, normalPathCnt int, filters []expression.Expression) (mvIndexPaths []*util.AccessPath, err error) { - for idx := 0; idx < normalPathCnt; idx++ { - if !isMVIndexPath(ds.PossibleAccessPaths[idx]) { - continue // not a MVIndex path - } - // for single MV index usage, if specified use the specified one, if not, all can be access and chosen by cost model. - if !isInIndexMergeHints(ds, ds.PossibleAccessPaths[idx].Index.Name.L) { - continue - } + if needConsiderIndexMerge { + return "", nil + } - for current, filter := range filters { - sf, ok := filter.(*expression.ScalarFunction) - if !ok || sf.FuncName.L != ast.LogicOr { - continue + var containMVPath bool + for i := regularPathCount; i < len(ds.PossibleAccessPaths); i++ { + path := ds.PossibleAccessPaths[i] + for _, p := range util.SliceRecursiveFlattenIter[*util.AccessPath](path.PartialAlternativeIndexPaths) { + if isMVIndexPath(p) { + containMVPath = true + break } - dnfFilters := expression.SplitDNFItems(sf) // [(1 member of (a) and b=1), (2 member of (a) and b=2)] + } - unfinishedIndexMergePath := genUnfinishedPathFromORList( - ds, - dnfFilters, - []*util.AccessPath{ds.PossibleAccessPaths[idx]}, - ) - finishedIndexMergePath := handleTopLevelANDList( - ds, - filters, - current, - []*util.AccessPath{ds.PossibleAccessPaths[idx]}, - unfinishedIndexMergePath, - ) - if finishedIndexMergePath != nil { - mvIndexPaths = append(mvIndexPaths, finishedIndexMergePath) - } + if !containMVPath { + ds.PossibleAccessPaths = slices.Delete(ds.PossibleAccessPaths, i, i+1) } } - return + if len(ds.PossibleAccessPaths) == regularPathCount { + return "IndexMerge is inapplicable or disabled. ", nil + } + return "", nil } -// generateIndexMerge4ComposedIndex generates index path composed of multi indexes including multivalued index from -// (json_member_of / json_overlaps / json_contains) and single-valued index from normal indexes. +// generateANDIndexMerge4ComposedIndex tries to generate AND type index merge AccessPath for ( json_member_of / +// json_overlaps / json_contains) on multiple multi-valued or normal indexes. /* -CNF path 1. select * from t where ((1 member of (a) and c=1) and (2 member of (b) and d=2) and (other index predicates)) flatten as: select * from t where 1 member of (a) and 2 member of (b) and c=1 and c=2 and other index predicates analyze: find and utilize index access filter items as much as possible: @@ -846,37 +623,8 @@ CNF path IndexRangeScan(a, [3,3]) --- COP IndexRangeScan(non-mv-index-if-any)(?) --- COP TableRowIdScan(t) --- COP -DNF path - Note: in DNF pattern, every dnf item should be utilized as index path with full range or prefix range. Otherwise,index merge is invalid. - 1. select * from t where ((1 member of (a)) or (2 member of (b)) or (other index predicates)) - analyze: find and utilize index access filter items as much as possible: - IndexMerge(OR-UNION) --- ROOT - IndexRangeScan(mv-index-a)(1) --- COP ---> simplified from member-of index merge, defer TableRowIdScan(t) to the outer index merge. - IndexRangeScan(mv-index-b)(2) --- COP - IndexRangeScan(non-mv-index-if-any)(?) --- COP - TableRowIdScan(t) --- COP - 2. select * from t where ((1 member of (a)) or (json_contains(b, '[1, 2, 3]')) or (other index predicates)) - analyze: find and utilize index access filter items as much as possible: - IndexMerge(OR-UNION) --- ROOT - IndexRangeScan(mv-index-a)(1) --- COP - IndexMerge(mv-index-b AND-INTERSECTION) --- ROOT (embedded index merge) ---> can't be simplified - IndexRangeScan(a, [1,1]) --- COP - IndexRangeScan(a, [2,2]) --- COP - IndexRangeScan(a, [3,3]) --- COP - IndexRangeScan(non-mv-index-if-any)(?) --- COP - TableRowIdScan(t) --- COP - 3. select * from t where ((1 member of (a) and c=1) or (json_overlap(a, '[1, 2,3]') and d=2) or (other index predicates) - analyze: find and utilize index access filter items as much as possible: - IndexMerge(OR-UNION) --- ROOT - IndexRangeScan(mv-index-a)(1) --- COP - IndexMerge(mv-index-b OR-UNION) --- ROOT (embedded index merge) ---> simplify: we can merge with outer index union merge, and defer TableRowIdScan(t). - IndexRangeScan(a, [1,1]) --- COP - IndexRangeScan(a, [2,2]) --- COP - IndexRangeScan(a, [3,3]) --- COP - IndexRangeScan(non-mv-index-if-any)(?) --- COP - TableRowIdScan(t) --- COP */ -func generateIndexMerge4ComposedIndex(ds *logicalop.DataSource, normalPathCnt int, indexMergeConds []expression.Expression) error { +func generateANDIndexMerge4ComposedIndex(ds *logicalop.DataSource, normalPathCnt int, indexMergeConds []expression.Expression) error { isPossibleIdxMerge := len(indexMergeConds) > 0 && // have corresponding access conditions, and len(ds.PossibleAccessPaths) > 1 // have multiple index paths if !isPossibleIdxMerge { @@ -886,7 +634,7 @@ func generateIndexMerge4ComposedIndex(ds *logicalop.DataSource, normalPathCnt in // Collect access paths that satisfy the hints, and make sure there is at least one MV index path. var mvIndexPathCnt int candidateAccessPaths := make([]*util.AccessPath, 0, len(ds.PossibleAccessPaths)) - for idx := 0; idx < normalPathCnt; idx++ { + for idx := range normalPathCnt { if (ds.PossibleAccessPaths[idx].IsTablePath() && !isInIndexMergeHints(ds, "primary")) || (!ds.PossibleAccessPaths[idx].IsTablePath() && @@ -902,53 +650,6 @@ func generateIndexMerge4ComposedIndex(ds *logicalop.DataSource, normalPathCnt in return nil } - for current, filter := range indexMergeConds { - // DNF path. - sf, ok := filter.(*expression.ScalarFunction) - if !ok || sf.FuncName.L != ast.LogicOr { - // targeting: cond1 or cond2 or cond3 - continue - } - dnfFilters := expression.SplitDNFItems(sf) - - unfinishedIndexMergePath := genUnfinishedPathFromORList( - ds, - dnfFilters, - candidateAccessPaths, - ) - finishedIndexMergePath := handleTopLevelANDList( - ds, - indexMergeConds, - current, - candidateAccessPaths, - unfinishedIndexMergePath, - ) - if finishedIndexMergePath == nil { - return nil - } - - var mvIndexPartialPathCnt, normalIndexPartialPathCnt int - for _, oneAlternative := range finishedIndexMergePath.PartialAlternativeIndexPaths { - for _, paths := range oneAlternative { - for _, path := range paths { - if isMVIndexPath(path) { - mvIndexPartialPathCnt++ - } else { - normalIndexPartialPathCnt++ - } - } - } - } - - // Keep the same behavior with previous implementation, we only handle the "composed" case here. - if mvIndexPartialPathCnt == 0 || (mvIndexPartialPathCnt == 1 && normalIndexPartialPathCnt == 0) { - return nil - } - ds.PossibleAccessPaths = append(ds.PossibleAccessPaths, finishedIndexMergePath) - return nil - } - // CNF path. - // after fillIndexPath, all cnf items are filled into the suitable index paths, for these normal index paths, // fetch them out as partialIndexPaths of a outerScope index merge. // note that: @@ -1018,7 +719,8 @@ func generateIndexMerge4ComposedIndex(ds *logicalop.DataSource, normalPathCnt in return nil } -// generateIndexMerge4MVIndex generates paths for (json_member_of / json_overlaps / json_contains) on multi-valued index. +// generateANDIndexMerge4MVIndex tries to generate AND type index merge AccessPath for ( json_member_of / +// json_overlaps / json_contains) on a single multi-valued index. /* 1. select * from t where 1 member of (a) IndexMerge(AND) @@ -1030,20 +732,8 @@ func generateIndexMerge4ComposedIndex(ds *logicalop.DataSource, normalPathCnt in IndexRangeScan(a, [2,2]) IndexRangeScan(a, [3,3]) TableRowIdScan(t) - 3. select * from t where json_overlap(a, '[1, 2, 3]') - IndexMerge(OR) - IndexRangeScan(a, [1,1]) - IndexRangeScan(a, [2,2]) - IndexRangeScan(a, [3,3]) - TableRowIdScan(t) */ -func generateIndexMerge4MVIndex(ds *logicalop.DataSource, normalPathCnt int, filters []expression.Expression) error { - dnfMVIndexPaths, err := generateIndexMergeOnDNF4MVIndex(ds, normalPathCnt, filters) - if err != nil { - return err - } - ds.PossibleAccessPaths = append(ds.PossibleAccessPaths, dnfMVIndexPaths...) - +func generateANDIndexMerge4MVIndex(ds *logicalop.DataSource, normalPathCnt int, filters []expression.Expression) error { for idx := 0; idx < normalPathCnt; idx++ { if !isMVIndexPath(ds.PossibleAccessPaths[idx]) { continue // not a MVIndex path @@ -1379,7 +1069,7 @@ func collectFilters4MVIndex( // 2: `x=1 and x=2 and (1 member of a) and z=1`, remaining: `x+z>0`. // Note: x=1 and x=2 will derive an invalid range in ranger detach, for now because of heuristic rule above, we ignore this case here. // -// just as the 3rd point as we said in generateIndexMerge4ComposedIndex +// just as the 3rd point as we said in generateANDIndexMerge4ComposedIndex // // 3: The predicate of mv index can not converge to a linear interval range at physical phase like EQ and // GT in normal index. Among the predicates in mv index (member-of/contains/overlap), multi conditions diff --git a/pkg/planner/core/indexmerge_unfinished_path.go b/pkg/planner/core/indexmerge_unfinished_path.go index e866e2e897b03..3de6bf0d37805 100644 --- a/pkg/planner/core/indexmerge_unfinished_path.go +++ b/pkg/planner/core/indexmerge_unfinished_path.go @@ -19,15 +19,43 @@ import ( "slices" "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/planner/cardinality" "github.com/pingcap/tidb/pkg/planner/core/cost" "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop" "github.com/pingcap/tidb/pkg/planner/util" + "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/util/logutil" "go.uber.org/zap" ) +// generateORIndexMerge handles all (MV and non-MV index) OR type IndexMerge path generation. +// The input filters are implicitly connected by AND. +func generateORIndexMerge(ds *logicalop.DataSource, filters []expression.Expression) error { + usedIndexCount := len(ds.PossibleAccessPaths) + // 1. Iterate the input filters and try to find an OR list. + for k, cond := range filters { + sf, ok := cond.(*expression.ScalarFunction) + if !ok || sf.FuncName.L != ast.LogicOr { + continue + } + + dnfFilters := expression.SplitDNFItems(sf) + candidatesAccessPaths := ds.PossibleAccessPaths[:usedIndexCount] + + // 2. Try to collect usable filters for each candidate access path using the OR list. + unfinishedIndexMergePath := genUnfinishedPathFromORList(ds, dnfFilters, candidatesAccessPaths) + // 3. Try to collect more usable filters from the top level AND list and build it into a valid AccessPath. + indexMergePath := handleTopLevelANDList(ds, filters, k, candidatesAccessPaths, unfinishedIndexMergePath) + if indexMergePath != nil { + ds.PossibleAccessPaths = append(ds.PossibleAccessPaths, indexMergePath) + } + } + return nil +} + // unfinishedAccessPath collects usable filters in preparation for building an OR type IndexMerge access path. // It maintains the information during iterating all filters. Importantly, it maintains incomplete access filters, which // means they may not be able to build a valid range, but could build a valid range after collecting more access filters. @@ -67,7 +95,7 @@ type unfinishedAccessPath struct { // OR type IndexMerge access path. type unfinishedAccessPathList []*unfinishedAccessPath -// generateUnfinishedIndexMergePathFromORList handles a list of filters connected by OR, collects access filters for +// genUnfinishedPathFromORList handles a list of filters connected by OR, collects access filters for // each candidate access path, and returns an unfinishedAccessPath, which must be an index merge OR unfinished path, // each partial path of which corresponds to one filter in the input orList. /* @@ -160,6 +188,12 @@ func initUnfinishedPathsFromExpr( continue } cnfItems := expression.SplitCNFItems(expr) + pushDownCtx := util.GetPushDownCtx(ds.SCtx()) + for _, cnfItem := range cnfItems { + if !expression.CanExprsPushDown(pushDownCtx, []expression.Expression{cnfItem}, kv.TiKV) { + ret[i].needKeepFilter = true + } + } // case 2: try to use the previous logic to handle mv index if isMVIndexPath(path) { @@ -175,6 +209,7 @@ func initUnfinishedPathsFromExpr( // case 3: use the new logic if the previous logic didn't succeed to collect access filters that can build a // valid range directly. ret[i].idxColHasUsableFilter = make([]bool, len(idxCols)) + ret[i].needKeepFilter = true for j, col := range idxCols { for _, cnfItem := range cnfItems { if ok, tp := checkAccessFilter4IdxCol(ds.SCtx(), cnfItem, col); ok && @@ -350,7 +385,7 @@ func buildIntoAccessPath( if err != nil || !ok || (isIntersection && len(oneAlternative) > 1) { continue } - needSelection = len(remainingFilters) > 0 || len(unfinishedPath.idxColHasUsableFilter) > 0 + needSelection = len(remainingFilters) > 0 } else { // case 2: non-mv index var path *util.AccessPath @@ -382,20 +417,66 @@ func buildIntoAccessPath( allAlternativePaths = append(allAlternativePaths, alternativesForORBranch) } - // 2. TODO + // 2. Some extra setup and checks. + + pushDownCtx := util.GetPushDownCtx(ds.SCtx()) + possibleIdxIDs := make(map[int64]struct{}, len(allAlternativePaths)) + var containMVPath bool + // We do two things in this loop: + // 1. Clean/Set KeepIndexMergeORSourceFilter, InexFilters and TableFilters for each partial path. + // 2. Collect all index IDs and check if there is any MV index. + for _, p := range util.SliceRecursiveFlattenIter[*util.AccessPath](allAlternativePaths) { + // A partial path can handle TableFilters only if it's a table path, and the filters can be pushed to TiKV. + // Otherwise, we should clear TableFilters and set KeepIndexMergeORSourceFilter to true. + if len(p.TableFilters) > 0 { + // Note: Theoretically, we don't need to set KeepIndexMergeORSourceFilter to true if we can handle the + // TableFilters. But filters that contain non-handle columns will be unexpectedly removed in + // convertToPartialTableScan(). Not setting it to true will cause the final plan to miss those filters. + // The behavior related to convertToPartialTableScan() needs more investigation. + // Anyway, now we set it to true here, and it's also consistent with the previous implementation. + p.KeepIndexMergeORSourceFilter = true + if !expression.CanExprsPushDown(pushDownCtx, p.TableFilters, kv.TiKV) || !p.IsTablePath() { + p.TableFilters = nil + } + } + // A partial path can handle IndexFilters if the filters can be pushed to TiKV. + if len(p.IndexFilters) != 0 && !expression.CanExprsPushDown(pushDownCtx, p.IndexFilters, kv.TiKV) { + p.KeepIndexMergeORSourceFilter = true + p.IndexFilters = nil + } + + if p.IsTablePath() { + possibleIdxIDs[-1] = struct{}{} + } else { + possibleIdxIDs[p.Index.ID] = struct{}{} + } + if isMVIndexPath(p) { + containMVPath = true + } + } + + if !containMVPath && len(possibleIdxIDs) <= 1 { + return nil + } + + // Keep this filter as a part of table filters for safety if it has any parameter. + needKeepORSourceFilter := expression.MaybeOverOptimized4PlanCache(ds.SCtx().GetExprCtx(), + []expression.Expression{allConds[orListIdxInAllConds]}, + ) // 3. Build the final access path. possiblePath := &util.AccessPath{ PartialAlternativeIndexPaths: allAlternativePaths, TableFilters: slices.Delete(slices.Clone(allConds), orListIdxInAllConds, orListIdxInAllConds+1), IndexMergeORSourceFilter: allConds[orListIdxInAllConds], + KeepIndexMergeORSourceFilter: needKeepORSourceFilter, } // For estimation, we need the decided partial paths. So we use a simple heuristic to choose the partial paths by // comparing the row count just for estimation here. pathsForEstimate := make([]*util.AccessPath, 0, len(allAlternativePaths)) for _, oneORBranch := range allAlternativePaths { - pathsWithMinRowCount := slices.MinFunc(oneORBranch, cmpAlternativesByRowCount) + pathsWithMinRowCount := slices.MinFunc(oneORBranch, cmpAlternatives(ds.SCtx().GetSessionVars())) pathsForEstimate = append(pathsForEstimate, pathsWithMinRowCount...) } possiblePath.CountAfterAccess = estimateCountAfterAccessForIndexMergeOR(ds, pathsForEstimate) @@ -403,7 +484,24 @@ func buildIntoAccessPath( return possiblePath } -func cmpAlternativesByRowCount(a, b []*util.AccessPath) int { +func cmpAlternatives(sessionVars *variable.SessionVars) func(lhs, rhs []*util.AccessPath) int { + allPointOrEmptyRange := func(paths []*util.AccessPath) bool { + // Prefer the path with empty range or all point ranges. + for _, path := range paths { + // 1. It's not empty range. + if len(path.Ranges) > 0 && + // 2-1. It's not point range on table path. + ((path.IsTablePath() && + !path.OnlyPointRange(sessionVars.StmtCtx.TypeCtx())) || + // 2-2. It's not point range on unique index. + (!path.IsTablePath() && + len(path.Ranges) > 0 && + !(path.OnlyPointRange(sessionVars.StmtCtx.TypeCtx()) && path.Index.Unique))) { + return false + } + } + return true + } // If one alternative consists of multiple AccessPath, we use the maximum row count of them to compare. getMaxRowCountFromPaths := func(paths []*util.AccessPath) float64 { maxRowCount := 0.0 @@ -416,9 +514,19 @@ func cmpAlternativesByRowCount(a, b []*util.AccessPath) int { } return maxRowCount } - lhsRowCount := getMaxRowCountFromPaths(a) - rhsRowCount := getMaxRowCountFromPaths(b) - return cmp.Compare(lhsRowCount, rhsRowCount) + return func(a, b []*util.AccessPath) int { + lhsBetterRange := allPointOrEmptyRange(a) + rhsBetterRange := allPointOrEmptyRange(b) + if lhsBetterRange != rhsBetterRange { + if lhsBetterRange { + return -1 + } + return 1 + } + lhsRowCount := getMaxRowCountFromPaths(a) + rhsRowCount := getMaxRowCountFromPaths(b) + return cmp.Compare(lhsRowCount, rhsRowCount) + } } func estimateCountAfterAccessForIndexMergeOR(ds *logicalop.DataSource, decidedPartialPaths []*util.AccessPath) float64 { diff --git a/pkg/planner/util/BUILD.bazel b/pkg/planner/util/BUILD.bazel index c8ff84e38d288..5212f0e28b71b 100644 --- a/pkg/planner/util/BUILD.bazel +++ b/pkg/planner/util/BUILD.bazel @@ -33,6 +33,7 @@ go_library( "//pkg/util/codec", "//pkg/util/collate", "//pkg/util/hint", + "//pkg/util/intest", "//pkg/util/intset", "//pkg/util/ranger", "//pkg/util/size", @@ -46,9 +47,11 @@ go_test( srcs = [ "main_test.go", "path_test.go", + "slice_recursive_flatten_iter_test.go", ], embed = [":util"], flaky = True, + shard_count = 3, deps = [ "//pkg/domain", "//pkg/meta/model", diff --git a/pkg/planner/util/misc.go b/pkg/planner/util/misc.go index 00dcc934d4308..623219d151e28 100644 --- a/pkg/planner/util/misc.go +++ b/pkg/planner/util/misc.go @@ -17,7 +17,9 @@ package util import ( "encoding/binary" "fmt" + "iter" "math" + "reflect" "time" "unsafe" @@ -31,6 +33,7 @@ import ( "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" "github.com/pingcap/tidb/pkg/types" h "github.com/pingcap/tidb/pkg/util/hint" + "github.com/pingcap/tidb/pkg/util/intest" "github.com/pingcap/tidb/pkg/util/ranger" ) @@ -47,6 +50,60 @@ func SliceDeepClone[T interface{ Clone() T }](s []T) []T { return cloned } +// SliceRecursiveFlattenIter returns an iterator (iter.Seq2) that recursively iterates over all elements of an +// any-dimensional slice of any type. +// Performance note: +// For each slice, this function need to check the dynamic type before iterating over it. For each non-leaf slice, this +// function uses reflect to iterate over it. Be careful when trying to use this function in performance-critical code. +/* +Example: + paths := [][][]*AccessPath{...} + for idx, path := range SliceRecursiveFlattenIter[*AccessPath](paths) { + // path is a *AccessPath here + } +*/ +func SliceRecursiveFlattenIter[E any, T any, Slice ~[]T](s Slice) iter.Seq2[int, E] { + return func(yield func(int, E) bool) { + sliceRecursiveFlattenIterHelper(s, yield, 0) + } +} + +func sliceRecursiveFlattenIterHelper[E any, Slice any]( + s Slice, + yield func(int, E) bool, + startIdx int, +) (nextIdx int, stop bool) { + intest.AssertFunc(func() bool { + return reflect.TypeOf(s).Kind() == reflect.Slice + }) + // Case 1: Input slice is []E, which means it's already the lowest level. + if leafSlice, isLeafSlice := any(s).([]E); isLeafSlice { + idx := startIdx + for _, v := range leafSlice { + if !yield(idx, v) { + return idx + 1, true + } + idx++ + } + return idx, false + } + // Case 2: Otherwise, element of Slice is still a slice, we need to flatten it recursively. + idx := startIdx + // We have to use reflect to iterate over the slice here. + v := reflect.ValueOf(s) + for i := range v.Len() { + val := v.Index(i).Interface() + intest.AssertFunc(func() bool { + return reflect.TypeOf(val).Kind() == reflect.Slice + }) + idx, stop = sliceRecursiveFlattenIterHelper[E](val, yield, idx) + if stop { + return idx, true + } + } + return idx, false +} + // CloneFieldNames uses types.FieldName.Clone to clone a slice of types.FieldName. func CloneFieldNames(names []*types.FieldName) []*types.FieldName { if names == nil { diff --git a/pkg/planner/util/slice_recursive_flatten_iter_test.go b/pkg/planner/util/slice_recursive_flatten_iter_test.go new file mode 100644 index 0000000000000..8bc991d345a73 --- /dev/null +++ b/pkg/planner/util/slice_recursive_flatten_iter_test.go @@ -0,0 +1,130 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util_test + +import ( + "fmt" + "slices" + "testing" + + "github.com/pingcap/tidb/pkg/planner/util" + "github.com/stretchr/testify/require" +) + +type input struct { + slice [][][]string + continueVals []string + breakVal string +} + +type outputItem struct { + idx int + value string +} + +func TestSliceRecursiveFlattenIter(t *testing.T) { + var testCases = []struct { + in input + out []outputItem + }{ + { + in: input{ + slice: nil, + }, + out: []outputItem{}, + }, + { + in: input{ + slice: [][][]string{{{}, nil}, nil, {}, {{}, {}}, nil, {}, {{}}, {nil, nil}}, + }, + out: []outputItem{}, + }, + { + in: input{ + slice: [][][]string{{{"111", "", "333"}}}, + }, + out: []outputItem{ + {0, "111"}, + {1, ""}, + {2, "333"}, + }, + }, + { + in: input{ + slice: [][][]string{nil, {{"111", "", "333"}, nil, {"234"}}, nil}, + }, + out: []outputItem{ + {0, "111"}, + {1, ""}, + {2, "333"}, + {3, "234"}, + }, + }, + { + in: input{ + slice: [][][]string{{{"111", "", "333"}, {}, {"444", "555", "666"}}}, + continueVals: []string{"444"}, + }, + out: []outputItem{ + {0, "111"}, + {1, ""}, + {2, "333"}, + {4, "555"}, + {5, "666"}, + }, + }, + { + in: input{ + slice: [][][]string{{{"111", "", "333"}, nil, {"444", "555", "666"}}}, + continueVals: []string{"111"}, + breakVal: "555", + }, + out: []outputItem{ + {1, ""}, + {2, "333"}, + {3, "444"}, + }, + }, + { + in: input{ + slice: [][][]string{{nil, {"", "", "998877"}, nil, {}, {}, nil, {"321"}}, {{"555", "222", "1"}}}, + continueVals: []string{"321"}, + breakVal: "222", + }, + out: []outputItem{ + {0, ""}, + {1, ""}, + {2, "998877"}, + {4, "555"}, + }, + }, + } + + for i, tc := range testCases { + t.Run(fmt.Sprintf("Case #%d", i), func(t *testing.T) { + out := make([]outputItem, 0, len(tc.out)) + for idx, val := range util.SliceRecursiveFlattenIter[string](tc.in.slice) { + if slices.Contains(tc.in.continueVals, val) { + continue + } + if tc.in.breakVal != "" && val == tc.in.breakVal { + break + } + out = append(out, outputItem{idx, val}) + } + require.Equal(t, tc.out, out) + }) + } +} diff --git a/tests/integrationtest/r/imdbload.result b/tests/integrationtest/r/imdbload.result index e5d3dcee0727f..787c49b2e81eb 100644 --- a/tests/integrationtest/r/imdbload.result +++ b/tests/integrationtest/r/imdbload.result @@ -286,7 +286,7 @@ IndexLookUp_7 2.00 root └─TableRowIDScan_6(Probe) 2.00 cop[tikv] table:char_name keep order:false trace plan target = 'estimation' select * from char_name where ((imdb_index = 'I') and (surname_pcode < 'E436')) or ((imdb_index = 'L') and (surname_pcode < 'E436')); CE_trace -[{"table_name":"char_name","type":"Column Stats-Point","expr":"((imdb_index = 'I'))","row_count":1},{"table_name":"char_name","type":"Column Stats-Point","expr":"((imdb_index = 'L'))","row_count":1},{"table_name":"char_name","type":"Column Stats-Range","expr":"((id >= -9223372036854775808 and id <= 9223372036854775807))","row_count":4314864},{"table_name":"char_name","type":"Column Stats-Range","expr":"((surname_pcode < 'E436'))","row_count":1005030},{"table_name":"char_name","type":"Index Stats-Range","expr":"((imdb_index = 'I') and (surname_pcode < 'E436')) or ((imdb_index = 'L') and (surname_pcode < 'E436'))","row_count":2},{"table_name":"char_name","type":"Index Stats-Range","expr":"((surname_pcode < 'E436'))","row_count":1005030},{"table_name":"char_name","type":"Table Stats-Expression-CNF","expr":"`or`(`and`(`eq`(imdbload.char_name.imdb_index, 'I'), `lt`(imdbload.char_name.surname_pcode, 'E436')), `and`(`eq`(imdbload.char_name.imdb_index, 'L'), `lt`(imdbload.char_name.surname_pcode, 'E436')))","row_count":2}] +[{"table_name":"char_name","type":"Column Stats-Point","expr":"((imdb_index = 'I'))","row_count":1},{"table_name":"char_name","type":"Column Stats-Point","expr":"((imdb_index = 'L'))","row_count":1},{"table_name":"char_name","type":"Column Stats-Range","expr":"((id >= -9223372036854775808 and id <= 9223372036854775807))","row_count":4314864},{"table_name":"char_name","type":"Column Stats-Range","expr":"((surname_pcode < 'E436'))","row_count":1005030},{"table_name":"char_name","type":"Index Stats-Range","expr":"((imdb_index = 'I') and (surname_pcode < 'E436'))","row_count":1},{"table_name":"char_name","type":"Index Stats-Range","expr":"((imdb_index = 'I') and (surname_pcode < 'E436')) or ((imdb_index = 'L') and (surname_pcode < 'E436'))","row_count":2},{"table_name":"char_name","type":"Index Stats-Range","expr":"((imdb_index = 'L') and (surname_pcode < 'E436'))","row_count":1},{"table_name":"char_name","type":"Index Stats-Range","expr":"((surname_pcode < 'E436'))","row_count":1005030},{"table_name":"char_name","type":"Table Stats-Expression-CNF","expr":"`lt`(imdbload.char_name.surname_pcode, 'E436')","row_count":1005030},{"table_name":"char_name","type":"Table Stats-Expression-CNF","expr":"`or`(`and`(`eq`(imdbload.char_name.imdb_index, 'I'), `lt`(imdbload.char_name.surname_pcode, 'E436')), `and`(`eq`(imdbload.char_name.imdb_index, 'L'), `lt`(imdbload.char_name.surname_pcode, 'E436')))","row_count":2}] explain select * from char_name where ((imdb_index = 'V') and (surname_pcode < 'L3416')); id estRows task access object operator info diff --git a/tests/integrationtest/r/index_merge.result b/tests/integrationtest/r/index_merge.result index 1a842a0ef04ff..8b2c9d056dd6e 100644 --- a/tests/integrationtest/r/index_merge.result +++ b/tests/integrationtest/r/index_merge.result @@ -767,11 +767,11 @@ c1 c2 c3 c4 c5 explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and (c1 between 1 and 2) order by 1; id estRows task access object operator info Sort_5 138.56 root index_merge.t1.c1 -└─IndexMerge_12 138.56 root type: union - ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo +└─IndexMerge_12 87.26 root type: union + ├─IndexRangeScan_8(Build) 250.00 cop[tikv] table:t1, index:c1(c1) range:[1,2], keep order:false, stats:pseudo ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo - └─Selection_11(Probe) 138.56 cop[tikv] ge(index_merge.t1.c1, 1), le(index_merge.t1.c1, 2) - └─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo + └─Selection_11(Probe) 87.26 cop[tikv] ge(index_merge.t1.c1, 1), le(index_merge.t1.c1, 2) + └─TableRowIDScan_10 3490.25 cop[tikv] table:t1 keep order:false, stats:pseudo select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and (c1 between 1 and 2) order by 1; c1 c2 c3 c4 c5 1 1 1 1 1 diff --git a/tests/integrationtest/r/planner/core/casetest/index/index.result b/tests/integrationtest/r/planner/core/casetest/index/index.result index 8c0603bb9abf5..5756dc84dee9f 100644 --- a/tests/integrationtest/r/planner/core/casetest/index/index.result +++ b/tests/integrationtest/r/planner/core/casetest/index/index.result @@ -442,10 +442,10 @@ Projection 10000.00 root planner__core__casetest__index__index.tt1.c_int └─StreamAgg(Probe) 10000.00 root funcs:max(Column#14)->Column#9, funcs:sum(Column#15)->Column#10, funcs:count(1)->Column#11 └─Projection 17.91 root planner__core__casetest__index__index.tt2.c_decimal->Column#14, cast(isnull(planner__core__casetest__index__index.tt2.c_decimal), decimal(20,0) BINARY)->Column#15 └─IndexMerge 17.91 root type: union - ├─Selection(Build) 10000.00 cop[tikv] lt(7, planner__core__casetest__index__index.tt1.c_decimal) - │ └─TableRangeScan 10000.00 cop[tikv] table:tt2 range:[7,7], keep order:false, stats:pseudo ├─Selection(Build) 33333.33 cop[tikv] eq(planner__core__casetest__index__index.tt1.c_int, planner__core__casetest__index__index.tt2.c_int) │ └─IndexRangeScan 33333333.33 cop[tikv] table:tt2, index:c_str(c_str) range:["zzzzzzzzzzzzzzzzzzz",+inf], keep order:false, stats:pseudo + ├─Selection(Build) 10000.00 cop[tikv] lt(7, planner__core__casetest__index__index.tt1.c_decimal) + │ └─TableRangeScan 10000.00 cop[tikv] table:tt2 range:[7,7], keep order:false, stats:pseudo └─Selection(Probe) 17.91 cop[tikv] or(and(eq(planner__core__casetest__index__index.tt2.c_int, 7), lt(7, planner__core__casetest__index__index.tt1.c_decimal)), and(ge(planner__core__casetest__index__index.tt2.c_str, "zzzzzzzzzzzzzzzzzzz"), eq(planner__core__casetest__index__index.tt1.c_int, planner__core__casetest__index__index.tt2.c_int))) └─TableRowIDScan 43330.00 cop[tikv] table:tt2 keep order:false, stats:pseudo select c_int from tt1 where c_decimal > all (select /*+ use_index_merge(tt2) */ c_decimal from tt2 where tt2.c_int = 7 and tt2.c_int < tt1.c_decimal or tt2.c_str >= 'zzzzzzzzzzzzzzzzzzz' and tt1.c_int = tt2.c_int) order by 1; diff --git a/tests/integrationtest/r/planner/core/casetest/physicalplantest/physical_plan.result b/tests/integrationtest/r/planner/core/casetest/physicalplantest/physical_plan.result index bf67504ca0b82..d0272b3486bb4 100644 --- a/tests/integrationtest/r/planner/core/casetest/physicalplantest/physical_plan.result +++ b/tests/integrationtest/r/planner/core/casetest/physicalplantest/physical_plan.result @@ -3620,12 +3620,12 @@ Level Code Message explain format = 'brief' select * from t where (a = 1 or b = 2) and c = 3 order by c limit 2; id estRows task access object operator info TopN 0.02 root planner__core__casetest__physicalplantest__physical_plan.t.c, offset:0, count:2 -└─IndexMerge 0.02 root type: union - ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx(a, c) range:[1,1], keep order:false, stats:pseudo - ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx2(b, c) range:[2,2], keep order:false, stats:pseudo - └─TopN(Probe) 0.02 cop[tikv] planner__core__casetest__physicalplantest__physical_plan.t.c, offset:0, count:2 - └─Selection 0.02 cop[tikv] eq(planner__core__casetest__physicalplantest__physical_plan.t.c, 3) - └─TableRowIDScan 19.99 cop[tikv] table:t keep order:false, stats:pseudo +└─IndexMerge 0.20 root type: union + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:idx(a, c) range:[1 3,1 3], keep order:false, stats:pseudo + ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:idx2(b, c) range:[2 3,2 3], keep order:false, stats:pseudo + └─TopN(Probe) 0.20 cop[tikv] planner__core__casetest__physicalplantest__physical_plan.t.c, offset:0, count:2 + └─Selection 0.20 cop[tikv] eq(planner__core__casetest__physicalplantest__physical_plan.t.c, 3) + └─TableRowIDScan 0.20 cop[tikv] table:t keep order:false, stats:pseudo show warnings; Level Code Message explain format = 'brief' select * from t where (a = 1 or b = 2) and c in (1, 2, 3) order by c limit 2; @@ -3836,12 +3836,12 @@ explain format = 'brief' select min(col_304) from (select /*+ use_index_merge( t id estRows task access object operator info StreamAgg 1.00 root funcs:min(planner__core__casetest__physicalplantest__physical_plan.tbl_43.col_304)->Column#2 └─IndexMerge 1.00 root type: union, limit embedded(offset:0, count:1) - ├─Limit(Build) 0.00 cop[tikv] offset:0, count:1 - │ └─TableRangeScan 0.00 cop[tikv] table:tbl_43 range:["LUBGzGMA","LUBGzGMA"], keep order:true, stats:pseudo ├─Limit(Build) 0.33 cop[tikv] offset:0, count:1 - │ └─IndexRangeScan 0.33 cop[tikv] table:tbl_43, index:idx_261(col_304) range:[-inf,"YEpfYfPVvhMlHGHSMKm"), keep order:true, stats:pseudo + │ └─TableRangeScan 0.33 cop[tikv] table:tbl_43 range:[-inf,"YEpfYfPVvhMlHGHSMKm"), keep order:true, stats:pseudo ├─Limit(Build) 0.33 cop[tikv] offset:0, count:1 - │ └─IndexRangeScan 0.33 cop[tikv] table:tbl_43, index:idx_262(col_304) range:("PE",+inf], keep order:true, stats:pseudo + │ └─IndexRangeScan 0.33 cop[tikv] table:tbl_43, index:idx_261(col_304) range:("PE",+inf], keep order:true, stats:pseudo + ├─Limit(Build) 0.00 cop[tikv] offset:0, count:1 + │ └─IndexRangeScan 0.00 cop[tikv] table:tbl_43, index:idx_262(col_304) range:["LUBGzGMA","LUBGzGMA"], keep order:true, stats:pseudo ├─Limit(Build) 0.33 cop[tikv] offset:0, count:1 │ └─TableRangeScan 0.33 cop[tikv] table:tbl_43 range:[-inf,"MFWmuOsoyDv"), keep order:true, stats:pseudo ├─Limit(Build) 0.33 cop[tikv] offset:0, count:1 @@ -3854,12 +3854,12 @@ explain format = 'brief' select max(col_304) from (select /*+ use_index_merge( t id estRows task access object operator info StreamAgg 1.00 root funcs:max(planner__core__casetest__physicalplantest__physical_plan.tbl_43.col_304)->Column#2 └─IndexMerge 1.00 root type: union, limit embedded(offset:0, count:1) - ├─Limit(Build) 0.00 cop[tikv] offset:0, count:1 - │ └─TableRangeScan 0.00 cop[tikv] table:tbl_43 range:["LUBGzGMA","LUBGzGMA"], keep order:true, desc, stats:pseudo ├─Limit(Build) 0.33 cop[tikv] offset:0, count:1 - │ └─IndexRangeScan 0.33 cop[tikv] table:tbl_43, index:idx_261(col_304) range:[-inf,"YEpfYfPVvhMlHGHSMKm"), keep order:true, desc, stats:pseudo + │ └─TableRangeScan 0.33 cop[tikv] table:tbl_43 range:[-inf,"YEpfYfPVvhMlHGHSMKm"), keep order:true, desc, stats:pseudo ├─Limit(Build) 0.33 cop[tikv] offset:0, count:1 - │ └─IndexRangeScan 0.33 cop[tikv] table:tbl_43, index:idx_262(col_304) range:("PE",+inf], keep order:true, desc, stats:pseudo + │ └─IndexRangeScan 0.33 cop[tikv] table:tbl_43, index:idx_261(col_304) range:("PE",+inf], keep order:true, desc, stats:pseudo + ├─Limit(Build) 0.00 cop[tikv] offset:0, count:1 + │ └─IndexRangeScan 0.00 cop[tikv] table:tbl_43, index:idx_262(col_304) range:["LUBGzGMA","LUBGzGMA"], keep order:true, desc, stats:pseudo ├─Limit(Build) 0.33 cop[tikv] offset:0, count:1 │ └─TableRangeScan 0.33 cop[tikv] table:tbl_43 range:[-inf,"MFWmuOsoyDv"), keep order:true, desc, stats:pseudo ├─Limit(Build) 0.33 cop[tikv] offset:0, count:1 diff --git a/tests/integrationtest/r/planner/core/indexmerge_path.result b/tests/integrationtest/r/planner/core/indexmerge_path.result index 2acd8c5d4f40e..7bdcb3bdeecb4 100644 --- a/tests/integrationtest/r/planner/core/indexmerge_path.result +++ b/tests/integrationtest/r/planner/core/indexmerge_path.result @@ -1194,10 +1194,12 @@ index iac(a,c), index iad(a,d)); explain format = brief select /*+ use_index_merge(t) */ * from t where a = 1 and (b = 2 or c = 3 or d = 4); id estRows task access object operator info -IndexLookUp 0.03 root -├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:iab(a, b) range:[1,1], keep order:false, stats:pseudo -└─Selection(Probe) 0.03 cop[tikv] or(eq(planner__core__indexmerge_path.t.b, 2), or(eq(planner__core__indexmerge_path.t.c, 3), eq(planner__core__indexmerge_path.t.d, 4))) - └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo +IndexMerge 0.00 root type: union +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:iab(a, b) range:[1 2,1 2], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:iac(a, c) range:[1 3,1 3], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:iad(a, d) range:[1 4,1 4], keep order:false, stats:pseudo +└─Selection(Probe) 0.00 cop[tikv] eq(planner__core__indexmerge_path.t.a, 1), or(eq(planner__core__indexmerge_path.t.b, 2), or(eq(planner__core__indexmerge_path.t.c, 3), eq(planner__core__indexmerge_path.t.d, 4))) + └─TableRowIDScan 8.00 cop[tikv] table:t keep order:false, stats:pseudo drop table if exists t; create table t(a int, b int, c int, d int, index iab(b,a), @@ -1205,10 +1207,12 @@ index iac(c,a,d), index iad(a,d)); explain format = brief select /*+ use_index_merge(t) */ * from t where a = 1 and (b = 2 or (c = 3 and d = 4) or d = 5); id estRows task access object operator info -IndexLookUp 0.02 root -├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:iad(a, d) range:[1,1], keep order:false, stats:pseudo -└─Selection(Probe) 0.02 cop[tikv] or(eq(planner__core__indexmerge_path.t.b, 2), or(and(eq(planner__core__indexmerge_path.t.c, 3), eq(planner__core__indexmerge_path.t.d, 4)), eq(planner__core__indexmerge_path.t.d, 5))) - └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo +IndexMerge 0.00 root type: union +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:iab(b, a) range:[2 1,2 1], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:iad(a, d) range:[1 5,1 5], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 0.00 cop[tikv] table:t, index:iac(c, a, d) range:[3 1 4,3 1 4], keep order:false, stats:pseudo +└─Selection(Probe) 0.00 cop[tikv] eq(planner__core__indexmerge_path.t.a, 1), or(eq(planner__core__indexmerge_path.t.b, 2), or(and(eq(planner__core__indexmerge_path.t.c, 3), eq(planner__core__indexmerge_path.t.d, 4)), eq(planner__core__indexmerge_path.t.d, 5))) + └─TableRowIDScan 8.00 cop[tikv] table:t keep order:false, stats:pseudo drop table if exists t; create table t (a int, b int, c int, j1 json, j2 json, key mvi1((cast(j1 as unsigned array)), a),