Skip to content

Commit

Permalink
planner: derive index filters for mv index paths (#54877) (#55027)
Browse files Browse the repository at this point in the history 
close #54876
  • Loading branch information
@ti-chi-bot
ti-chi-bot committed Jul 30, 2024
1 parent 31fb9de commit cb0fa82
Show file tree
Hide file tree
Showing 3 changed files with 261 additions and 4 deletions.
53 changes: 49 additions & 4 deletions pkg/planner/core/indexmerge_path.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1014,7 +1014,29 @@ func (ds *DataSource) generateIndexMerge4ComposedIndex(normalPathCnt int, indexM
remainedCNFs = append(remainedCNFs, CNFItem)
}
}
mvp := ds.buildPartialPathUp4MVIndex(combinedPartialPaths, true, remainedCNFs, ds.tableStats.HistColl)

condInIdxFilter := make(map[string]struct{}, len(remainedCNFs))
// try to derive index filters for each path
for _, path := range combinedPartialPaths {
idxFilters, _ := ds.splitIndexFilterConditions(remainedCNFs, path.FullIdxCols, path.FullIdxColLens)
idxFilters = util.CloneExprs(idxFilters)
path.IndexFilters = append(path.IndexFilters, idxFilters...)
for _, idxFilter := range idxFilters {
condInIdxFilter[string(idxFilter.HashCode())] = struct{}{}
}
}

// Collect the table filters.
// Since it's the intersection type index merge here, as long as a filter appears in one path, we don't need it in
// the table filters.
var tableFilters []expression.Expression
for _, CNFItem := range remainedCNFs {
if _, ok := condInIdxFilter[string(CNFItem.HashCode())]; !ok {
tableFilters = append(tableFilters, CNFItem)
}
}

mvp := ds.buildPartialPathUp4MVIndex(combinedPartialPaths, true, tableFilters, ds.tableStats.HistColl)

ds.possibleAccessPaths = append(ds.possibleAccessPaths, mvp)
return nil
Expand Down Expand Up @@ -1079,10 +1101,27 @@ func (ds *DataSource) generateIndexMerge4MVIndex(normalPathCnt int, filters []ex
continue
}

// Here, all partial paths are built from the same MV index, so we can directly use the first one to get the
// metadata.
// And according to buildPartialPaths4MVIndex, there must be at least one partial path if it returns ok.
firstPath := partialPaths[0]
idxFilters, tableFilters := ds.splitIndexFilterConditions(
remainingFilters,
firstPath.FullIdxCols,
firstPath.FullIdxColLens,
)

// Add the index filters to every partial path.
// For union type index merge, this is necessary for correctness.
for _, path := range partialPaths {
clonedIdxFilters := util.CloneExprs(idxFilters)
path.IndexFilters = append(path.IndexFilters, clonedIdxFilters...)
}

ds.possibleAccessPaths = append(ds.possibleAccessPaths, ds.buildPartialPathUp4MVIndex(
partialPaths,
isIntersection,
remainingFilters,
tableFilters,
ds.tableStats.HistColl,
),
)
Expand Down Expand Up @@ -1245,10 +1284,16 @@ func buildPartialPath4MVIndex(
partialPath := &util.AccessPath{Index: mvIndex}
partialPath.Ranges = ranger.FullRange()
for i := 0; i < len(idxCols); i++ {
length := mvIndex.Columns[i].Length
// For full length prefix index, we consider it as non prefix index.
// This behavior is the same as in IndexInfo2Cols(), which is used for non mv index.
if length == idxCols[i].RetType.GetFlen() {
length = types.UnspecifiedLength
}
partialPath.IdxCols = append(partialPath.IdxCols, idxCols[i])
partialPath.IdxColLens = append(partialPath.IdxColLens, mvIndex.Columns[i].Length)
partialPath.IdxColLens = append(partialPath.IdxColLens, length)
partialPath.FullIdxCols = append(partialPath.FullIdxCols, idxCols[i])
partialPath.FullIdxColLens = append(partialPath.FullIdxColLens, mvIndex.Columns[i].Length)
partialPath.FullIdxColLens = append(partialPath.FullIdxColLens, length)
}
if err := detachCondAndBuildRangeForPath(sctx, partialPath, accessFilters, histColl); err != nil {
return nil, false, err
Expand Down
124 changes: 124 additions & 0 deletions tests/integrationtest/r/planner/core/indexmerge_path.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1079,3 +1079,127 @@ IndexMerge 0.05 root type: union
└─Selection(Probe) 0.05 cop[tikv] eq(planner__core__indexmerge_path.t.a, 1)
└─TableRowIDScan 49.94 cop[tikv] table:t keep order:false, stats:pseudo
SET @@tidb_opt_fix_control = default;
drop table if exists t, t1;
create table t (a int, b varchar(30), c float, j json, pk int primary key,
key mvi1(c, (cast(j->'$.a' as unsigned array)), b),
key mvi2(a, (cast(j->'$.c' as unsigned array))),
key mvi3((cast(j->'$.d' as unsigned array)), c),
key idx(b, c)
);
insert into t values (1, 'test', 1, '{"a":[3,4,5], "c":[7,8,9], "d":[10,11,12]}', 1);
insert into t values (2, 'text', 1, '{"a":[4,5,6], "c":[10,11,12], "d":[13,14,15]}', 2);
insert into t values (1, 'abcd', 1, '{"a":[7,8,9], "c":[13,14,15], "d":[16,17,18]}', 3);
SELECT /*+ use_index_merge(t, mvi1) */ * from t where
c = 1 and
json_overlaps(j->'$.a', '[4,5,6]') and
b not like '%test%';
a b c j pk
2 text 1 {"a": [4, 5, 6], "c": [10, 11, 12], "d": [13, 14, 15]} 2
EXPLAIN format = brief SELECT /*+ use_index_merge(t, mvi1) */ * from t where
c = 1 and
json_overlaps(j->'$.a', '[4,5,6]') and
b not like '%test%';
id estRows task access object operator info
Selection 0.24 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j, "$.a"), cast("[4,5,6]", json BINARY))
└─IndexMerge 0.30 root type: union
├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92))
│ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 4,1 4], keep order:false, stats:pseudo
├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92))
│ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 5,1 5], keep order:false, stats:pseudo
├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92))
│ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 6,1 6], keep order:false, stats:pseudo
└─TableRowIDScan(Probe) 0.30 cop[tikv] table:t keep order:false, stats:pseudo
SELECT /*+ use_index_merge(t, mvi1) */ * from t where
c = 1 and
json_contains(j->'$.a', '[4,5]') and
b not like '%test%';
a b c j pk
2 text 1 {"a": [4, 5, 6], "c": [10, 11, 12], "d": [13, 14, 15]} 2
EXPLAIN format = brief SELECT /*+ use_index_merge(t, mvi1) */ * from t where
c = 1 and
json_contains(j->'$.a', '[4,5]') and
b not like '%test%';
id estRows task access object operator info
IndexMerge 0.00 root type: intersection
├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92))
│ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 4,1 4], keep order:false, stats:pseudo
├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92))
│ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 5,1 5], keep order:false, stats:pseudo
└─TableRowIDScan(Probe) 0.00 cop[tikv] table:t keep order:false, stats:pseudo
SELECT /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where
a = 1 and
b > 'abc' and
b not like '%test%' and
c = 10 and
3 member of (j->'$.a') and
3 member of (j->'$.c');
a b c j pk
EXPLAIN format=brief SELECT /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where
a = 1 and
b > 'abc' and
b not like '%test%' and
c = 10 and
3 member of (j->'$.a') and
3 member of (j->'$.c');
id estRows task access object operator info
IndexMerge 0.00 root type: intersection
├─Selection(Build) 2.67 cop[tikv] eq(planner__core__indexmerge_path.t.c, 10), not(like(planner__core__indexmerge_path.t.b, "%test%", 92)), not(like(planner__core__indexmerge_path.t.b, "%test%", 92))
│ └─IndexRangeScan 3333.33 cop[tikv] table:t, index:idx(b, c) range:("abc",+inf], keep order:false, stats:pseudo
├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92))
│ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[10 3,10 3], keep order:false, stats:pseudo
├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi2(a, cast(json_extract(`j`, _utf8mb4'$.c') as unsigned array)) range:[1 3,1 3], keep order:false, stats:pseudo
└─TableRowIDScan(Probe) 0.00 cop[tikv] table:t keep order:false, stats:pseudo
create table t1 (
a int,
b varchar(30),
c float,
d tinytext,
j json,
key mvi1(b(3), (cast(j as unsigned array))),
key mvi2((cast(j as unsigned array)), b),
key mvi3((cast(j as unsigned array)), d(30)),
key mvi4((cast(j as unsigned array)), d(255))
);
EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi1) */ * from t1 where
c = 1 and
json_contains(j, '[4,5]') and
b = 'abcdefg';
id estRows task access object operator info
TableReader 0.00 root data:Selection
└─Selection 0.00 cop[tikv] eq(planner__core__indexmerge_path.t1.b, "abcdefg"), eq(planner__core__indexmerge_path.t1.c, 1), json_contains(planner__core__indexmerge_path.t1.j, cast("[4,5]", json BINARY))
└─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi2) */ * from t1 where
c = 1 and
json_contains(j, '[4,5]') and
b = 'abcdefg' and
b like '%test%';
id estRows task access object operator info
IndexMerge 0.00 root type: intersection
├─Selection(Build) 0.00 cop[tikv] like(planner__core__indexmerge_path.t1.b, "%test%", 92)
│ └─IndexRangeScan 0.10 cop[tikv] table:t1, index:mvi2(cast(`j` as unsigned array), b) range:[4 "abcdefg",4 "abcdefg"], keep order:false, stats:pseudo
├─Selection(Build) 0.00 cop[tikv] like(planner__core__indexmerge_path.t1.b, "%test%", 92)
│ └─IndexRangeScan 0.10 cop[tikv] table:t1, index:mvi2(cast(`j` as unsigned array), b) range:[5 "abcdefg",5 "abcdefg"], keep order:false, stats:pseudo
└─Selection(Probe) 0.00 cop[tikv] eq(planner__core__indexmerge_path.t1.c, 1)
└─TableRowIDScan 0.00 cop[tikv] table:t1 keep order:false, stats:pseudo
EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi3) */ * from t1 where
c = 1 and
json_contains(j, '[4,5]') and
d not like '%test%';
id estRows task access object operator info
IndexMerge 0.01 root type: intersection
├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:mvi3(cast(`j` as unsigned array), d) range:[4,4], keep order:false, stats:pseudo
├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:mvi3(cast(`j` as unsigned array), d) range:[5,5], keep order:false, stats:pseudo
└─Selection(Probe) 0.01 cop[tikv] eq(planner__core__indexmerge_path.t1.c, 1), not(like(planner__core__indexmerge_path.t1.d, "%test%", 92))
└─TableRowIDScan 0.01 cop[tikv] table:t1 keep order:false, stats:pseudo
EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi4) */ * from t1 where
c = 1 and
json_contains(j, '[4,5]') and
d not like '%test%';
id estRows task access object operator info
IndexMerge 0.01 root type: intersection
├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t1.d, "%test%", 92))
│ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:mvi4(cast(`j` as unsigned array), d) range:[4,4], keep order:false, stats:pseudo
├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t1.d, "%test%", 92))
│ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:mvi4(cast(`j` as unsigned array), d) range:[5,5], keep order:false, stats:pseudo
└─Selection(Probe) 0.01 cop[tikv] eq(planner__core__indexmerge_path.t1.c, 1)
└─TableRowIDScan 0.01 cop[tikv] table:t1 keep order:false, stats:pseudo
88 changes: 88 additions & 0 deletions tests/integrationtest/t/planner/core/indexmerge_path.test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -425,3 +425,91 @@ EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR b =
EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR b = '4' OR c = 5 OR b = '12' OR c = 13);
EXPLAIN format = brief SELECT * FROM t WHERE a = 1 AND (c = 13 OR c = 15 OR c = 5 OR b = '12' OR c = 13 OR b = '11');
SET @@tidb_opt_fix_control = default;

# Test deriving index filters for mv index paths
drop table if exists t, t1;
create table t (a int, b varchar(30), c float, j json, pk int primary key,
key mvi1(c, (cast(j->'$.a' as unsigned array)), b),
key mvi2(a, (cast(j->'$.c' as unsigned array))),
key mvi3((cast(j->'$.d' as unsigned array)), c),
key idx(b, c)
);

insert into t values (1, 'test', 1, '{"a":[3,4,5], "c":[7,8,9], "d":[10,11,12]}', 1);
insert into t values (2, 'text', 1, '{"a":[4,5,6], "c":[10,11,12], "d":[13,14,15]}', 2);
insert into t values (1, 'abcd', 1, '{"a":[7,8,9], "c":[13,14,15], "d":[16,17,18]}', 3);

# case 1: union type index merge on single mv index from single condition
SELECT /*+ use_index_merge(t, mvi1) */ * from t where
c = 1 and
json_overlaps(j->'$.a', '[4,5,6]') and
b not like '%test%';

EXPLAIN format = brief SELECT /*+ use_index_merge(t, mvi1) */ * from t where
c = 1 and
json_overlaps(j->'$.a', '[4,5,6]') and
b not like '%test%';

# case 2: intersection type index merge on single mv index from single condition
SELECT /*+ use_index_merge(t, mvi1) */ * from t where
c = 1 and
json_contains(j->'$.a', '[4,5]') and
b not like '%test%';

EXPLAIN format = brief SELECT /*+ use_index_merge(t, mvi1) */ * from t where
c = 1 and
json_contains(j->'$.a', '[4,5]') and
b not like '%test%';

# case 3: intersection type index merge on multiple indexes from different conditions
SELECT /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where
a = 1 and
b > 'abc' and
b not like '%test%' and
c = 10 and
3 member of (j->'$.a') and
3 member of (j->'$.c');

EXPLAIN format=brief SELECT /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where
a = 1 and
b > 'abc' and
b not like '%test%' and
c = 10 and
3 member of (j->'$.a') and
3 member of (j->'$.c');

# case 4: test prefix indexes
create table t1 (
a int,
b varchar(30),
c float,
d tinytext,
j json,
key mvi1(b(3), (cast(j as unsigned array))),
key mvi2((cast(j as unsigned array)), b),
key mvi3((cast(j as unsigned array)), d(30)),
key mvi4((cast(j as unsigned array)), d(255))
);


EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi1) */ * from t1 where
c = 1 and
json_contains(j, '[4,5]') and
b = 'abcdefg';

EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi2) */ * from t1 where
c = 1 and
json_contains(j, '[4,5]') and
b = 'abcdefg' and
b like '%test%';

EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi3) */ * from t1 where
c = 1 and
json_contains(j, '[4,5]') and
d not like '%test%';

EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi4) */ * from t1 where
c = 1 and
json_contains(j, '[4,5]') and
d not like '%test%';

0 comments on commit cb0fa82

Please sign in to comment.