Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

util/ranger: avoid building ranges twice in detachCNFCondAndBuildRangeForIndex #37177

Merged
merged 10 commits into from
Aug 25, 2022
Merged

util/ranger: avoid building ranges twice in detachCNFCondAndBuildRangeForIndex #37177

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
bf2a0e3
avoid building ranges twice
xuyifangreeneyes Aug 17, 2022
95304d9
Merge branch 'master' into build-range-once
xuyifangreeneyes Aug 23, 2022
c2dda16
fix and add comments
xuyifangreeneyes Aug 23, 2022
76a0ae0
fix
xuyifangreeneyes Aug 23, 2022
d576606
fix
xuyifangreeneyes Aug 23, 2022
a66feca
add micro bench
xuyifangreeneyes Aug 23, 2022
2661e8b
Merge branch 'master' into build-range-once
xuyifangreeneyes Aug 23, 2022
afd056b
remove unnecessary code
xuyifangreeneyes Aug 24, 2022
bb5f1c0
Merge branch 'build-range-once' of https://github.com/xuyifangreeneye…
xuyifangreeneyes Aug 24, 2022
c861615
Merge branch 'master' into build-range-once
ti-chi-bot Aug 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 133 additions & 0 deletions util/ranger/bench_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
// Copyright 2021 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ranger_test

import (
"context"
"testing"

"github.com/pingcap/tidb/expression"
plannercore "github.com/pingcap/tidb/planner/core"
"github.com/pingcap/tidb/session"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/testkit"
"github.com/pingcap/tidb/util/ranger"
"github.com/stretchr/testify/require"
)

func BenchmarkDetachCondAndBuildRangeForIndex(b *testing.B) {
store := testkit.CreateMockStore(b)
testKit := testkit.NewTestKit(b, store)
testKit.MustExec("USE test")
testKit.MustExec("DROP TABLE IF EXISTS t")
testKit.MustExec(`
CREATE TABLE t (
id bigint(20) NOT NULL,
flag tinyint(1) NOT NULL,
start_time bigint(20) NOT NULL,
end_time bigint(20) NOT NULL,
org_id bigint(20) NOT NULL,
work_type varchar(20) CHARACTER SET latin1 DEFAULT NULL,
KEY idx (work_type, flag, org_id, end_time)
)`)
const longInListQuery = `
SELECT
*
FROM
test.t
WHERE
flag = false
AND work_type = 'ART'
AND start_time < 1437233819011
AND org_id IN (
6914445279216,13370400597894,23573357446410,25887731612857,47914138043476,55587486395595,81718641401629,116900287316705,145561135354847,146149695520705,150737101640717,
156461412195734,159282589442914,172009309011258,177247737846954,186431984199126,192702036012003,198028383844036,203822128217766,204361899227892,222307981616040,
223676791310456,239710781952147,284504079630201,293577747630605,294033490212963,295616321229667,296595862564733,306758307660237,312453546107536,341725138386801,
365564508529854,367823101074613,372136375994705,379986503803645,391811651805728,400703265358976,402101198418969,418909618067022,431707392441212,433127041410999,
453871162499705,459423393488794,468411967568789,470295426508062,474431880357867,486212302560292,491505628336549,501258676986205,512885499269076,541562269064881,
573472004072939,588733301257768,615894236361514,615983097695492,632401842501612,647142957465854,718424118160213,727429900786891,730535283550760,738198024914914,
739163331366841,752647396960214,756654620675819,771528730062551,780002237988119,819197371932508,841300462416232,850412457085478,874244695288775,881570046324012,
888197716219703,891639557310370,891974587948562,894056936863746,896673560435518,907536217395121,911688951561329,915825484067466,955508673412475,970655461426927,
972378619997200,995940920336141,1001616070548723,1002328390838858,1008841511217759,1027157044666161,1048868628313964,1066571897813670,1077847212985966,1080168008312256,
1096474319900512,1102806753403857,1117700255389200,1121170709282654,1160894105829436,1173396752247253,1174823990738043,1176756778119821,1183639496028232,1224377491431012,
1240466389734677,1265076660322902,1269569491282720,1272581574031499,1294849094933502,1295099688484884,1298881176988511,1299992642087679,1307669929489952,1338074385647926,
1342415156765380,1360218717881999,1377658957083475,1392922543432192,1407444098594050,1438256495838735,1445134088155147,1486929584894557,1494681055271250,1500940354882616,
1530516421560327,1532590570974295,1544648947254643,1561923580960451,1563587565476473,1565067823935596,1573069534228327,1573167213450271,1573297960637648,1577324450386436,
1595234199645128,1595320706602899,1595934401297767,1616085587597345,1652295812716667,1655495487920136,1663475672388133,1668352817492466,1681094348275341,1689623403182214,
1701682724927093,1705012823832699,1710393138044599,1716535649128474,1733575964270463,1750190609804974,1754580077690816,1756061776687456,1758058273255859,1775158332937577,
1786728287430927,1816461420376899,1828580334315536,1858008005313000,1878841219054602,1878932921719554,1904081347331116,1904820184794904,1913069596895373,1941380005857044,
1954836070968071,1955618820347782,1983296066429676,1987819713385690,1998098943250021,2013403425656222,2026046763398088,2026621786756595,2042249014990205,2046741004470190,
2054370107316514,2082578854015062,2087183461591329,2087192311688265,2095277921868705,2103249621417272,2105159282073449,2106385538583803,2115025577188264,2115892671475192,
2144855844328126,2145526421460724,2155282047243675,2170620433275766,2189596848694026,2194468311513858,2213049505974092,2213713514793282,2213911591088204,2215904332099994,
2220235232411590,2226449138693468,2281727415070895,2288760906462231,2292201263531973,2292800860074434,2305015986098173,2311091146951322,2345959699274993,2356192386220089,
2358508761766515,2372253813770528,2372994254274287,2374279183118353,2382395547546489,2397593929551113,2414904392525386,2416320859150803,2432407185506251,2439133858011514,
2456909988876966,2490057196713078,2491000908839300,2501843499019962,2539856169115632,2542691236689315,2554165079560216,2565016421484028,2579373020942520,2583310536030502,
2601385761450563,2603243552830302,2609138752573551,2628285706286347,2631734058797888,2638342575912817,2640167419150776,2651402302096646,2652627728219962,2677347449018607,
2680209147172480,2683108753662485,2697695717735514,2699481485241986,2706019556864874,2707225343321107,2707841703322085,2724535386459144,2751805187614069,2759827465036125,
2761426575202406,2768997857008184,2780782950787373,2789872834409453,2801402100587551,2806075464399632,2811893029385689,2816433481597910,2817217705468440,2817327738406355,
2834191662385443,2838661299874842,2841835162527294,2842790844846179,2873548708258327,2879581389559553,2881798195775557,2943715564248539,2951224334569493,2958397742216527,
2962117760574537,2977264272542363,2982743447396379,3009349759215772,3010012117130847,3010096874529870,3010115852485268,3012535704934837,3019483201458596,3036314602733381,
3036837746136599,3039514679368453,3052506678397436,3059152178913030,3061095276596365,3063233426258797,3085115838323303,3093308450248420,3101208987984055,3106059275305341,
3107559394454149,3110050933644367,3138545183162445,3140678914029638,3142109724205254,3142527081183238,3160670144726081,3160829106050702,3172902656372574,3181996104936476,
3185563316673259,3189751274857877,3211729550313903,3213599609783958,3213659878256809,3215918533522245,3225137047012900,3231023124763183,3237856608607317,3247879729114765,
3253596244997212,3256221808530784,3259756319203329,3260467132021662,3270706364495298,3273179837457878,3281263986182695,3296686215236784,3315584830022847,3317781471030011,
3338120669693067,3342373672540130,3348941534532661,3359715878023917,3383970565360566,3397868551281252,3428808017724584,3449238661063482,3450439330534765,3471286956110444,
3472931370488350,3473088266701087,3474445823605231,3477506057124755,3477541077050002,3488929919781737,3502330373943603,3504844077112105,3508329172255490,3510690524473209,
3512324709745663,3514007754921924,3558011694836438,3559330712604565,3562145253615132,3581148885399840,3591312468185522,3597256243542091,3618160287855458,3625686898155792,
3650194338715713,3650610878210125,3668345904483321,3668494673430762,3677776463522008,3694233081600122,3709632134619598,3725419513943971,3730099951927002,3734961839633730,
3788380331922768,3811246425255446,3815582365292463,3829203122180282,3847292141308076,3877507310017402,3882136043994493,3887872593644033,3891280433757250,3924114035682754,
3930635798027692,3940634174809349,3955900287161214,3957844806020309,3973236219509940,3981294421878412,3984846206013660,3993802859865565,3996764828980278,4023461371356880,
4030830455174294,4032434581680299,4041625011713530,4041957068079946,4058955264781991,4083680454731905,4095581705542542,4112420671677445,4121292361441010,4133226631387396,
4139004365572538,4161250756754756,4163706760683594,4170471653379067,4171134267004311,4177796537235481,4180802682160942,4203191696595400,4209464578955045,4223422057959415,
4237541104444937,4257691911774311,4260020795088571,4301574030764989,4301922471400280,4304478206038048,4314941265701364,4320330498423583,4339739390410992,4342413486284428,
4347136230283432,4351145740656078,4358837874704787,4362622126951624,4364582223851552,4366497646764759,4385815379876479,4406431382404050,4409339407622327,4411432076559821,
4449849757340102,4453892102562139,4466153465045159,4479272299804907,4486938493241801,4500505590495671,4502678993390350,4508608408879950,4515778758013390,4516708986545589,
4534650929880461,4537213242644302,4537280160911816,4544757774741059,4544796825985803,4567299173366450,4586846032456054,4607174945068672,4613242210966642,4625035792839901,
4634756593700865,4635123568273790,4659875963875224,4670322637424975,4683035442855866,4689033135620199,4714163851388857,4714214015125119,4744848633979578,4750937375207328,
4776805289846989,4804031931975645,4837280540915990,4840957238353452,4868526553354967,4875063418864529,4909882543513230,4912268119820614,4919123728900332,4929754602909549,
4941072993543698,4941174020949904,4948032640819331,4955057670206957,4989823480030237,4990195550280933,5011619499820208,5013143564325843,5016786248387969,5019292677276101,
5045230878000223,5053158166772953,5058611677018883,5083565032770599,5100847523417394,5105223137691724,5116076462386020,5117104556161083,5137323839372187,5159591155175114,
5174145884534911,5182009696238822,5207529188216676,5229363397364492,5231081768308950,5233170527517877,5238337785560206,5259149223152958,5259347545688076,5265284989590946,
5288711160650011,5297955243309025,5300535720011230,5311590629866593,5313248950470856,5313960220028487,5321218075331795,5334031591400346,5340934779949776,5380276587818617,
5380604989545853,5392427172834832,5419648071490001,5436430269421440,5438720576124743,5442272167466546,5443531545450195,5462404261617760,5484761325677647
)
`
sctx := testKit.Session().(sessionctx.Context)
stmts, err := session.Parse(sctx, longInListQuery)
require.NoError(b, err)
require.Len(b, stmts, 1)
ret := &plannercore.PreprocessorReturn{}
err = plannercore.Preprocess(sctx, stmts[0], plannercore.WithPreprocessorReturn(ret))
require.NoError(b, err)
ctx := context.Background()
p, _, err := plannercore.BuildLogicalPlanForTest(ctx, sctx, stmts[0], ret.InfoSchema)
require.NoError(b, err)
selection := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection)
tbl := selection.Children()[0].(*plannercore.DataSource).TableInfo()
require.NotNil(b, selection)
conds := make([]expression.Expression, len(selection.Conditions))
for i, cond := range selection.Conditions {
conds[i] = expression.PushDownNot(sctx, cond)
}
cols, lengths := expression.IndexInfo2PrefixCols(tbl.Columns, selection.Schema().Columns, tbl.Indices[0])
require.NotNil(b, cols)

b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths)
require.NoError(b, err)
}
b.StopTimer()
}
40 changes: 28 additions & 12 deletions util/ranger/detacher.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,22 +283,38 @@ func (d *rangeDetacher) detachCNFCondAndBuildRangeForIndex(conditions []expressi
eqOrInCount := len(accessConds)
res.EqCondCount = eqCount
res.EqOrInCount = eqOrInCount
ranges, err = d.buildCNFIndexRange(tpSlice, eqOrInCount, accessConds)
ranges, err = d.buildRangeOnColsByCNFCond(tpSlice, eqOrInCount, accessConds)
if err != nil {
return nil, err
}

// Though ranges are built from equal/in conditions, some range may not be a single point after UnionRanges in buildCNFIndexRange.
// In order to prepare for the following appendRanges2PointRanges, we set d.mergeConsecutive to false and call buildCNFIndexRange
// again to get pointRanges, in which each range must be a single point. If we use ranges rather than pointRanges when calling
// appendRanges2PointRanges, wrong ranges would be calculated as issue https://github.com/pingcap/tidb/issues/26029 describes.
mergeConsecutive := d.mergeConsecutive
d.mergeConsecutive = false
pointRanges, err := d.buildCNFIndexRange(tpSlice, eqOrInCount, accessConds)
if err != nil {
return nil, err
// If index has prefix column and d.mergeConsecutive is true, ranges may not be point ranges anymore after UnionRanges.
// Therefore, we need to calculate pointRanges separately so that it can be used to append tail ranges in considerDNF branch.
// See https://github.com/pingcap/tidb/issues/26029 for details.
var pointRanges []*Range
if hasPrefix(d.lengths) && fixPrefixColRange(ranges, d.lengths, tpSlice) {
if d.mergeConsecutive {
pointRanges = make([]*Range, 0, len(ranges))
for _, ran := range ranges {
pointRanges = append(pointRanges, ran.Clone())
}
ranges, err = UnionRanges(d.sctx, ranges, d.mergeConsecutive)
if err != nil {
return nil, errors.Trace(err)
}
pointRanges, err = UnionRanges(d.sctx, pointRanges, false)
if err != nil {
return nil, errors.Trace(err)
}
} else {
ranges, err = UnionRanges(d.sctx, ranges, d.mergeConsecutive)
if err != nil {
return nil, errors.Trace(err)
}
pointRanges = ranges
}
} else {
pointRanges = ranges
}
d.mergeConsecutive = mergeConsecutive

res.Ranges = ranges
res.AccessConds = accessConds
Expand Down
16 changes: 11 additions & 5 deletions util/ranger/ranger.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -366,17 +366,13 @@ func BuildColumnRange(conds []expression.Expression, sctx sessionctx.Context, tp
return buildColumnRange(conds, sctx, tp, false, colLen)
}

// buildCNFIndexRange builds the range for index where the top layer is CNF.
func (d *rangeDetacher) buildCNFIndexRange(newTp []*types.FieldType,
func (d *rangeDetacher) buildRangeOnColsByCNFCond(newTp []*types.FieldType,
eqAndInCount int, accessCondition []expression.Expression) ([]*Range, error) {
rb := builder{sc: d.sctx.GetSessionVars().StmtCtx}
var (
ranges []*Range
err error
)
for _, col := range d.cols {
newTp = append(newTp, newFieldType(col.RetType))
}
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

newTp has already been set properly when passing it in.

for i := 0; i < eqAndInCount; i++ {
// Build ranges for equal or in access conditions.
point := rb.build(accessCondition[i], collate.GetCollator(newTp[i].GetCollate()))
Expand Down Expand Up @@ -409,6 +405,16 @@ func (d *rangeDetacher) buildCNFIndexRange(newTp []*types.FieldType,
if err != nil {
return nil, errors.Trace(err)
}
return ranges, nil
}

// buildCNFIndexRange builds the range for index where the top layer is CNF.
func (d *rangeDetacher) buildCNFIndexRange(newTp []*types.FieldType,
eqAndInCount int, accessCondition []expression.Expression) ([]*Range, error) {
ranges, err := d.buildRangeOnColsByCNFCond(newTp, eqAndInCount, accessCondition)
if err != nil {
return nil, err
}

// Take prefix index into consideration.
if hasPrefix(d.lengths) {
Expand Down