Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

planner: improve range underestimation #53860

Merged
merged 18 commits into from
Jun 16, 2024
Merged

planner: improve range underestimation #53860

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
5f031e8
improve range underestimation
terry1purcell Jun 6, 2024
0d4b5f0
Merge branch 'pingcap:master' into rangeest
terry1purcell Jun 7, 2024
6ae3b74
Merge branch 'pingcap:master' into rangeest
terry1purcell Jun 8, 2024
6bd5921
improve range test1
terry1purcell Jun 10, 2024
853392c
improve range review1
terry1purcell Jun 11, 2024
800c2ed
Merge branch 'pingcap:master' into rangeest
terry1purcell Jun 11, 2024
42c2d99
Merge branch 'pingcap:master' into rangeest
terry1purcell Jun 11, 2024
43eac27
improve range review2
terry1purcell Jun 12, 2024
389cf76
improve range review3
terry1purcell Jun 12, 2024
2ad83b4
improve range review3
terry1purcell Jun 12, 2024
66435fb
improve range review4
terry1purcell Jun 12, 2024
a08975e
Merge branch 'pingcap:master' into rangeest
terry1purcell Jun 12, 2024
ab222f7
Merge branch 'pingcap:master' into rangeest
terry1purcell Jun 13, 2024
7da11d0
Merge branch 'pingcap:master' into rangeest
terry1purcell Jun 13, 2024
a6b4699
Merge branch 'pingcap:master' into rangeest
terry1purcell Jun 13, 2024
8ea761c
Merge branch 'pingcap:master' into rangeest
terry1purcell Jun 14, 2024
24c7d3e
Merge branch 'pingcap:master' into rangeest
terry1purcell Jun 14, 2024
bfca395
Merge branch 'pingcap:master' into rangeest
terry1purcell Jun 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 123 additions & 3 deletions pkg/planner/cardinality/testdata/cardinality_suite_out.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
{
"Start": 800,
"End": 900,
"Count": 735.504166655054
"Count": 771.504166655054
},
{
"Start": 900,
Expand Down Expand Up @@ -79,7 +79,7 @@
{
"Start": 800,
"End": 1000,
"Count": 1193.696869573942
"Count": 1229.696869573942
},
{
"Start": 900,
Expand All @@ -104,7 +104,7 @@
{
"Start": 200,
"End": 400,
"Count": 1237.5288209899081
"Count": 1226.2788209899081
},
{
"Start": 200,
Expand Down Expand Up @@ -2535,6 +2535,23 @@
}
]
},
{
"github.com/pingcap/tidb/pkg/statistics.(*Histogram).EqualRowCount": [
{
"Locate value in buckets": {
"BucketIdx": 0,
"Exceed": false,
"InBucket": false,
"MatchLastValue": false,
"Value": "KindMinNotNull <nil>"
}
},
{
"Count": 0,
"Matched": false
}
]
},
{
"github.com/pingcap/tidb/pkg/statistics.(*Histogram).OutOfRangeRowCount": [
{
Expand Down Expand Up @@ -2816,6 +2833,23 @@
}
]
},
{
"github.com/pingcap/tidb/pkg/statistics.(*Histogram).EqualRowCount": [
{
"Locate value in buckets": {
"BucketIdx": 0,
"Exceed": false,
"InBucket": false,
"MatchLastValue": false,
"Value": "KindBytes \\x01"
}
},
{
"Count": 0,
"Matched": false
}
]
},
{
"github.com/pingcap/tidb/pkg/statistics.(*Histogram).OutOfRangeRowCount": [
{
Expand Down Expand Up @@ -3526,6 +3560,23 @@
}
]
},
{
"github.com/pingcap/tidb/pkg/statistics.(*Histogram).EqualRowCount": [
{
"Locate value in buckets": {
"BucketIdx": 0,
"Exceed": false,
"InBucket": false,
"MatchLastValue": false,
"Value": "KindMinNotNull <nil>"
}
},
{
"Count": 0,
"Matched": false
}
]
},
{
"github.com/pingcap/tidb/pkg/statistics.(*TopN).BetweenCount": {
"Result": 0
Expand Down Expand Up @@ -3666,6 +3717,32 @@
}
]
},
{
"github.com/pingcap/tidb/pkg/statistics.(*Histogram).EqualRowCount": [
{
"Locate value in buckets": {
"BucketIdx": 111,
"Exceed": false,
"InBucket": true,
"MatchLastValue": false,
"Value": "KindInt64 400"
}
},
{
"Related Buckets in Histogram": [
{
"Count": 896,
"Index": 111,
"Repeat": 1
}
]
},
{
"Count": 0.99,
"Matched": false
}
]
},
{
"github.com/pingcap/tidb/pkg/statistics.(*TopN).BetweenCount": [
{
Expand Down Expand Up @@ -3813,6 +3890,23 @@
}
]
},
{
"github.com/pingcap/tidb/pkg/statistics.(*Histogram).EqualRowCount": [
{
"Locate value in buckets": {
"BucketIdx": 0,
"Exceed": false,
"InBucket": false,
"MatchLastValue": false,
"Value": "KindBytes \\x01"
}
},
{
"Count": 0,
"Matched": false
}
]
},
{
"github.com/pingcap/tidb/pkg/statistics.(*TopN).BetweenCount": {
"Result": 0
Expand Down Expand Up @@ -3942,6 +4036,32 @@
}
]
},
{
"github.com/pingcap/tidb/pkg/statistics.(*Histogram).EqualRowCount": [
{
"Locate value in buckets": {
"BucketIdx": 111,
"Exceed": false,
"InBucket": true,
"MatchLastValue": false,
"Value": "KindBytes \\x03\\x80\\x00\\x00\\x00\\x00\\x00\\x01\\x91"
}
},
{
"Related Buckets in Histogram": [
{
"Count": 896,
"Index": 111,
"Repeat": 1
}
]
},
{
"Count": 0.99,
"Matched": false
}
]
},
{
"github.com/pingcap/tidb/pkg/statistics.(*TopN).BetweenCount": {
"Result": 0
Expand Down
15 changes: 9 additions & 6 deletions pkg/statistics/histogram.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -593,14 +593,17 @@ func (hg *Histogram) LessRowCount(sctx context.PlanContext, value types.Datum) f
func (hg *Histogram) BetweenRowCount(sctx context.PlanContext, a, b types.Datum) float64 {
lessCountA := hg.LessRowCount(sctx, a)
lessCountB := hg.LessRowCount(sctx, b)
// If lessCountA is not less than lessCountB, it may be that they fall to the same bucket and we cannot estimate
// the fraction, so we use `totalCount / NDV` to estimate the row count, but the result should not greater than
// lessCountB or notNullCount-lessCountA.
if lessCountA >= lessCountB && hg.NDV > 0 {
rangeEst := lessCountB - lessCountA
lowEqual, _ := hg.EqualRowCount(sctx, a, false)
ndvAvg := hg.NotNullCount() / float64(hg.NDV)
// If values fall in the same bucket, we may underestimate the fractional result. So estimate the low value (a) as an equals, and
// estimate the high value as the default (because the input high value may be "larger" than the true high value). The range should
// not be less than both the low+high - or the lesser of the estimate for the individual range of a or b is used as a bound.
if rangeEst < math.Max(lowEqual, ndvAvg) && hg.NDV > 0 {
result := math.Min(lessCountB, hg.NotNullCount()-lessCountA)
return math.Min(result, hg.NotNullCount()/float64(hg.NDV))
return math.Min(result, lowEqual+ndvAvg)
}
return lessCountB - lessCountA
return rangeEst
}

// TotalRowCount returns the total count of this histogram.
Expand Down