Merge cockroachdb#38036

38036: opt: fix overflow causing zero row-count r=justinj a=justinj

We were subtracting two ints that could overflow and then casting the
result to a float64. There's an easy solution to avoid the overflow:
just cast each integer to a float64 *before* performing the subtraction.

I will backport this to 19.1 and 2.1.

Release note (bug fix): Previously, due to a bug when estimating result
set sizes in the optimizer, queries involving int ranges that were very
large could result in poor plans being generated.

Co-authored-by: Justin Jaffray <justin@cockroachlabs.com>

pkg/sql/opt/memo/statistics_builder.go

@@ -2525,11 +2525,12 @@ func (sb *statisticsBuilder) updateDistinctCountsFromConstraint(
 					end := int(*endVal.(*tree.DInt))
 					// We assume that both start and end boundaries are inclusive. This
 					// should be the case for integer valued columns (due to normalization
-					// by constraint.PreferInclusive).
+					// by constraint.PreferInclusive). We must cast each end to a float
+					// *before* performing the subtraction to avoid overflow.
 					if c.Columns.Get(col).Ascending() {
-						distinctCount += float64(end - start)
+						distinctCount += float64(end) - float64(start)
 					} else {
-						distinctCount += float64(start - end)
+						distinctCount += float64(start) - float64(end)
 					}
 				} else {
 					// We can't determine the distinct count for this column. For example,

pkg/sql/opt/memo/testdata/stats/scan

@@ -509,3 +509,16 @@ SELECT * FROM empty
 scan empty
  ├── columns: x:1(int)
  └── stats: [rows=1]
+
+# Regression test: previously, overflow when computing estimated distinct count
+# here resulted in a row count of zero being estimated.
+opt
+SELECT x FROM a WHERE x >= -9223372036854775808 AND x <= 0 ORDER BY x LIMIT 10
+----
+scan a
+ ├── columns: x:1(int!null)
+ ├── constraint: /1: [/-9223372036854775808 - /0]
+ ├── limit: 10
+ ├── stats: [rows=10]
+ ├── key: (1)
+ └── ordering: +1