server: Make float rounding compatible with mysql for types: float, float(m,n) and double(m,n)

cmd/explaintest/r/select.result

@@ -476,14 +476,24 @@ create table precise_types (
 a BIGINT UNSIGNED NOT NULL,
 b BIGINT NOT NULL,
 c DECIMAL(21,1) NOT NULL,
-d DOUBLE(21,1) NOT NULL
+d DOUBLE(21,1) NOT NULL,
+e DOUBLE(3,1) NOT NULL,
+f DOUBLE NOT NULL,
+g FLOAT(10,2) NOT NULL,
+h FLOAT(2,1) NOT NULL,
+i FLOAT NOT NULL
 );
 insert into precise_types values (
 18446744073709551614,
 -9223372036854775806,
 99999999999999999999,
-18446744073709551614
+18446744073709551614,
+12.123,
+18446744073709551614,
+-34028234.66,
+-3.123456,
+-34028234.123456
 );
-SELECT a, b, c, d FROM precise_types;
-a	b	c	d
-18446744073709551614	-9223372036854775806	99999999999999999999.0	1.8446744073709552e19
+SELECT a, b, c, d, e, f, g, h, i FROM precise_types;
+a	b	c	d	e	f	g	h	i
+18446744073709551614	-9223372036854775806	99999999999999999999.0	18446744073709552000.0	12.1	1.8446744073709552e19	-34028236.00	-3.1	-34028200

cmd/explaintest/t/select.test

@@ -230,17 +230,27 @@ CREATE TABLE t (id int(10) unsigned NOT NULL AUTO_INCREMENT,
 );
 explain format = 'brief' select row_number() over( partition by i ) - x as rnk from t;
 
-# for issue 21692
+# for issue 21692 and 22791
 create table precise_types (
     a BIGINT UNSIGNED NOT NULL,
     b BIGINT NOT NULL,
     c DECIMAL(21,1) NOT NULL,
-    d DOUBLE(21,1) NOT NULL
+    d DOUBLE(21,1) NOT NULL,
+    e DOUBLE(3,1) NOT NULL,
+    f DOUBLE NOT NULL,
+    g FLOAT(10,2) NOT NULL,
+    h FLOAT(2,1) NOT NULL,
+    i FLOAT NOT NULL
 );
 insert into precise_types values (
     18446744073709551614,
     -9223372036854775806,
     99999999999999999999,
-    18446744073709551614
+    18446744073709551614,
+    12.123,
+    18446744073709551614,
+    -34028234.66,
+    -3.123456,
+    -34028234.123456
 );
-SELECT a, b, c, d FROM precise_types;
+SELECT a, b, c, d, e, f, g, h, i FROM precise_types;

server/util.go

@@ -37,10 +37,12 @@ package server
 import (
 	"bytes"
 	"encoding/binary"
+	"fmt"
 	"io"
 	"math"
 	"net/http"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/pingcap/parser/mysql"
@@ -307,18 +309,10 @@ func dumpTextRow(buffer []byte, columns []*ColumnInfo, row chunk.Row) ([]byte, e
 			}
 			buffer = dumpLengthEncodedString(buffer, tmp)
 		case mysql.TypeFloat:
-			prec := -1
-			if columns[i].Decimal > 0 && int(col.Decimal) != mysql.NotFixedDec && col.Table == "" {
-				prec = int(col.Decimal)
-			}
-			tmp = appendFormatFloat(tmp[:0], float64(row.GetFloat32(i)), prec, 32)
+			tmp = appendFormatFloat(tmp[:0], float64(row.GetFloat32(i)), 32, col.Decimal, col.ColumnLength, col.Table == "")
 			buffer = dumpLengthEncodedString(buffer, tmp)
 		case mysql.TypeDouble:
-			prec := types.UnspecifiedLength
-			if col.Decimal > 0 && int(col.Decimal) != mysql.NotFixedDec && col.Table == "" {
-				prec = int(col.Decimal)
-			}
-			tmp = appendFormatFloat(tmp[:0], row.GetFloat64(i), prec, 64)
+			tmp = appendFormatFloat(tmp[:0], row.GetFloat64(i), 64, col.Decimal, col.ColumnLength, col.Table == "")
 			buffer = dumpLengthEncodedString(buffer, tmp)
 		case mysql.TypeNewDecimal:
 			buffer = dumpLengthEncodedString(buffer, hack.Slice(row.GetMyDecimal(i).String()))
@@ -359,52 +353,118 @@ func lengthEncodedIntSize(n uint64) int {
 }
 
 const (
-	expFormatBig     = 1e15
-	expFormatSmall   = 1e-15
-	defaultMySQLPrec = 5
+	expFormatBig       = 1e15
+	expFormatSmall     = 1e-15
+	defaultMySQLPrec   = 5
+	defaultMySQLPrec64 = 16
 )
 
-func appendFormatFloat(in []byte, fVal float64, prec, bitSize int) []byte {
+/*
+	Formats a float value into a string (byte slice) and returns.
+
+	The rounding/formatting is split into the following cases:
+	1. Scientific notation (1.23.e4): Used for big numbers except when value comes from a custom width column or from the select part of the query
+	2. mysql.TypeDouble: return string with as much precision as needed (precision = -1)
+	3. mysql.TypeFloat: return a rounded value that corresponds to mysql's (precision = 5)
+	4. mysql.TypeDouble(m,n):
+		4.1 If the columnLength < defaultMySQLPrec64, return string with as much precision as needed (precision = n)
+		4.2 else: return rounded string to precision defaultMySQLPrec64, and padded with 0's.
+	5. mysql.TypeFloat(m,n): return a rounded value rounded with (precision = 5) and formatted with (precision = n)
+
+	Note: 	Custom float/double column width is deprecated in mysql 8.0.14.
+			However for compatibility, we still try to make the output the same as mysql for these types.
+*/
+func appendFormatFloat(in []byte, fVal float64, bitSize int, colDecimal uint8, colLength uint32, isFromSelect bool) []byte {
+	isCustomColumnWidth := false
+	prec := types.UnspecifiedLength
+	if colDecimal > 0 && colDecimal != mysql.NotFixedDec {
+		prec = int(colDecimal)
+		isCustomColumnWidth = true
+	}
+
 	absVal := math.Abs(fVal)
 	if absVal > math.MaxFloat64 || math.IsNaN(absVal) {
 		return []byte{'0'}
 	}
 	isEFormat := false
-	if bitSize == 32 {
-		isEFormat = float32(absVal) >= expFormatBig || (float32(absVal) != 0 && float32(absVal) < expFormatSmall)
-	} else {
-		isEFormat = absVal >= expFormatBig || (absVal != 0 && absVal < expFormatSmall)
-	}
-	var out []byte
-	if isEFormat {
+	// Scientific format (1.2e3) should be used for big numbers. However, it is never used for columns with custom length/decimal
+	if !isCustomColumnWidth || isFromSelect {
 		if bitSize == 32 {
-			prec = defaultMySQLPrec
+			isEFormat = float32(absVal) >= expFormatBig || (float32(absVal) != 0 && float32(absVal) < expFormatSmall)
+		} else {
+			isEFormat = absVal >= expFormatBig || (absVal != 0 && absVal < expFormatSmall)
 		}
-		out = strconv.AppendFloat(in, fVal, 'e', prec, bitSize)
-		valStr := out[len(in):]
-		// remove the '+' from the string for compatibility.
-		plusPos := bytes.IndexByte(valStr, '+')
-		if plusPos > 0 {
-			plusPosInOut := len(in) + plusPos
-			out = append(out[:plusPosInOut], out[plusPosInOut+1:]...)
+	}
+
+	switch {
+	case isEFormat:
+		return formatFloatScientificNotation(in, fVal, prec, bitSize)
+	case bitSize == 32 && isCustomColumnWidth:
+		return strconv.AppendFloat(in, fVal, 'f', prec, bitSize)
+	case bitSize == 32:
+		roundedValue := roundFloatToPrecision(fVal, defaultMySQLPrec)
+		return strconv.AppendFloat(in, roundedValue, 'f', -1, bitSize)
+	case isCustomColumnWidth && colLength > defaultMySQLPrec64:
+		roundedValue := roundFloatToPrecision(fVal, defaultMySQLPrec64)
+		valueStr := strconv.FormatFloat(roundedValue, 'f', -1, bitSize)
+		// Pad with 0's if the value has no decimals
+		if len(strings.Split(valueStr, ".")) == 1 {
+			valueStr = valueStr + fmt.Sprintf(".%0*v", prec, 0)
 		}
-		// remove extra '0'
-		ePos := bytes.IndexByte(valStr, 'e')
-		pointPos := bytes.IndexByte(valStr, '.')
-		ePosInOut := len(in) + ePos
-		pointPosInOut := len(in) + pointPos
-		validPos := ePosInOut
-		for i := ePosInOut - 1; i >= pointPosInOut; i-- {
-			if out[i] == '0' || out[i] == '.' {
-				validPos = i
-			} else {
-				break
-			}
+		return []byte(valueStr)
+	default:
+		return strconv.AppendFloat(in, fVal, 'f', prec, bitSize)
+	}
+}
+
+func roundFloatToPrecision(fValue float64, precision int) float64 {
+	base := math.Pow10(precision)
+
+	// Calculate the mantissa and exponent (scientific notation)
+	exponent := math.Floor(math.Log10(math.Abs(fValue)))
+	mantissa := fValue / math.Pow(10, exponent)
+
+	// Round the mantissa to <precision> number of digits, and multiply with exponent to get the result
+	roundedValue := math.Round(mantissa*base) / base * math.Pow(10, exponent)
+
+	if math.IsNaN(roundedValue) {
+		return 0
+	}
+	return roundedValue
+}
+
+func formatFloatScientificNotation(in []byte, fVal float64, prec, bitSize int) []byte {
+	absVal := math.Abs(fVal)
+	if absVal > math.MaxFloat64 || math.IsNaN(absVal) {
+		return []byte{'0'}
+	}
+	if bitSize == 32 {
+		prec = defaultMySQLPrec
+	}
+	var out []byte
+	out = strconv.AppendFloat(in, fVal, 'e', prec, bitSize)
+	valStr := out[len(in):]
+	// remove the '+' from the string for compatibility.
+	plusPos := bytes.IndexByte(valStr, '+')
+	if plusPos > 0 {
+		plusPosInOut := len(in) + plusPos
+		out = append(out[:plusPosInOut], out[plusPosInOut+1:]...)
+	}
+	// remove extra '0'
+	ePos := bytes.IndexByte(valStr, 'e')
+	pointPos := bytes.IndexByte(valStr, '.')
+	ePosInOut := len(in) + ePos
+	pointPosInOut := len(in) + pointPos
+	validPos := ePosInOut
+	for i := ePosInOut - 1; i >= pointPosInOut; i-- {
+		if out[i] == '0' || out[i] == '.' {
+			validPos = i
+		} else {
+			break
 		}
-		out = append(out[:validPos], out[ePosInOut:]...)
-	} else {
-		out = strconv.AppendFloat(in, fVal, 'f', prec, bitSize)
 	}
+	out = append(out[:validPos], out[ePosInOut:]...)
+
 	return out
 }