expression: cast charset according to the function's resulting charset (

#29905)
pingcap · Nov 24, 2021 · 3d26751 · 3d26751
1 parent a90c988
commit 3d26751
Show file tree

Hide file tree

Showing 16 changed files with 495 additions and 45 deletions.
diff --git a/cmd/explaintest/r/new_character_set_builtin.result b/cmd/explaintest/r/new_character_set_builtin.result
@@ -286,3 +286,207 @@ select ord(a), ord(b), ord(c) from t;
 ord(a)	ord(b)	ord(c)
 14989485	54992	228
 set @@tidb_enable_vectorized_expression = false;
+drop table if exists t;
+create table t (a char(20) charset utf8mb4, b char(20) charset gbk, c binary(20));
+insert into t values ('一', '一', 0xe4b880);
+insert into t values ('一', '一', 0xd2bb);
+insert into t values ('一', '一', 0xe4ba8c);
+insert into t values ('一', '一', 0xb6fe);
+set @@tidb_enable_vectorized_expression = true;
+select hex(concat(a, c)), hex(concat(b, c)) from t;
+hex(concat(a, c))	hex(concat(b, c))
+E4B880E4B8800000000000000000000000000000000000	D2BBE4B8800000000000000000000000000000000000
+E4B880D2BB000000000000000000000000000000000000	D2BBD2BB000000000000000000000000000000000000
+E4B880E4BA8C0000000000000000000000000000000000	D2BBE4BA8C0000000000000000000000000000000000
+E4B880B6FE000000000000000000000000000000000000	D2BBB6FE000000000000000000000000000000000000
+select hex(concat(a, 0xe4b880)), hex(concat(b, 0xd2bb)) from t;
+hex(concat(a, 0xe4b880))	hex(concat(b, 0xd2bb))
+E4B880E4B880	D2BBD2BB
+E4B880E4B880	D2BBD2BB
+E4B880E4B880	D2BBD2BB
+E4B880E4B880	D2BBD2BB
+select a = 0xe4b880, b = 0xd2bb from t;
+a = 0xe4b880	b = 0xd2bb
+1	1
+1	1
+1	1
+1	1
+select a = c, b = c from t;
+a = c	b = c
+0	0
+0	0
+0	0
+0	0
+select hex(insert(a, 1, 2, 0xe4ba8c)), hex(insert(b, 1, 2, 0xb6fe)) from t;
+hex(insert(a, 1, 2, 0xe4ba8c))	hex(insert(b, 1, 2, 0xb6fe))
+E4BA8C	B6FE
+E4BA8C	B6FE
+E4BA8C	B6FE
+E4BA8C	B6FE
+select hex(insert(a, 1, 2, c)), hex(insert(b, 1, 2, c)) from t;
+hex(insert(a, 1, 2, c))	hex(insert(b, 1, 2, c))
+E4B880000000000000000000000000000000000080	E4B8800000000000000000000000000000000000
+D2BB00000000000000000000000000000000000080	D2BB000000000000000000000000000000000000
+E4BA8C000000000000000000000000000000000080	E4BA8C0000000000000000000000000000000000
+B6FE00000000000000000000000000000000000080	B6FE000000000000000000000000000000000000
+select hex(lpad(a, 5, 0xe4ba8c)), hex(lpad(b, 5, 0xb6fe)) from t;
+hex(lpad(a, 5, 0xe4ba8c))	hex(lpad(b, 5, 0xb6fe))
+E4BA8CE4BA8CE4BA8CE4BA8CE4B880	B6FEB6FEB6FEB6FED2BB
+E4BA8CE4BA8CE4BA8CE4BA8CE4B880	B6FEB6FEB6FEB6FED2BB
+E4BA8CE4BA8CE4BA8CE4BA8CE4B880	B6FEB6FEB6FEB6FED2BB
+E4BA8CE4BA8CE4BA8CE4BA8CE4B880	B6FEB6FEB6FEB6FED2BB
+select hex(lpad(a, 5, c)), hex(lpad(b, 5, c)) from t;
+hex(lpad(a, 5, c))	hex(lpad(b, 5, c))
+E4B8E4B880	E4B880D2BB
+D2BBE4B880	D2BB00D2BB
+E4BAE4B880	E4BA8CD2BB
+B6FEE4B880	B6FE00D2BB
+select hex(rpad(a, 5, 0xe4ba8c)), hex(rpad(b, 5, 0xb6fe)) from t;
+hex(rpad(a, 5, 0xe4ba8c))	hex(rpad(b, 5, 0xb6fe))
+E4B880E4BA8CE4BA8CE4BA8CE4BA8C	D2BBB6FEB6FEB6FEB6FE
+E4B880E4BA8CE4BA8CE4BA8CE4BA8C	D2BBB6FEB6FEB6FEB6FE
+E4B880E4BA8CE4BA8CE4BA8CE4BA8C	D2BBB6FEB6FEB6FEB6FE
+E4B880E4BA8CE4BA8CE4BA8CE4BA8C	D2BBB6FEB6FEB6FEB6FE
+select hex(rpad(a, 5, c)), hex(rpad(b, 5, c)) from t;
+hex(rpad(a, 5, c))	hex(rpad(b, 5, c))
+E4B880E4B8	D2BBE4B880
+E4B880D2BB	D2BBD2BB00
+E4B880E4BA	D2BBE4BA8C
+E4B880B6FE	D2BBB6FE00
+select hex(elt(2, a, 0xe4ba8c)), hex(elt(2, b, 0xb6fe)) from t;
+hex(elt(2, a, 0xe4ba8c))	hex(elt(2, b, 0xb6fe))
+E4BA8C	B6FE
+E4BA8C	B6FE
+E4BA8C	B6FE
+E4BA8C	B6FE
+select hex(elt(2, a, c)), hex(elt(2, b, c)) from t;
+hex(elt(2, a, c))	hex(elt(2, b, c))
+E4B8800000000000000000000000000000000000	E4B8800000000000000000000000000000000000
+D2BB000000000000000000000000000000000000	D2BB000000000000000000000000000000000000
+E4BA8C0000000000000000000000000000000000	E4BA8C0000000000000000000000000000000000
+B6FE000000000000000000000000000000000000	B6FE000000000000000000000000000000000000
+select hex(instr(a, 0xe4b880)), hex(instr(b, 0xd2bb)) from t;
+hex(instr(a, 0xe4b880))	hex(instr(b, 0xd2bb))
+1	1
+1	1
+1	1
+1	1
+select hex(position(a in 0xe4b880)), hex(position(b in 0xd2bb)) from t;
+hex(position(a in 0xe4b880))	hex(position(b in 0xd2bb))
+1	1
+1	1
+1	1
+1	1
+select a like 0xe4b880, b like 0xd2bb from t;
+a like 0xe4b880	b like 0xd2bb
+1	1
+1	1
+1	1
+1	1
+select a = 0xb6fe from t;
+Error 3854: Cannot convert string 'B6FE' from binary to utf8mb4
+select b = 0xe4ba8c from t;
+Error 3854: Cannot convert string 'E4BA8C' from binary to gbk
+select concat(a, 0xb6fe) from t;
+Error 3854: Cannot convert string 'B6FE' from binary to utf8mb4
+select concat(b, 0xe4ba8c) from t;
+Error 3854: Cannot convert string 'E4BA8C' from binary to gbk
+set @@tidb_enable_vectorized_expression = false;
+select hex(concat(a, c)), hex(concat(b, c)) from t;
+hex(concat(a, c))	hex(concat(b, c))
+E4B880E4B8800000000000000000000000000000000000	D2BBE4B8800000000000000000000000000000000000
+E4B880D2BB000000000000000000000000000000000000	D2BBD2BB000000000000000000000000000000000000
+E4B880E4BA8C0000000000000000000000000000000000	D2BBE4BA8C0000000000000000000000000000000000
+E4B880B6FE000000000000000000000000000000000000	D2BBB6FE000000000000000000000000000000000000
+select hex(concat(a, 0xe4b880)), hex(concat(b, 0xd2bb)) from t;
+hex(concat(a, 0xe4b880))	hex(concat(b, 0xd2bb))
+E4B880E4B880	D2BBD2BB
+E4B880E4B880	D2BBD2BB
+E4B880E4B880	D2BBD2BB
+E4B880E4B880	D2BBD2BB
+select a = 0xe4b880, b = 0xd2bb from t;
+a = 0xe4b880	b = 0xd2bb
+1	1
+1	1
+1	1
+1	1
+select a = c, b = c from t;
+a = c	b = c
+0	0
+0	0
+0	0
+0	0
+select hex(insert(a, 1, 2, 0xe4ba8c)), hex(insert(b, 1, 2, 0xb6fe)) from t;
+hex(insert(a, 1, 2, 0xe4ba8c))	hex(insert(b, 1, 2, 0xb6fe))
+E4BA8C	B6FE
+E4BA8C	B6FE
+E4BA8C	B6FE
+E4BA8C	B6FE
+select hex(insert(a, 1, 2, c)), hex(insert(b, 1, 2, c)) from t;
+hex(insert(a, 1, 2, c))	hex(insert(b, 1, 2, c))
+E4B880000000000000000000000000000000000080	E4B8800000000000000000000000000000000000
+D2BB00000000000000000000000000000000000080	D2BB000000000000000000000000000000000000
+E4BA8C000000000000000000000000000000000080	E4BA8C0000000000000000000000000000000000
+B6FE00000000000000000000000000000000000080	B6FE000000000000000000000000000000000000
+select hex(lpad(a, 5, 0xe4ba8c)), hex(lpad(b, 5, 0xb6fe)) from t;
+hex(lpad(a, 5, 0xe4ba8c))	hex(lpad(b, 5, 0xb6fe))
+E4BA8CE4BA8CE4BA8CE4BA8CE4B880	B6FEB6FEB6FEB6FED2BB
+E4BA8CE4BA8CE4BA8CE4BA8CE4B880	B6FEB6FEB6FEB6FED2BB
+E4BA8CE4BA8CE4BA8CE4BA8CE4B880	B6FEB6FEB6FEB6FED2BB
+E4BA8CE4BA8CE4BA8CE4BA8CE4B880	B6FEB6FEB6FEB6FED2BB
+select hex(lpad(a, 5, c)), hex(lpad(b, 5, c)) from t;
+hex(lpad(a, 5, c))	hex(lpad(b, 5, c))
+E4B8E4B880	E4B880D2BB
+D2BBE4B880	D2BB00D2BB
+E4BAE4B880	E4BA8CD2BB
+B6FEE4B880	B6FE00D2BB
+select hex(rpad(a, 5, 0xe4ba8c)), hex(rpad(b, 5, 0xb6fe)) from t;
+hex(rpad(a, 5, 0xe4ba8c))	hex(rpad(b, 5, 0xb6fe))
+E4B880E4BA8CE4BA8CE4BA8CE4BA8C	D2BBB6FEB6FEB6FEB6FE
+E4B880E4BA8CE4BA8CE4BA8CE4BA8C	D2BBB6FEB6FEB6FEB6FE
+E4B880E4BA8CE4BA8CE4BA8CE4BA8C	D2BBB6FEB6FEB6FEB6FE
+E4B880E4BA8CE4BA8CE4BA8CE4BA8C	D2BBB6FEB6FEB6FEB6FE
+select hex(rpad(a, 5, c)), hex(rpad(b, 5, c)) from t;
+hex(rpad(a, 5, c))	hex(rpad(b, 5, c))
+E4B880E4B8	D2BBE4B880
+E4B880D2BB	D2BBD2BB00
+E4B880E4BA	D2BBE4BA8C
+E4B880B6FE	D2BBB6FE00
+select hex(elt(2, a, 0xe4ba8c)), hex(elt(2, b, 0xb6fe)) from t;
+hex(elt(2, a, 0xe4ba8c))	hex(elt(2, b, 0xb6fe))
+E4BA8C	B6FE
+E4BA8C	B6FE
+E4BA8C	B6FE
+E4BA8C	B6FE
+select hex(elt(2, a, c)), hex(elt(2, b, c)) from t;
+hex(elt(2, a, c))	hex(elt(2, b, c))
+E4B8800000000000000000000000000000000000	E4B8800000000000000000000000000000000000
+D2BB000000000000000000000000000000000000	D2BB000000000000000000000000000000000000
+E4BA8C0000000000000000000000000000000000	E4BA8C0000000000000000000000000000000000
+B6FE000000000000000000000000000000000000	B6FE000000000000000000000000000000000000
+select hex(instr(a, 0xe4b880)), hex(instr(b, 0xd2bb)) from t;
+hex(instr(a, 0xe4b880))	hex(instr(b, 0xd2bb))
+1	1
+1	1
+1	1
+1	1
+select hex(position(a in 0xe4b880)), hex(position(b in 0xd2bb)) from t;
+hex(position(a in 0xe4b880))	hex(position(b in 0xd2bb))
+1	1
+1	1
+1	1
+1	1
+select a like 0xe4b880, b like 0xd2bb from t;
+a like 0xe4b880	b like 0xd2bb
+1	1
+1	1
+1	1
+1	1
+select a = 0xb6fe from t;
+Error 3854: Cannot convert string 'B6FE' from binary to utf8mb4
+select b = 0xe4ba8c from t;
+Error 3854: Cannot convert string 'E4BA8C' from binary to gbk
+select concat(a, 0xb6fe) from t;
+Error 3854: Cannot convert string 'B6FE' from binary to utf8mb4
+select concat(b, 0xe4ba8c) from t;
+Error 3854: Cannot convert string 'E4BA8C' from binary to gbk
diff --git a/cmd/explaintest/r/select.result b/cmd/explaintest/r/select.result
@@ -495,3 +495,7 @@ insert into precise_types values (
 SELECT a, b, c, d FROM precise_types;
 a	b	c	d
 18446744073709551614	-9223372036854775806	99999999999999999999.0	1.8446744073709552e19
+create table t3(a char(10), primary key (a));
+insert into t3 values ('a');
+select * from t3 where a > 0x80;
+Error 1105: Cannot convert string '80' from binary to utf8mb4
diff --git a/cmd/explaintest/t/new_character_set_builtin.test b/cmd/explaintest/t/new_character_set_builtin.test
@@ -161,3 +161,62 @@ select ord(a), ord(b), ord(c) from t;
 set @@tidb_enable_vectorized_expression = true;
 select ord(a), ord(b), ord(c) from t;
 set @@tidb_enable_vectorized_expression = false;
+
+drop table if exists t;
+create table t (a char(20) charset utf8mb4, b char(20) charset gbk, c binary(20));
+insert into t values ('一', '一', 0xe4b880);
+insert into t values ('一', '一', 0xd2bb);
+insert into t values ('一', '一', 0xe4ba8c);
+insert into t values ('一', '一', 0xb6fe);
+
+set @@tidb_enable_vectorized_expression = true;
+select hex(concat(a, c)), hex(concat(b, c)) from t;
+select hex(concat(a, 0xe4b880)), hex(concat(b, 0xd2bb)) from t;
+select a = 0xe4b880, b = 0xd2bb from t;
+select a = c, b = c from t;
+select hex(insert(a, 1, 2, 0xe4ba8c)), hex(insert(b, 1, 2, 0xb6fe)) from t;
+select hex(insert(a, 1, 2, c)), hex(insert(b, 1, 2, c)) from t;
+select hex(lpad(a, 5, 0xe4ba8c)), hex(lpad(b, 5, 0xb6fe)) from t;
+select hex(lpad(a, 5, c)), hex(lpad(b, 5, c)) from t;
+select hex(rpad(a, 5, 0xe4ba8c)), hex(rpad(b, 5, 0xb6fe)) from t;
+select hex(rpad(a, 5, c)), hex(rpad(b, 5, c)) from t;
+select hex(elt(2, a, 0xe4ba8c)), hex(elt(2, b, 0xb6fe)) from t;
+select hex(elt(2, a, c)), hex(elt(2, b, c)) from t;
+select hex(instr(a, 0xe4b880)), hex(instr(b, 0xd2bb)) from t;
+select hex(position(a in 0xe4b880)), hex(position(b in 0xd2bb)) from t;
+select a like 0xe4b880, b like 0xd2bb from t;
+
+--error ER_CANNOT_CONVERT_STRING
+select a = 0xb6fe from t;
+--error ER_CANNOT_CONVERT_STRING
+select b = 0xe4ba8c from t;
+--error ER_CANNOT_CONVERT_STRING
+select concat(a, 0xb6fe) from t;
+--error ER_CANNOT_CONVERT_STRING
+select concat(b, 0xe4ba8c) from t;
+
+set @@tidb_enable_vectorized_expression = false;
+select hex(concat(a, c)), hex(concat(b, c)) from t;
+select hex(concat(a, 0xe4b880)), hex(concat(b, 0xd2bb)) from t;
+select a = 0xe4b880, b = 0xd2bb from t;
+select a = c, b = c from t;
+select hex(insert(a, 1, 2, 0xe4ba8c)), hex(insert(b, 1, 2, 0xb6fe)) from t;
+select hex(insert(a, 1, 2, c)), hex(insert(b, 1, 2, c)) from t;
+select hex(lpad(a, 5, 0xe4ba8c)), hex(lpad(b, 5, 0xb6fe)) from t;
+select hex(lpad(a, 5, c)), hex(lpad(b, 5, c)) from t;
+select hex(rpad(a, 5, 0xe4ba8c)), hex(rpad(b, 5, 0xb6fe)) from t;
+select hex(rpad(a, 5, c)), hex(rpad(b, 5, c)) from t;
+select hex(elt(2, a, 0xe4ba8c)), hex(elt(2, b, 0xb6fe)) from t;
+select hex(elt(2, a, c)), hex(elt(2, b, c)) from t;
+select hex(instr(a, 0xe4b880)), hex(instr(b, 0xd2bb)) from t;
+select hex(position(a in 0xe4b880)), hex(position(b in 0xd2bb)) from t;
+select a like 0xe4b880, b like 0xd2bb from t;
+
+--error ER_CANNOT_CONVERT_STRING
+select a = 0xb6fe from t;
+--error ER_CANNOT_CONVERT_STRING
+select b = 0xe4ba8c from t;
+--error ER_CANNOT_CONVERT_STRING
+select concat(a, 0xb6fe) from t;
+--error ER_CANNOT_CONVERT_STRING
+select concat(b, 0xe4ba8c) from t;
diff --git a/cmd/explaintest/t/select.test b/cmd/explaintest/t/select.test
@@ -244,3 +244,8 @@ insert into precise_types values (
     18446744073709551614
 );
 SELECT a, b, c, d FROM precise_types;
+
+create table t3(a char(10), primary key (a));
+insert into t3 values ('a');
+--error ER_CANNOT_CONVERT_STRING
+select * from t3 where a > 0x80;
diff --git a/errno/errcode.go b/errno/errcode.go
@@ -901,6 +901,7 @@ const (
 	ErrFKIncompatibleColumns                                 = 3780
 	ErrFunctionalIndexRowValueIsNotAllowed                   = 3800
 	ErrDependentByFunctionalIndex                            = 3837
+	ErrCannotConvertString                                   = 3854
 	ErrInvalidJSONValueForFuncIndex                          = 3903
 	ErrJSONValueOutOfRangeForFuncIndex                       = 3904
 	ErrFunctionalIndexDataIsTooLong                          = 3907

diff --git a/errno/errname.go b/errno/errname.go
@@ -896,6 +896,7 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
 	ErrFKIncompatibleColumns:                                 mysql.Message("Referencing column '%s' in foreign key constraint '%s' are incompatible", nil),
 	ErrFunctionalIndexRowValueIsNotAllowed:                   mysql.Message("Expression of expression index '%s' cannot refer to a row value", nil),
 	ErrDependentByFunctionalIndex:                            mysql.Message("Column '%s' has an expression index dependency and cannot be dropped or renamed", nil),
+	ErrCannotConvertString:                                   mysql.Message("Cannot convert string '%.64s' from %s to %s", nil),
 	ErrInvalidJSONValueForFuncIndex:                          mysql.Message("Invalid JSON value for CAST for expression index '%s'", nil),
 	ErrJSONValueOutOfRangeForFuncIndex:                       mysql.Message("Out of range JSON value for CAST for expression index '%s'", nil),
 	ErrFunctionalIndexDataIsTooLong:                          mysql.Message("Data too long for expression index '%s'", nil),

diff --git a/expression/builtin.go b/expression/builtin.go
@@ -140,7 +140,7 @@ func newBaseBuiltinFuncWithTp(ctx sessionctx.Context, funcName string, args []Ex
 			args[i] = WrapWithCastAsDecimal(ctx, args[i])
 		case types.ETString:
 			args[i] = WrapWithCastAsString(ctx, args[i])
-			args[i] = WrapWithToBinary(ctx, args[i], funcName)
+			args[i] = HandleBinaryLiteral(ctx, args[i], ec, funcName)
 		case types.ETDatetime:
 			args[i] = WrapWithCastAsTime(ctx, args[i], types.NewFieldType(mysql.TypeDatetime))
 		case types.ETTimestamp:
@@ -880,9 +880,6 @@ var funcs = map[string]functionClass{
 	ast.NextVal: &nextValFunctionClass{baseFunctionClass{ast.NextVal, 1, 1}},
 	ast.LastVal: &lastValFunctionClass{baseFunctionClass{ast.LastVal, 1, 1}},
 	ast.SetVal:  &setValFunctionClass{baseFunctionClass{ast.SetVal, 2, 2}},
-
-	// TiDB implicit internal functions.
-	InternalFuncToBinary: &tidbConvertCharsetFunctionClass{baseFunctionClass{InternalFuncToBinary, 1, 1}},
 }
 
 // IsFunctionSupported check if given function name is a builtin sql function.
@@ -906,7 +903,6 @@ func GetDisplayName(name string) string {
 func GetBuiltinList() []string {
 	res := make([]string, 0, len(funcs))
 	notImplementedFunctions := []string{ast.RowFunc, ast.IsTruthWithNull}
-	implicitFunctions := []string{InternalFuncToBinary}
 	for funcName := range funcs {
 		skipFunc := false
 		// Skip not implemented functions
@@ -915,11 +911,6 @@ func GetBuiltinList() []string {
 				skipFunc = true
 			}
 		}
-		for _, implicitFunc := range implicitFunctions {
-			if funcName == implicitFunc {
-				skipFunc = true
-			}
-		}
 		// Skip literal functions
 		// (their names are not readable: 'tidb`.(dateliteral, for example)
 		// See: https://github.com/pingcap/parser/pull/591