*: Support modify table/column charset from latin1 to utf8/utf8mb4

cmd/explaintest/r/collation_misc_disabled.result

@@ -14,19 +14,57 @@ select * from t;
 a
 t_value
 alter table t modify column a varchar(20) charset utf8;
-Error 8200: Unsupported modify charset from latin1 to utf8
+admin check table t;
+
+select * from t;
+a
+t_value
+drop table t;
+create table t(a varchar(20) charset latin1);
+insert into t values ("t_value");
 alter table t modify column a varchar(20) charset utf8mb4;
-Error 8200: Unsupported modify charset from latin1 to utf8mb4
+admin check table t;
+
+select * from t;
+a
+t_value
+drop table t;
+create table t(a varchar(20) charset latin1);
+insert into t values ("t_value");
 alter table t modify column a varchar(20) charset utf8 collate utf8_bin;
-Error 8200: Unsupported modify charset from latin1 to utf8
+admin check table t;
+
+select * from t;
+a
+t_value
+drop table t;
+create table t(a varchar(20) charset latin1);
+insert into t values ("t_value");
 alter table t modify column a varchar(20) charset utf8mb4 collate utf8mb4_general_ci;
-Error 8200: Unsupported modify charset from latin1 to utf8mb4
 alter table t modify column a varchar(20) charset utf8mb4 collate utf8bin;
 [ddl:1273]Unknown collation: 'utf8bin'
 alter table t collate LATIN1_GENERAL_CI charset utf8 collate utf8_bin;
 Error 1302: Conflicting declarations: 'CHARACTER SET latin1' and 'CHARACTER SET utf8'
 alter table t collate LATIN1_GENERAL_CI collate UTF8MB4_UNICODE_ci collate utf8_bin;
 Error 1253: COLLATION 'utf8mb4_unicode_ci' is not valid for CHARACTER SET 'latin1'
+drop table t;
+create table t(a varchar(20) charset latin1);
+insert into t values ("t_value");
+alter table t modify column a varchar(19) charset utf8mb4;
+admin check table t;
+
+select * from t;
+a
+t_value
+drop table t;
+create table t(a varchar(20) charset latin1);
+insert into t values ("t_value");
+alter table t modify column a varchar(19) charset utf8 collate utf8_bin;
+admin check table t;
+
+select * from t;
+a
+t_value
 create database if not exists cd_test_utf8 CHARACTER SET utf8 COLLATE utf8_bin;
 create database if not exists cd_test_latin1 CHARACTER SET latin1 COLLATE latin1_swedish_ci;
 use cd_test_utf8;

cmd/explaintest/r/collation_misc_enabled.result

@@ -14,19 +14,57 @@ select * from t;
 a
 t_value
 alter table t modify column a varchar(20) charset utf8;
-Error 8200: Unsupported modify charset from latin1 to utf8
+admin check table t;
+
+select * from t;
+a
+t_value
+drop table t;
+create table t(a varchar(20) charset latin1);
+insert into t values ("t_value");
 alter table t modify column a varchar(20) charset utf8mb4;
-Error 8200: Unsupported modify charset from latin1 to utf8mb4
+admin check table t;
+
+select * from t;
+a
+t_value
+drop table t;
+create table t(a varchar(20) charset latin1);
+insert into t values ("t_value");
 alter table t modify column a varchar(20) charset utf8 collate utf8_bin;
-Error 8200: Unsupported modify charset from latin1 to utf8
+admin check table t;
+
+select * from t;
+a
+t_value
+drop table t;
+create table t(a varchar(20) charset latin1);
+insert into t values ("t_value");
 alter table t modify column a varchar(20) charset utf8mb4 collate utf8mb4_general_ci;
-Error 8200: Unsupported modify charset from latin1 to utf8mb4
 alter table t modify column a varchar(20) charset utf8mb4 collate utf8bin;
 [ddl:1273]Unknown collation: 'utf8bin'
 alter table t collate LATIN1_GENERAL_CI charset utf8 collate utf8_bin;
 Error 1273: Unsupported collation when new collation is enabled: 'latin1_general_ci'
 alter table t collate LATIN1_GENERAL_CI collate UTF8MB4_UNICODE_ci collate utf8_bin;
 Error 1273: Unsupported collation when new collation is enabled: 'latin1_general_ci'
+drop table t;
+create table t(a varchar(20) charset latin1);
+insert into t values ("t_value");
+alter table t modify column a varchar(19) charset utf8mb4;
+admin check table t;
+
+select * from t;
+a
+t_value
+drop table t;
+create table t(a varchar(20) charset latin1);
+insert into t values ("t_value");
+alter table t modify column a varchar(19) charset utf8 collate utf8_bin;
+admin check table t;
+
+select * from t;
+a
+t_value
 create database if not exists cd_test_utf8 CHARACTER SET utf8 COLLATE utf8_bin;
 create database if not exists cd_test_latin1 CHARACTER SET latin1 COLLATE latin1_swedish_ci;
 Error 1273: Unsupported collation when new collation is enabled: 'latin1_swedish_ci'

cmd/explaintest/t/collation_misc.test

@@ -12,12 +12,28 @@ create table t(a varchar(20) charset latin1);
 insert into t values ("t_value");
 alter table t modify column a varchar(20) charset latin1;
 select * from t;
---error 8200
+
 alter table t modify column a varchar(20) charset utf8;
---error 8200
+admin check table t;
+select * from t;
+
+drop table t;
+create table t(a varchar(20) charset latin1);
+insert into t values ("t_value");
 alter table t modify column a varchar(20) charset utf8mb4;
---error 8200
+admin check table t;
+select * from t;
+
+drop table t;
+create table t(a varchar(20) charset latin1);
+insert into t values ("t_value");
 alter table t modify column a varchar(20) charset utf8 collate utf8_bin;
+admin check table t;
+select * from t;
+
+drop table t;
+create table t(a varchar(20) charset latin1);
+insert into t values ("t_value");
 --error 8200
 alter table t modify column a varchar(20) charset utf8mb4 collate utf8mb4_general_ci;
 --error 1273
@@ -27,6 +43,21 @@ alter table t collate LATIN1_GENERAL_CI charset utf8 collate utf8_bin;
 --error 1253, 1273
 alter table t collate LATIN1_GENERAL_CI collate UTF8MB4_UNICODE_ci collate utf8_bin;
 
+# ChangingCharsetToUtf8 with reorg
+drop table t;
+create table t(a varchar(20) charset latin1);
+insert into t values ("t_value");
+alter table t modify column a varchar(19) charset utf8mb4;
+admin check table t;
+select * from t;
+
+drop table t;
+create table t(a varchar(20) charset latin1);
+insert into t values ("t_value");
+alter table t modify column a varchar(19) charset utf8 collate utf8_bin;
+admin check table t;
+select * from t;
+
 # TestCharsetDatabase
 create database if not exists cd_test_utf8 CHARACTER SET utf8 COLLATE utf8_bin;
 --error 1273

ddl/db_integration_test.go

@@ -760,7 +760,7 @@ func TestChangingTableCharset(t *testing.T) {
 	tk := testkit.NewTestKit(t, store)
 
 	tk.MustExec("USE test")
-	tk.MustExec("create table t(a char(10)) charset latin1 collate latin1_bin")
+	tk.MustExec("create table t(a char(10), index i(a)) charset latin1 collate latin1_bin")
 
 	tk.MustGetErrCode("alter table t charset gbk", errno.ErrUnsupportedDDLOperation)
 	tk.MustGetErrCode("alter table t charset ''", errno.ErrUnknownCharacterSet)
@@ -771,9 +771,18 @@ func TestChangingTableCharset(t *testing.T) {
 	tk.MustGetErrCode("alter table t charset utf8 collate utf8mb4_bin;", errno.ErrCollationCharsetMismatch)
 	tk.MustGetErrCode("alter table t charset utf8 collate utf8_bin collate utf8mb4_bin collate utf8_bin;", errno.ErrCollationCharsetMismatch)
 
-	tk.MustGetErrCode("alter table t charset utf8", errno.ErrUnsupportedDDLOperation)
-	tk.MustGetErrCode("alter table t charset utf8mb4", errno.ErrUnsupportedDDLOperation)
-	tk.MustGetErrCode("alter table t charset utf8mb4 collate utf8mb4_bin", errno.ErrUnsupportedDDLOperation)
+	tk.MustExec("alter table t charset utf8")
+	tk.MustExec("admin check table t")
+
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a char(10), index i(a)) charset latin1 collate latin1_bin")
+	tk.MustExec("alter table t charset utf8mb4")
+	tk.MustExec("admin check table t")
+
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a char(10), index i(a)) charset latin1 collate latin1_bin")
+	tk.MustExec("alter table t charset utf8mb4 collate utf8mb4_bin")
+	tk.MustExec("admin check table t")
 
 	tk.MustGetErrCode("alter table t charset latin1 charset utf8 charset utf8mb4 collate utf8_bin;", errno.ErrConflictingDeclarations)
 
@@ -793,6 +802,13 @@ func TestChangingTableCharset(t *testing.T) {
 	}
 	checkCharset(charset.CharsetUTF8MB4, charset.CollationUTF8MB4)
 
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a varchar(20), key i(a)) charset=latin1")
+	tk.MustGetErrCode("alter table t convert to charset utf8 collate utf8_unicode_ci", errno.ErrUnsupportedDDLOperation)
+	tk.MustGetErrCode("alter table t convert to charset utf8 collate utf8_general_ci", errno.ErrUnsupportedDDLOperation)
+	tk.MustExec("alter table t convert to charset utf8 collate utf8_bin")
+	tk.MustGetErrCode("alter table t convert to charset latin1", errno.ErrUnsupportedDDLOperation)
+
 	// Test when column charset can not convert to the target charset.
 	tk.MustExec("drop table t;")
 	tk.MustExec("create table t(a varchar(10) character set ascii) charset utf8mb4")

ddl/ddl_api.go

@@ -4177,8 +4177,10 @@ func checkModifyCharsetAndCollation(toCharset, toCollate, origCharset, origColla
 
 	if (origCharset == charset.CharsetUTF8 && toCharset == charset.CharsetUTF8MB4) ||
 		(origCharset == charset.CharsetUTF8 && toCharset == charset.CharsetUTF8) ||
-		(origCharset == charset.CharsetUTF8MB4 && toCharset == charset.CharsetUTF8MB4) {
-		// TiDB only allow utf8 to be changed to utf8mb4, or changing the collation when the charset is utf8/utf8mb4.
+		(origCharset == charset.CharsetUTF8MB4 && toCharset == charset.CharsetUTF8MB4) ||
+		(origCharset == charset.CharsetLatin1 && toCharset == charset.CharsetUTF8) ||
+		(origCharset == charset.CharsetLatin1 && toCharset == charset.CharsetUTF8MB4) {
+		// TiDB only allow utf8/latin1 to be changed to utf8mb4, or changing the collation when the charset is utf8/utf8mb4/latin1.
 		return nil
 	}
 

expression/integration_test.go

@@ -648,7 +648,7 @@ func TestStringBuiltin(t *testing.T) {
 	result = tk.MustQuery("select ord('123'), ord(123), ord(''), ord('你好'), ord(NULL), ord('👍')")
 	result.Check(testkit.Rows("49 49 0 14990752 <nil> 4036989325"))
 	result = tk.MustQuery("select ord(X''), ord(X'6161'), ord(X'e4bd'), ord(X'e4bda0'), ord(_ascii'你'), ord(_latin1'你')")
-	result.Check(testkit.Rows("0 97 228 228 228 228"))
+	result.Check(testkit.Rows("0 97 228 228 228 14990752"))
 
 	// for space
 	result = tk.MustQuery(`select space(0), space(2), space(-1), space(1.1), space(1.9)`)

parser/charset/encoding_latin1.go

@@ -14,13 +14,11 @@
 package charset
 
 import (
-	"bytes"
-
 	"golang.org/x/text/encoding"
 )
 
 // EncodingLatin1Impl is the instance of encodingLatin1.
-// TiDB uses utf8 implementation for latin1 charset because of the backward compatibility.
+// In TiDB, latin1 is an alias for utf8, so uses utf8 implementation for latin1.
 var EncodingLatin1Impl = &encodingLatin1{encodingUTF8{encodingBase{enc: encoding.Nop}}}
 
 func init() {
@@ -36,25 +34,3 @@ type encodingLatin1 struct {
 func (e *encodingLatin1) Name() string {
 	return CharsetLatin1
 }
-
-// Peek implements Encoding interface.
-func (e *encodingLatin1) Peek(src []byte) []byte {
-	if len(src) == 0 {
-		return src
-	}
-	return src[:1]
-}
-
-// IsValid implements Encoding interface.
-func (e *encodingLatin1) IsValid(src []byte) bool {
-	return true
-}
-
-// Tp implements Encoding interface.
-func (e *encodingLatin1) Tp() EncodingTp {
-	return EncodingTpLatin1
-}
-
-func (e *encodingLatin1) Transform(dest *bytes.Buffer, src []byte, op Op) ([]byte, error) {
-	return src, nil
-}

parser/mysql/charset.go

@@ -574,6 +574,7 @@ var CollationNames = map[string]uint8{
 const (
 	UTF8Charset    = "utf8"
 	UTF8MB4Charset = "utf8mb4"
+	Latin1Charset  = "latin1"
 	DefaultCharset = UTF8MB4Charset
 	// DefaultCollationID is utf8mb4_bin(46)
 	DefaultCollationID        = 46
@@ -592,9 +593,9 @@ const (
 	MaxBytesOfCharacter = 4
 )
 
-// IsUTF8Charset checks if charset is utf8 or utf8mb4
+// IsUTF8Charset checks if charset is utf8, utf8mb4 or latin1.
 func IsUTF8Charset(charset string) bool {
-	return charset == UTF8Charset || charset == UTF8MB4Charset
+	return charset == UTF8Charset || charset == UTF8MB4Charset || charset == Latin1Charset
 }
 
 // RangeGraph defines valid unicode characters to use in column names. It strictly follows MySQL's definition.

util/collate/collate.go

@@ -98,7 +98,7 @@ func NewCollationEnabled() bool {
 func CompatibleCollate(collate1, collate2 string) bool {
 	if (collate1 == "utf8mb4_general_ci" || collate1 == "utf8_general_ci") && (collate2 == "utf8mb4_general_ci" || collate2 == "utf8_general_ci") {
 		return true
-	} else if (collate1 == "utf8mb4_bin" || collate1 == "utf8_bin") && (collate2 == "utf8mb4_bin" || collate2 == "utf8_bin") {
+	} else if (collate1 == "utf8mb4_bin" || collate1 == "utf8_bin" || collate1 == "latin1_bin") && (collate2 == "utf8mb4_bin" || collate2 == "utf8_bin") {
 		return true
 	} else if (collate1 == "utf8mb4_unicode_ci" || collate1 == "utf8_unicode_ci") && (collate2 == "utf8mb4_unicode_ci" || collate2 == "utf8_unicode_ci") {
 		return true