Merge pull request #8322 from planetscale/online-ddl-vrepl-suite-non-…

…utf-text

Online DDL/VReplication: support non-UTF8 character sets

go/mysql/constants.go

@@ -16,7 +16,13 @@ limitations under the License.
 
 package mysql
 
-import "strings"
+import (
+ "strings"
+
+ "golang.org/x/text/encoding"
+ "golang.org/x/text/encoding/charmap"
+ "golang.org/x/text/encoding/simplifiedchinese"
+)
 
 const (
  // MaxPacketSize is the maximum payload length of a packet
@@ -626,6 +632,41 @@ var CharacterSetMap = map[string]uint8{
  "eucjpms": 97,
 }
 
+// CharacterSetEncoding maps a charset name to a golang encoder.
+// golang does not support encoders for all MySQL charsets.
+// A charset not in this map is unsupported.
+// A trivial encoding (e.g. utf8) has a `nil` encoder
+var CharacterSetEncoding = map[string]encoding.Encoding{
+ "cp850": charmap.CodePage850,
+ "koi8r": charmap.KOI8R,
+ "latin1": charmap.Windows1252,
+ "latin2": charmap.ISO8859_2,
+ "ascii": nil,
+ "hebrew": charmap.ISO8859_8,
+ "greek": charmap.ISO8859_7,
+ "cp1250": charmap.Windows1250,
+ "gbk": simplifiedchinese.GBK,
+ "latin5": charmap.ISO8859_9,
+ "utf8": nil,
+ "cp866": charmap.CodePage866,
+ "cp852": charmap.CodePage852,
+ "latin7": charmap.ISO8859_13,
+ "utf8mb4": nil,
+ "cp1251": charmap.Windows1251,
+ "cp1256": charmap.Windows1256,
+ "cp1257": charmap.Windows1257,
+ "binary": nil,
+}
+
+// ReverseCharacterSetMap maps a charset integer code to charset name
+var ReverseCharacterSetMap = map[uint8]string{}
+
+func init() {
+ for c, i := range CharacterSetMap {
+ ReverseCharacterSetMap[i] = c
+ }
+}
+
 // IsNum returns true if a MySQL type is a numeric value.
 // It is the same as IS_NUM defined in mysql.h.
 func IsNum(typ uint8) bool {

go/test/endtoend/onlineddl/vrepl_suite/testdata/alter-charset-non-utf8-80/create.sql

@@ -0,0 +1,30 @@
+drop table if exists onlineddl_test;
+create table onlineddl_test (
+ id int auto_increment,
+ t1 varchar(128) charset latin1 collate latin1_swedish_ci,
+ t2 varchar(128) charset latin1 collate latin1_swedish_ci,
+ tutf8 varchar(128) charset utf8,
+ tutf8mb4 varchar(128) charset utf8mb4,
+ tlatin1 varchar(128) charset latin1 collate latin1_swedish_ci,
+ primary key(id)
+) auto_increment=1;
+
+insert into onlineddl_test values (null, md5(rand()), md5(rand()), md5(rand()), md5(rand()), md5(rand()));
+insert into onlineddl_test values (null, 'átesting', 'átesting', 'átesting', 'átesting', 'átesting');
+insert into onlineddl_test values (null, 'testátest', 'testátest', 'testátest', '🍻😀', 'átesting');
+
+drop event if exists onlineddl_test;
+delimiter ;;
+create event onlineddl_test
+ on schedule every 1 second
+ starts current_timestamp
+ ends current_timestamp + interval 60 second
+ on completion not preserve
+ enable
+ do
+begin
+ insert into onlineddl_test values (null, md5(rand()), md5(rand()), md5(rand()), md5(rand()), md5(rand()));
+ insert into onlineddl_test values (null, 'átesting-binlog', 'átesting-binlog', 'átesting-binlog', 'átesting-binlog', 'átesting-binlog');
+ insert into onlineddl_test values (null, 'testátest-binlog', 'testátest-binlog', 'testátest-binlog', '🍻😀', 'átesting-binlog');
+ insert into onlineddl_test values (null, 'átesting-bnull', 'átesting-bnull', 'átesting-bnull', null, null);
+end ;;

go/test/endtoend/onlineddl/vrepl_suite/testdata/alter-charset-non-utf8-80/ignore_versions

@@ -0,0 +1 @@
+(5.5|5.6|5.7)

go/test/endtoend/onlineddl/vrepl_suite/untestdata/alter-charset-all-dml/create.sql → go/test/endtoend/onlineddl/vrepl_suite/testdata/alter-charset-non-utf8/create.sql

@@ -5,10 +5,13 @@ create table onlineddl_test (
  t2 varchar(128) charset latin1 collate latin1_swedish_ci,
  tutf8 varchar(128) charset utf8,
  tutf8mb4 varchar(128) charset utf8mb4,
- random_value varchar(128) charset ascii,
+ tlatin1 varchar(128)  charset latin1 collate latin1_swedish_ci,
  primary key(id)
 ) auto_increment=1;
 
+insert into onlineddl_test values (null, md5(rand()), md5(rand()), md5(rand()), md5(rand()), md5(rand()));
+insert into onlineddl_test values (null, 'átesting', 'átesting', 'átesting', 'átesting', 'átesting');
+
 drop event if exists onlineddl_test;
 delimiter ;;
 create event onlineddl_test
@@ -20,9 +23,6 @@ create event onlineddl_test
  do
 begin
  insert into onlineddl_test values (null, md5(rand()), md5(rand()), md5(rand()), md5(rand()), md5(rand()));
- insert into onlineddl_test values (null, 'átesting', 'átesting', 'átesting', 'átesting', md5(rand()));
- insert into onlineddl_test values (null, 'átesting_del', 'átesting', 'átesting', 'átesting', md5(rand()));
- insert into onlineddl_test values (null, 'testátest', 'testátest', 'testátest', '🍻😀', md5(rand()));
- update onlineddl_test set t1='átesting2' where t1='átesting' order by id desc limit 1;
- delete from onlineddl_test where t1='átesting_del' order by id desc limit 1;
+ insert into onlineddl_test values (null, 'átesting-binlog', 'átesting-binlog', 'átesting-binlog', 'átesting-binlog', 'átesting-binlog');
+ insert into onlineddl_test values (null, 'átesting-bnull', 'átesting-bnull', 'átesting-bnull', null, null);
 end ;;

go/test/endtoend/onlineddl/vrepl_suite/testdata/alter-charset-non-utf8/ignore_versions

@@ -0,0 +1 @@
+(5.5|5.6)

go/test/endtoend/onlineddl/vrepl_suite/untestdata/convert-utf8mb4/create.sql → go/test/endtoend/onlineddl/vrepl_suite/testdata/convert-utf8mb4/create.sql

@@ -7,9 +7,6 @@ create table onlineddl_test (
  primary key(id)
 ) auto_increment=1;
 
-insert into onlineddl_test values (null, 'átesting');
-
-
 insert into onlineddl_test values (null, 'Hello world, Καλημέρα κόσμε, コンニチハ', 'átesting0', 'initial');
 
 drop event if exists onlineddl_test;