lightning: add a integration test for gbk (#31337)

close #31293

br/pkg/lightning/restore/restore.go

@@ -1817,6 +1817,7 @@ func (rc *Controller) setGlobalVariables(ctx context.Context) error {
 	// we should enable/disable new collation here since in server mode, tidb config
 	// may be different in different tasks
 	collate.SetNewCollationEnabledForTest(enabled)
+	log.L().Info("new_collation_enabled", zap.Bool("enabled", enabled))
 
 	return nil
 }

br/tests/lightning_new_collation/data-gbk/nc.gbk_source-schema.sql

@@ -0,0 +1,5 @@
+CREATE TABLE `gbk_source` (
+	`id` BIGINT PRIMARY KEY COMMENT "×ÔÔö ID",
+	`v` VARCHAR(32) COMMENT "ÖÐÎÄÆÀÂÛ",
+	KEY `idx_gbk` (`v`, `id`)
+) DEFAULT CHARSET=gbk COLLATE=gbk_chinese_ci;

br/tests/lightning_new_collation/data-gbk/nc.gbk_source.0.csv

@@ -0,0 +1,4 @@
+id,v
+1,����
+2,ŶŶ
+3,��������

br/tests/lightning_new_collation/data/nc-schema-create.sql

@@ -0,0 +1 @@
+CREATE DATABASE nc;

br/tests/lightning_new_collation/data/nc.gbk_test-schema.sql

@@ -0,0 +1,7 @@
+CREATE TABLE `gbk_test` (
+      `id` int(11) DEFAULT NULL,
+      `v` varchar(32) DEFAULT NULL,
+      `v2` varchar(32) CHARACTER SET gbk COLLATE gbk_chinese_ci DEFAULT NULL,
+      KEY `idx_v` (`v`, `id`),
+      KEY `idx_gbk` (`v2`, `id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

br/tests/lightning_new_collation/data/nc.gbk_test.0.csv

@@ -0,0 +1,4 @@
+id,v,v2
+1,啊啊,啊啊
+2,哦哦,哦哦
+3,听听听听,听听听听

br/tests/lightning_new_collation/data/nc.t-schema.sql

@@ -0,0 +1 @@
+CREATE TABLE t(i INT PRIMARY KEY, s varchar(32), j TINYINT, KEY s_j (s, i)) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci;

br/tests/lightning_new_collation/data/nc.t.0.sql

@@ -0,0 +1,6 @@
+INSERT INTO t (s, i, j) VALUES
+  ("this_is_test1", 1, 1),
+  ("this_is_test2", 2, 2),
+  ("this_is_test3", 3, 3),
+  ("this_is_test4", 4, 4),
+  ("this_is_test5", 5, 5);

br/tests/lightning_new_collation/data/nc.t.1.sql

@@ -0,0 +1 @@
+INSERT INTO t(s, i, j) VALUES ("another test case", 6, 6);

br/tests/lightning_new_collation/gbk.toml

@@ -0,0 +1,3 @@
+[mydumper]
+data-character-set = "GBK"
+character-set = "gb18030"

br/tests/lightning_new_collation/run.sh

@@ -24,27 +24,11 @@ cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 # restart cluster with new collation enabled
 start_services --tidb-cfg $cur/tidb-new-collation.toml
 
-# Populate the mydumper source
-DBPATH="$TEST_DIR/nc.mydump"
-mkdir -p $DBPATH
-echo 'CREATE DATABASE nc;' > "$DBPATH/nc-schema-create.sql"
-# create table with collate `utf8_general_ci`, the index key will be different between old/new collation
-echo "CREATE TABLE t(i INT PRIMARY KEY, s varchar(32), j TINYINT, KEY s_j (s, i)) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci;" > "$DBPATH/nc.t-schema.sql"
-cat > "$DBPATH/nc.t.0.sql" << _EOF_
-INSERT INTO t (s, i, j) VALUES
-  ("this_is_test1", 1, 1),
-  ("this_is_test2", 2, 2),
-  ("this_is_test3", 3, 3),
-  ("this_is_test4", 4, 4),
-  ("this_is_test5", 5, 5);
-_EOF_
-echo 'INSERT INTO t(s, i, j) VALUES ("another test case", 6, 6);' > "$DBPATH/nc.t.1.sql"
-
 for BACKEND in local importer tidb; do
   # Start importing the tables.
   run_sql 'DROP DATABASE IF EXISTS nc'
 
-  run_lightning -d "$DBPATH" --backend $BACKEND 2> /dev/null
+  run_lightning --backend $BACKEND 2> /dev/null
 
   run_sql 'SELECT count(*), sum(i) FROM `nc`.t'
   check_contains "count(*): 6"
@@ -54,6 +38,23 @@ for BACKEND in local importer tidb; do
   run_sql "SELECT j FROM nc.t WHERE s = 'This_Is_Test4'";
   check_contains "j: 4"
 
+  run_sql 'SELECT id, v from nc.gbk_test order by v limit 1;'
+  check_contains "id: 3"
+  check_contains "v: 听听听听"
+
+  run_sql "SELECT id, v2 from nc.gbk_test order by v2 limit 1;"
+  check_contains "id: 1"
+  check_contains "v2: 啊啊"
+
+  run_lightning --backend $BACKEND -d "tests/$TEST_NAME/data-gbk" --config "tests/$TEST_NAME/gbk.toml"
+
+  run_sql 'SELECT count(*) from nc.gbk_source;'
+  check_contains "count(*): 3"
+
+  run_sql 'SELECT id, v from nc.gbk_source order by v limit 1;'
+  check_contains "id: 1"
+  check_contains "v: 啊啊"
+
 done
 
 # restart with original config

br/tests/lightning_new_collation/tidb-new-collation.toml

@@ -1,5 +1,4 @@
 # config of tidb
-
 new_collations_enabled_on_first_bootstrap = true
 
 [security]