From 0c214b57bca2a97090e628c0c60cdfcda556590c Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 24 Mar 2022 14:53:23 +0800 Subject: [PATCH 1/6] add create table case for upgrade test --- dm/tests/tiup/lib.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dm/tests/tiup/lib.sh b/dm/tests/tiup/lib.sh index 5a2664dfa71..2ece014870f 100755 --- a/dm/tests/tiup/lib.sh +++ b/dm/tests/tiup/lib.sh @@ -22,6 +22,7 @@ DB6=pes_sharding2 TBL1=T1 TBL2=t2 TBL3=T3 +TBL4=T4 TBL_LOWER1=t1 TBL_LOWER3=t3 @@ -100,6 +101,8 @@ function exec_incremental_stage1() { exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2) VALUES (102, '102');" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2) VALUES (111, '111');" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL_LOWER3 (c1, c2) VALUES (112, '112');" + exec_sql mysql1 3306 "CREATE TABLE $DB3.$TBL4(c1 INT PRIMARY KEY, c2 TEXT);" + exec_sql mariadb2 3306 "INSERT INTO $DB3.$TBL4 (c1, c2) VALUES (115, '115');" # optimistic shard ddls exec_sql mysql1 3306 "ALTER TABLE $DB3.$TBL_LOWER1 ADD COLUMN c3 INT;" @@ -112,6 +115,7 @@ function exec_incremental_stage1() { exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c4) VALUES (104, '104', 104);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3) VALUES (113, '113', 113);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL_LOWER3 (c1, c2, c4) VALUES (114, '114', 114);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL4 (c1, c2) VALUES (116, '116', 116);" # prepare pessimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB5.$TBL1 (c1, c2) VALUES (101, '101');" @@ -148,12 +152,15 @@ function exec_incremental_stage2() { exec_sql mysql1 3306 "ALTER TABLE $DB3.$TBL2 ADD COLUMN c3 INT AFTER c2;" exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL2 ADD COLUMN c4 INT;" exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL_LOWER3 ADD COLUMN c3 INT AFTER c2;" + exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL4 ADD COLUMN c4 INT;" + exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL4 ADD COLUMN c3 INT AFTER c2;" # prepare optimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL_LOWER1 (c1, c2, c3, c4) VALUES (203, '203', 203, 203);" exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c3, c4) VALUES (204, '204', 204, 204);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3, c4) VALUES (213, '213', 213, 213);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL_LOWER3 (c1, c2, c3, c4) VALUES (214, '214', 214, 214);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL4 (c1, c2, c3, c4) VALUES (215, '215', 215, 215);" # prepare pessimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB5.$TBL1 (c1, c2, c3) VALUES (201, '201', 201);" @@ -184,6 +191,7 @@ function exec_incremental_stage3() { exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c3, c4) VALUES (302, '302', 302, 302);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3, c4) VALUES (311, '311', 311, 311);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL_LOWER3 (c1, c2, c3, c4) VALUES (312, '312', 312, 312);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL4 (c1, c2, c3, c4) VALUES (313, '313', 313, 313);" # prepare pessimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB5.$TBL1 (c1, c2, c3) VALUES (303, '303', 303);" @@ -204,6 +212,7 @@ function exec_incremental_stage4() { exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c3, c4) VALUES (402, '402', 402, 402);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3, c4) VALUES (411, '411', 411, 411);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL_LOWER3 (c1, c2, c3, c4) VALUES (412, '412', 412, 412);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL4 (c1, c2, c3, c4) VALUES (413, '413', 413, 413);" # prepare pessimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB5.$TBL1 (c1, c2, c3) VALUES (403, '403', 403);" From dd365012c97c340a382d03cce6970963a82226ac Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 24 Mar 2022 15:25:39 +0800 Subject: [PATCH 2/6] fix test --- dm/tests/tiup/lib.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dm/tests/tiup/lib.sh b/dm/tests/tiup/lib.sh index 2ece014870f..83c337c3122 100755 --- a/dm/tests/tiup/lib.sh +++ b/dm/tests/tiup/lib.sh @@ -101,8 +101,8 @@ function exec_incremental_stage1() { exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2) VALUES (102, '102');" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2) VALUES (111, '111');" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL_LOWER3 (c1, c2) VALUES (112, '112');" - exec_sql mysql1 3306 "CREATE TABLE $DB3.$TBL4(c1 INT PRIMARY KEY, c2 TEXT);" - exec_sql mariadb2 3306 "INSERT INTO $DB3.$TBL4 (c1, c2) VALUES (115, '115');" + exec_sql mariadb2 3306 "CREATE TABLE $DB4.$TBL4(c1 INT PRIMARY KEY, c2 TEXT);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL4 (c1, c2) VALUES (115, '115');" # optimistic shard ddls exec_sql mysql1 3306 "ALTER TABLE $DB3.$TBL_LOWER1 ADD COLUMN c3 INT;" From 4a317802595003762e3b95d0a9965243c27efa57 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 24 Mar 2022 15:35:16 +0800 Subject: [PATCH 3/6] fix test --- dm/tests/tiup/lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dm/tests/tiup/lib.sh b/dm/tests/tiup/lib.sh index 83c337c3122..0e5af55cdd5 100755 --- a/dm/tests/tiup/lib.sh +++ b/dm/tests/tiup/lib.sh @@ -115,7 +115,7 @@ function exec_incremental_stage1() { exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c4) VALUES (104, '104', 104);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3) VALUES (113, '113', 113);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL_LOWER3 (c1, c2, c4) VALUES (114, '114', 114);" - exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL4 (c1, c2) VALUES (116, '116', 116);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL4 (c1, c2) VALUES (116, '116');" # prepare pessimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB5.$TBL1 (c1, c2) VALUES (101, '101');" From a57e6848dfcdebb90483b8e11a857b44e457f958 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 16 May 2022 11:59:00 +0800 Subject: [PATCH 4/6] use lower table name --- dm/tests/tiup/lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dm/tests/tiup/lib.sh b/dm/tests/tiup/lib.sh index 0e5af55cdd5..37f0c59d7bb 100755 --- a/dm/tests/tiup/lib.sh +++ b/dm/tests/tiup/lib.sh @@ -22,7 +22,7 @@ DB6=pes_sharding2 TBL1=T1 TBL2=t2 TBL3=T3 -TBL4=T4 +TBL4=t4 TBL_LOWER1=t1 TBL_LOWER3=t3 From 7860d64479b675f6b56d8f0c8168389207cf77ac Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 16 May 2022 14:29:19 +0800 Subject: [PATCH 5/6] add drop table --- dm/tests/tiup/lib.sh | 61 +++++++++++++++++++++++++++----- dm/tests/tiup/upgrade-from-v2.sh | 5 +++ 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/dm/tests/tiup/lib.sh b/dm/tests/tiup/lib.sh index 37f0c59d7bb..3a42b58d2e7 100755 --- a/dm/tests/tiup/lib.sh +++ b/dm/tests/tiup/lib.sh @@ -23,6 +23,7 @@ TBL1=T1 TBL2=t2 TBL3=T3 TBL4=t4 +TBL5=t5 TBL_LOWER1=t1 TBL_LOWER3=t3 @@ -30,6 +31,30 @@ function exec_sql() { echo $3 | mysql -h $1 -P $2 } +function run_sql_tidb_with_retry() { + rc=0 + for ((k = 1; k < 11; k++)); do + # in retry scenario sometimes run_sql_tidb will fail because "table not exist", we should keep retrying so turn + # off the error option temporarily. + set +e + echo $1 | mysql -h tidb -P 4000 >"/tmp/res.txt" + set -e + if grep -Fq "$2" "/tmp/res.txt"; then + rc=1 + break + fi + echo "run tidb sql failed $k-th time, retry later" + sleep 2 + done + if [[ $rc = 0 ]]; then + echo "TEST FAILED: OUTPUT DOES NOT CONTAIN '$2'" + echo "____________________________________" + cat "/tmp/res.txt" + echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" + exit 1 + fi +} + function install_sync_diff() { curl https://download.pingcap.org/tidb-enterprise-tools-nightly-linux-amd64.tar.gz | tar xz mkdir -p bin @@ -99,10 +124,19 @@ function exec_incremental_stage1() { # prepare optimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL_LOWER1 (c1, c2) VALUES (101, '101');" exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2) VALUES (102, '102');" + # v2.0.0 has bug with create/drop table + if [[ "$PRE_VER" != "v2.0.0" ]]; then + exec_sql mysql1 3306 "CREATE TABLE $DB3.$TBL5(c1 INT PRIMARY KEY, c2 TEXT);" + exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL5 (c1, c2) VALUES (1000, 1000);" + fi exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2) VALUES (111, '111');" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL_LOWER3 (c1, c2) VALUES (112, '112');" - exec_sql mariadb2 3306 "CREATE TABLE $DB4.$TBL4(c1 INT PRIMARY KEY, c2 TEXT);" - exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL4 (c1, c2) VALUES (115, '115');" + + if [[ "$PRE_VER" != "v2.0.0" ]]; then + exec_sql mariadb2 3306 "CREATE TABLE $DB4.$TBL4(c1 INT PRIMARY KEY, c2 TEXT);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL4 (c1, c2) VALUES (115, '115');" + exec_sql mysql1 3306 "DROP TABLE $DB3.$TBL5;" + fi # optimistic shard ddls exec_sql mysql1 3306 "ALTER TABLE $DB3.$TBL_LOWER1 ADD COLUMN c3 INT;" @@ -115,7 +149,9 @@ function exec_incremental_stage1() { exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c4) VALUES (104, '104', 104);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3) VALUES (113, '113', 113);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL_LOWER3 (c1, c2, c4) VALUES (114, '114', 114);" - exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL4 (c1, c2) VALUES (116, '116');" + if [[ "$PRE_VER" != "v2.0.0" ]]; then + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL4 (c1, c2) VALUES (116, '116');" + fi # prepare pessimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB5.$TBL1 (c1, c2) VALUES (101, '101');" @@ -152,15 +188,20 @@ function exec_incremental_stage2() { exec_sql mysql1 3306 "ALTER TABLE $DB3.$TBL2 ADD COLUMN c3 INT AFTER c2;" exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL2 ADD COLUMN c4 INT;" exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL_LOWER3 ADD COLUMN c3 INT AFTER c2;" - exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL4 ADD COLUMN c4 INT;" - exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL4 ADD COLUMN c3 INT AFTER c2;" + + if [[ "$PRE_VER" != "v2.0.0" ]]; then + exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL4 ADD COLUMN c4 INT;" + exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL4 ADD COLUMN c3 INT AFTER c2;" + fi # prepare optimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL_LOWER1 (c1, c2, c3, c4) VALUES (203, '203', 203, 203);" exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c3, c4) VALUES (204, '204', 204, 204);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3, c4) VALUES (213, '213', 213, 213);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL_LOWER3 (c1, c2, c3, c4) VALUES (214, '214', 214, 214);" - exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL4 (c1, c2, c3, c4) VALUES (215, '215', 215, 215);" + if [[ "$PRE_VER" != "v2.0.0" ]]; then + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL4 (c1, c2, c3, c4) VALUES (215, '215', 215, 215);" + fi # prepare pessimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB5.$TBL1 (c1, c2, c3) VALUES (201, '201', 201);" @@ -191,7 +232,9 @@ function exec_incremental_stage3() { exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c3, c4) VALUES (302, '302', 302, 302);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3, c4) VALUES (311, '311', 311, 311);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL_LOWER3 (c1, c2, c3, c4) VALUES (312, '312', 312, 312);" - exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL4 (c1, c2, c3, c4) VALUES (313, '313', 313, 313);" + if [[ "$PRE_VER" != "v2.0.0" ]]; then + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL4 (c1, c2, c3, c4) VALUES (313, '313', 313, 313);" + fi # prepare pessimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB5.$TBL1 (c1, c2, c3) VALUES (303, '303', 303);" @@ -212,7 +255,9 @@ function exec_incremental_stage4() { exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c3, c4) VALUES (402, '402', 402, 402);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3, c4) VALUES (411, '411', 411, 411);" exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL_LOWER3 (c1, c2, c3, c4) VALUES (412, '412', 412, 412);" - exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL4 (c1, c2, c3, c4) VALUES (413, '413', 413, 413);" + if [[ "$PRE_VER" != "v2.0.0" ]]; then + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL4 (c1, c2, c3, c4) VALUES (413, '413', 413, 413);" + fi # prepare pessimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB5.$TBL1 (c1, c2, c3) VALUES (403, '403', 403);" diff --git a/dm/tests/tiup/upgrade-from-v2.sh b/dm/tests/tiup/upgrade-from-v2.sh index 459266c66b7..e83dcff4c15 100755 --- a/dm/tests/tiup/upgrade-from-v2.sh +++ b/dm/tests/tiup/upgrade-from-v2.sh @@ -53,6 +53,11 @@ function migrate_in_previous_v2() { exec_incremental_stage1 + # drop table will be skipped + if [[ "$PRE_VER" != "v2.0.0" ]]; then + run_sql_tidb_with_retry "select count(*) from opt_db_target.t_target where c1=1000;" "1" + exec_sql tidb 4000 "delete from opt_db_target.t_target where c1=1000;" + fi check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml tiup dmctl:$PRE_VER --master-addr=master1:8261 pause-task $TASK_NAME From 5fe59272bdae9ee94c044694559c9f877ba317f2 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Mon, 16 May 2022 15:42:51 +0800 Subject: [PATCH 6/6] fix v1 --- dm/tests/tiup/lib.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dm/tests/tiup/lib.sh b/dm/tests/tiup/lib.sh index 3a42b58d2e7..40cbb6ad057 100755 --- a/dm/tests/tiup/lib.sh +++ b/dm/tests/tiup/lib.sh @@ -26,6 +26,7 @@ TBL4=t4 TBL5=t5 TBL_LOWER1=t1 TBL_LOWER3=t3 +PRE_VER=${PRE_VER:-v1.0.7} function exec_sql() { echo $3 | mysql -h $1 -P $2