From 48dacb1aed28184a325b27a41e3e9f06ae808045 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Fri, 14 May 2021 11:07:38 +0800 Subject: [PATCH] tests: add mariaDB to version upgrading CI (#1661) --- .github/workflows/upgrade-via-tiup.yml | 20 ++++++++ dm/master/bootstrap.go | 4 +- dm/master/scheduler/scheduler.go | 2 +- pkg/upgrade/upgrade.go | 1 + tests/_utils/test_prepare | 15 ++++++ tests/ha/run.sh | 6 +-- tests/tiup/ansible_data/inventory.ini | 2 +- tests/tiup/conf/diff_config.toml | 2 +- tests/tiup/conf/diff_config_optimistic.toml | 2 +- tests/tiup/conf/source2.yaml | 4 +- tests/tiup/conf/task.yaml | 2 +- tests/tiup/conf/task_optimistic.yaml | 2 +- tests/tiup/docker/control/Dockerfile | 10 ++-- tests/tiup/docker/docker-compose.yml | 10 ++-- tests/tiup/lib.sh | 56 ++++++++++----------- tests/tiup/upgrade-from-v1.sh | 2 +- tests/tiup/upgrade-from-v2.sh | 5 +- 17 files changed, 89 insertions(+), 56 deletions(-) diff --git a/.github/workflows/upgrade-via-tiup.yml b/.github/workflows/upgrade-via-tiup.yml index 198b6f5834..bf3909bca9 100644 --- a/.github/workflows/upgrade-via-tiup.yml +++ b/.github/workflows/upgrade-via-tiup.yml @@ -134,6 +134,26 @@ jobs: cd ${{ env.working-directory }}/tests/tiup/docker docker-compose exec -e ref=${{ github.ref }} -T control bash -c "cd /go/src/github.com/pingcap/dm && ./tests/tiup/upgrade-from-v2.sh ${{ matrix.previous_v2 }} nightly" + # if above step is passed, logs will be removed by tiup dm destroy + - name: Copy logs to hack permission + if: ${{ failure() }} + run: | + mkdir ./logs + docker cp -L master1:/home/tidb/dm/deploy/dm-master-8261/log ./logs/master + docker cp -L worker1:/home/tidb/dm/deploy/dm-worker-8262/log ./logs/worker1 + docker cp -L worker2:/home/tidb/dm/deploy/dm-worker-8262/log ./logs/worker2 + sudo chown -R runner ./logs + + # Update logs as artifact seems not stable, so we set `continue-on-error: true` here. + - name: Upload logs + continue-on-error: true + uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + name: upgrade-via-tiup-${{ matrix.previous_v2 }} + path: | + ./logs + # send Slack notify if failed. # NOTE: With the exception of `GITHUB_TOKEN`, secrets are not passed to the runner when a workflow is triggered from a forked repository. - name: Slack notification diff --git a/dm/master/bootstrap.go b/dm/master/bootstrap.go index 13d5db0b13..81bb5ffc04 100644 --- a/dm/master/bootstrap.go +++ b/dm/master/bootstrap.go @@ -66,12 +66,12 @@ func (s *Server) bootstrap(ctx context.Context) error { return err } } - + log.L().Info("successful bootstrapped") return nil } func (s *Server) bootstrapBeforeSchedulerStart(ctx context.Context) error { - log.L().Info("start before scheduler start") + log.L().Info("bootstrap before scheduler start") // no need for v1.0.x if s.cfg.V1SourcesPath != "" { return nil diff --git a/dm/master/scheduler/scheduler.go b/dm/master/scheduler/scheduler.go index 0caf2acaad..a27ef81afd 100644 --- a/dm/master/scheduler/scheduler.go +++ b/dm/master/scheduler/scheduler.go @@ -553,7 +553,7 @@ func (s *Scheduler) AddSubTasks(cfgs ...config.SubTaskConfig) error { return nil } -// RemoveSubTasks removes the information of one or more subtaks for one task. +// RemoveSubTasks removes the information of one or more subtasks for one task. func (s *Scheduler) RemoveSubTasks(task string, sources ...string) error { s.mu.Lock() defer s.mu.Unlock() diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go index fd34ee476c..41f007b267 100644 --- a/pkg/upgrade/upgrade.go +++ b/pkg/upgrade/upgrade.go @@ -98,6 +98,7 @@ func TryUpgrade(cli *clientv3.Client, uctx Context) error { // 5. put the current version into etcd. _, err = PutVersion(cli, CurrentVersion) + log.L().Info("upgrade cluster version", zap.Any("version", CurrentVersion), zap.Error(err)) return err } diff --git a/tests/_utils/test_prepare b/tests/_utils/test_prepare index 64612015a9..8f7aae98ef 100644 --- a/tests/_utils/test_prepare +++ b/tests/_utils/test_prepare @@ -80,6 +80,21 @@ function dmctl_stop_task() { dmctl_operate_task $task_name stop-task } +# shortcut for retryable stop task +function dmctl_stop_task_with_retry() { + task_name=$1 + master_port=$2 + for ((k=0; k<10; k++)); do + run_dm_ctl $WORK_DIR "127.0.0.1:$master_port" \ + "stop-task $task_name" + run_dm_ctl $WORK_DIR "127.0.0.1:$master_port" \ + "query-status $task_name" \ + "task test has no source or not exist" 1 && return 0 + sleep 1 + done + return 1 +} + # shortcut for pause task on two DM-workers function dmctl_pause_task() { task_name=$1 diff --git a/tests/ha/run.sh b/tests/ha/run.sh index a58961bee3..e6842b40d8 100755 --- a/tests/ha/run.sh +++ b/tests/ha/run.sh @@ -167,11 +167,7 @@ function run() { echo "now we will check whether joined masters can work normally" # we need some time for cluster to re-elect new available leader - run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT5" \ - "stop-task test" \ - "\"result\": true" 3 \ - "\"source\": \"$SOURCE_ID1\"" 1 \ - "\"source\": \"$SOURCE_ID2\"" 1 + dmctl_stop_task_with_retry "test" $MASTER_PORT5 run_sql_file $cur/data/db1.increment2.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 run_sql_file $cur/data/db2.increment2.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 diff --git a/tests/tiup/ansible_data/inventory.ini b/tests/tiup/ansible_data/inventory.ini index 858a563640..f7d3aad04d 100644 --- a/tests/tiup/ansible_data/inventory.ini +++ b/tests/tiup/ansible_data/inventory.ini @@ -5,7 +5,7 @@ dm_master ansible_host=172.28.0.101 [dm_worker_servers] dm-worker1 ansible_host=172.28.0.111 server_id=101 source_id="mysql-replica-01" mysql_host=mysql1 mysql_user=root mysql_password='' mysql_port=3306 -dm-worker2 ansible_host=172.28.0.112 server_id=101 source_id="mysql-replica-02" mysql_host=mysql2 mysql_user=root mysql_password='' mysql_port=3306 +dm-worker2 ansible_host=172.28.0.112 server_id=101 source_id="mariadb-replica-02" mysql_host=mariadb2 mysql_user=root mysql_password='' mysql_port=3306 ## Monitoring modules [prometheus_servers] diff --git a/tests/tiup/conf/diff_config.toml b/tests/tiup/conf/diff_config.toml index af9a624060..881528c575 100644 --- a/tests/tiup/conf/diff_config.toml +++ b/tests/tiup/conf/diff_config.toml @@ -28,7 +28,7 @@ password = "" instance-id = "source-1" [[source-db]] -host = "mysql2" +host = "mariadb2" port = 3306 user = "root" password = "" diff --git a/tests/tiup/conf/diff_config_optimistic.toml b/tests/tiup/conf/diff_config_optimistic.toml index 452c766e0e..968c53c0be 100644 --- a/tests/tiup/conf/diff_config_optimistic.toml +++ b/tests/tiup/conf/diff_config_optimistic.toml @@ -28,7 +28,7 @@ password = "" instance-id = "source-1" [[source-db]] -host = "mysql2" +host = "mariadb2" port = 3306 user = "root" password = "" diff --git a/tests/tiup/conf/source2.yaml b/tests/tiup/conf/source2.yaml index cf0a47d14e..30903199f5 100644 --- a/tests/tiup/conf/source2.yaml +++ b/tests/tiup/conf/source2.yaml @@ -1,8 +1,8 @@ -source-id: mysql-replica-02 +source-id: mariadb-replica-02 enable-gtid: true enable-relay: true from: - host: mysql2 + host: mariadb2 user: root port: 3306 checker: diff --git a/tests/tiup/conf/task.yaml b/tests/tiup/conf/task.yaml index 6d2d4e0105..89b7c8e2fb 100644 --- a/tests/tiup/conf/task.yaml +++ b/tests/tiup/conf/task.yaml @@ -15,7 +15,7 @@ mysql-instances: black-white-list: "instance" route-rules: ["sharding-route-rules-table", "sharding-route-rules-schema"] - - source-id: "mysql-replica-02" + - source-id: "mariadb-replica-02" black-white-list: "instance" route-rules: ["sharding-route-rules-table", "sharding-route-rules-schema"] diff --git a/tests/tiup/conf/task_optimistic.yaml b/tests/tiup/conf/task_optimistic.yaml index 979df04b52..c3d8d7d20a 100644 --- a/tests/tiup/conf/task_optimistic.yaml +++ b/tests/tiup/conf/task_optimistic.yaml @@ -16,7 +16,7 @@ mysql-instances: black-white-list: "instance" route-rules: ["sharding-route-rules-table", "sharding-route-rules-schema"] - - source-id: "mysql-replica-02" + - source-id: "mariadb-replica-02" black-white-list: "instance" route-rules: ["sharding-route-rules-table", "sharding-route-rules-schema"] diff --git a/tests/tiup/docker/control/Dockerfile b/tests/tiup/docker/control/Dockerfile index a847af65bb..2f181107e9 100644 --- a/tests/tiup/docker/control/Dockerfile +++ b/tests/tiup/docker/control/Dockerfile @@ -1,10 +1,10 @@ FROM golang:1.13 - RUN apt-get -y -q update && \ - apt-get -y -q install software-properties-common && \ - apt-get install -qqy \ - default-mysql-client \ - psmisc +RUN apt-get -y -q update && \ + apt-get -y -q install software-properties-common && \ + apt-get install -qqy \ + default-mysql-client \ + psmisc ADD bashrc /root/.bashrc ADD init.sh /init.sh diff --git a/tests/tiup/docker/docker-compose.yml b/tests/tiup/docker/docker-compose.yml index 267b87cb26..e31d7ad438 100644 --- a/tests/tiup/docker/docker-compose.yml +++ b/tests/tiup/docker/docker-compose.yml @@ -37,14 +37,14 @@ services: MYSQL_ROOT_PASSWORD: "" MYSQL_ALLOW_EMPTY_PASSWORD: 1 - mysql2: # upstream MySQL-2 - container_name: mysql2 - hostname: mysql2 + mariadb2: # upstream MariaDB + container_name: mariadb2 + hostname: mariadb2 networks: dm-tiup: ipv4_address: 172.28.0.202 - image: mysql:5.7.32 - command: --default-authentication-plugin=mysql_native_password --log-bin=/var/lib/mysql/mysql-bin --server-id=1 --binlog-format=ROW --gtid_mode=ON --enforce-gtid-consistency=true + image: mariadb:10.5.8 + command: --log-bin --server-id=1 --binlog-format=ROW restart: always environment: MYSQL_ROOT_PASSWORD: "" diff --git a/tests/tiup/lib.sh b/tests/tiup/lib.sh index 46f2239f90..408c5699f9 100755 --- a/tests/tiup/lib.sh +++ b/tests/tiup/lib.sh @@ -31,90 +31,90 @@ function install_sync_diff() { function exec_full_stage() { # drop previous data exec_sql mysql1 3306 "DROP DATABASE IF EXISTS $DB1;" - exec_sql mysql2 3306 "DROP DATABASE IF EXISTS $DB2;" + exec_sql mariadb2 3306 "DROP DATABASE IF EXISTS $DB2;" exec_sql mysql1 3306 "DROP DATABASE IF EXISTS $DB3;" - exec_sql mysql2 3306 "DROP DATABASE IF EXISTS $DB4;" + exec_sql mariadb2 3306 "DROP DATABASE IF EXISTS $DB4;" exec_sql tidb 4000 "DROP DATABASE IF EXISTS db_target;" exec_sql tidb 4000 "DROP DATABASE IF EXISTS opt_db_target;" exec_sql tidb 4000 "DROP DATABASE IF EXISTS dm_meta;" # # prepare full data exec_sql mysql1 3306 "CREATE DATABASE $DB1;" - exec_sql mysql2 3306 "CREATE DATABASE $DB2;" + exec_sql mariadb2 3306 "CREATE DATABASE $DB2;" exec_sql mysql1 3306 "CREATE TABLE $DB1.$TBL1 (c1 INT PRIMARY KEY, c2 TEXT);" exec_sql mysql1 3306 "CREATE TABLE $DB1.$TBL2 (c1 INT PRIMARY KEY, c2 TEXT);" - exec_sql mysql2 3306 "CREATE TABLE $DB2.$TBL2 (c1 INT PRIMARY KEY, c2 TEXT);" - exec_sql mysql2 3306 "CREATE TABLE $DB2.$TBL3 (c1 INT PRIMARY KEY, c2 TEXT);" + exec_sql mariadb2 3306 "CREATE TABLE $DB2.$TBL2 (c1 INT PRIMARY KEY, c2 TEXT);" + exec_sql mariadb2 3306 "CREATE TABLE $DB2.$TBL3 (c1 INT PRIMARY KEY, c2 TEXT);" exec_sql mysql1 3306 "INSERT INTO $DB1.$TBL1 (c1, c2) VALUES (1, '1');" exec_sql mysql1 3306 "INSERT INTO $DB1.$TBL2 (c1, c2) VALUES (2, '2');" - exec_sql mysql2 3306 "INSERT INTO $DB2.$TBL2 (c1, c2) VALUES (11, '11');" - exec_sql mysql2 3306 "INSERT INTO $DB2.$TBL3 (c1, c2) VALUES (12, '12');" + exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL2 (c1, c2) VALUES (11, '11');" + exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL3 (c1, c2) VALUES (12, '12');" # prepare optimsitic full data exec_sql mysql1 3306 "CREATE DATABASE $DB3;" - exec_sql mysql2 3306 "CREATE DATABASE $DB4;" + exec_sql mariadb2 3306 "CREATE DATABASE $DB4;" exec_sql mysql1 3306 "CREATE TABLE $DB3.$TBL1 (c1 INT PRIMARY KEY, c2 TEXT);" exec_sql mysql1 3306 "CREATE TABLE $DB3.$TBL2 (c1 INT PRIMARY KEY, c2 TEXT);" - exec_sql mysql2 3306 "CREATE TABLE $DB4.$TBL2 (c1 INT PRIMARY KEY, c2 TEXT);" - exec_sql mysql2 3306 "CREATE TABLE $DB4.$TBL3 (c1 INT PRIMARY KEY, c2 TEXT);" + exec_sql mariadb2 3306 "CREATE TABLE $DB4.$TBL2 (c1 INT PRIMARY KEY, c2 TEXT);" + exec_sql mariadb2 3306 "CREATE TABLE $DB4.$TBL3 (c1 INT PRIMARY KEY, c2 TEXT);" exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL1 (c1, c2) VALUES (1, '1');" exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2) VALUES (2, '2');" - exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2) VALUES (11, '11');" - exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2) VALUES (12, '12');" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2) VALUES (11, '11');" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2) VALUES (12, '12');" } function exec_incremental_stage1() { # prepare incremental data exec_sql mysql1 3306 "INSERT INTO $DB1.$TBL1 (c1, c2) VALUES (101, '101');" exec_sql mysql1 3306 "INSERT INTO $DB1.$TBL2 (c1, c2) VALUES (102, '102');" - exec_sql mysql2 3306 "INSERT INTO $DB2.$TBL2 (c1, c2) VALUES (111, '111');" - exec_sql mysql2 3306 "INSERT INTO $DB2.$TBL3 (c1, c2) VALUES (112, '112');" + exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL2 (c1, c2) VALUES (111, '111');" + exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL3 (c1, c2) VALUES (112, '112');" # prepare optimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL1 (c1, c2) VALUES (101, '101');" exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2) VALUES (102, '102');" - exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2) VALUES (111, '111');" - exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2) VALUES (112, '112');" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2) VALUES (111, '111');" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2) VALUES (112, '112');" # optimistic shard ddls exec_sql mysql1 3306 "ALTER TABLE $DB3.$TBL1 ADD COLUMN c3 INT;" exec_sql mysql1 3306 "ALTER TABLE $DB3.$TBL2 ADD COLUMN c4 INT;" - exec_sql mysql2 3306 "ALTER TABLE $DB4.$TBL2 ADD COLUMN c3 INT;" - exec_sql mysql2 3306 "ALTER TABLE $DB4.$TBL3 ADD COLUMN c4 INT;" + exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL2 ADD COLUMN c3 INT;" + exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL3 ADD COLUMN c4 INT;" # prepare optimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL1 (c1, c2, c3) VALUES (103, '103', 103);" exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c4) VALUES (104, '104', 104);" - exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3) VALUES (113, '113', 113);" - exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2, c4) VALUES (114, '113', 113);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3) VALUES (113, '113', 113);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2, c4) VALUES (114, '113', 113);" } function exec_incremental_stage2() { # prepare incremental data exec_sql mysql1 3306 "INSERT INTO $DB1.$TBL1 (c1, c2) VALUES (201, '201');" exec_sql mysql1 3306 "INSERT INTO $DB1.$TBL2 (c1, c2) VALUES (202, '202');" - exec_sql mysql2 3306 "INSERT INTO $DB2.$TBL2 (c1, c2) VALUES (211, '211');" - exec_sql mysql2 3306 "INSERT INTO $DB2.$TBL3 (c1, c2) VALUES (212, '212');" + exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL2 (c1, c2) VALUES (211, '211');" + exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL3 (c1, c2) VALUES (212, '212');" # prepare optimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL1 (c1, c2, c3) VALUES (201, '201', 201);" exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c4) VALUES (202, '202', 202);" - exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3) VALUES (211, '211', 211);" - exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2, c4) VALUES (212, '212', 212);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3) VALUES (211, '211', 211);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2, c4) VALUES (212, '212', 212);" # optimistic shard ddls exec_sql mysql1 3306 "ALTER TABLE $DB3.$TBL1 ADD COLUMN c4 INT;" exec_sql mysql1 3306 "ALTER TABLE $DB3.$TBL2 ADD COLUMN c3 INT AFTER c2;" - exec_sql mysql2 3306 "ALTER TABLE $DB4.$TBL2 ADD COLUMN c4 INT;" - exec_sql mysql2 3306 "ALTER TABLE $DB4.$TBL3 ADD COLUMN c3 INT AFTER c2;" + exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL2 ADD COLUMN c4 INT;" + exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL3 ADD COLUMN c3 INT AFTER c2;" # prepare optimistic incremental data exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL1 (c1, c2, c3, c4) VALUES (203, '203', 203, 203);" exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c3, c4) VALUES (204, '204', 204, 204);" - exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3, c4) VALUES (213, '213', 213, 213);" - exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2, c3, c4) VALUES (214, '214', 214, 214);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3, c4) VALUES (213, '213', 213, 213);" + exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2, c3, c4) VALUES (214, '214', 214, 214);" } function patch_nightly_with_tiup_mirror() { diff --git a/tests/tiup/upgrade-from-v1.sh b/tests/tiup/upgrade-from-v1.sh index 478868acc5..09da177fae 100755 --- a/tests/tiup/upgrade-from-v1.sh +++ b/tests/tiup/upgrade-from-v1.sh @@ -107,7 +107,7 @@ function migrate_in_v2 { exec_incremental_stage2 echo "check sources" - run_dmctl_with_retry $DM_V2_VER "operate-source show" "mysql-replica-01" 1 "mysql-replica-02" 1 + run_dmctl_with_retry $DM_V2_VER "operate-source show" "mysql-replica-01" 1 "mariadb-replica-02" 1 echo "check workers" run_dmctl_with_retry $DM_V2_VER "list-member --worker" "\"stage\": \"bound\"" 2 diff --git a/tests/tiup/upgrade-from-v2.sh b/tests/tiup/upgrade-from-v2.sh index e55b150339..88f466ffd3 100755 --- a/tests/tiup/upgrade-from-v2.sh +++ b/tests/tiup/upgrade-from-v2.sh @@ -27,9 +27,10 @@ function deploy_previous_v2() { function migrate_in_previous_v2() { exec_full_stage - # v2.0.0 doesn't support relay log + # v2.0.0 doesn't implement relay log, and enable-gtid for MariaDB has a bug if [[ "$PRE_VER" == "v2.0.0" ]]; then sed -i "s/enable-relay: true/enable-relay: false/g" $CUR/conf/source2.yaml + sed -i "s/enable-gtid: true/enable-gtid: false/g" $CUR/conf/source2.yaml fi tiup dmctl:$PRE_VER --master-addr=master1:8261 operate-source create $CUR/conf/source1.yaml @@ -55,7 +56,7 @@ function migrate_in_v2 { exec_incremental_stage2 echo "check sources" - run_dmctl_with_retry $CUR_VER "operate-source show" "mysql-replica-01" 1 "mysql-replica-02" 1 + run_dmctl_with_retry $CUR_VER "operate-source show" "mysql-replica-01" 1 "mariadb-replica-02" 1 echo "check workers" run_dmctl_with_retry $CUR_VER "list-member --worker" "\"stage\": \"bound\"" 2