tests: add mariaDB to version upgrading CI (#1661)

.github/workflows/upgrade-via-tiup.yml

@@ -134,6 +134,26 @@ jobs:
         cd ${{ env.working-directory }}/tests/tiup/docker
         docker-compose exec -e ref=${{ github.ref }} -T control bash -c "cd /go/src/github.com/pingcap/dm && ./tests/tiup/upgrade-from-v2.sh ${{ matrix.previous_v2 }} nightly"
 
+    # if above step is passed, logs will be removed by tiup dm destroy
+    - name: Copy logs to hack permission
+      if: ${{ failure() }}
+      run: |
+        mkdir ./logs
+        docker cp -L master1:/home/tidb/dm/deploy/dm-master-8261/log ./logs/master
+        docker cp -L worker1:/home/tidb/dm/deploy/dm-worker-8262/log ./logs/worker1
+        docker cp -L worker2:/home/tidb/dm/deploy/dm-worker-8262/log ./logs/worker2
+        sudo chown -R runner ./logs
+
+    # Update logs as artifact seems not stable, so we set `continue-on-error: true` here.
+    - name: Upload logs
+      continue-on-error: true
+      uses: actions/upload-artifact@v2
+      if: ${{ failure() }}
+      with:
+        name: upgrade-via-tiup-${{ matrix.previous_v2 }}
+        path: |
+          ./logs
+
     # send Slack notify if failed.
     # NOTE: With the exception of `GITHUB_TOKEN`, secrets are not passed to the runner when a workflow is triggered from a forked repository.
     - name: Slack notification

dm/master/bootstrap.go

@@ -66,12 +66,12 @@ func (s *Server) bootstrap(ctx context.Context) error {
 			return err
 		}
 	}
-
+	log.L().Info("successful bootstrapped")
 	return nil
 }
 
 func (s *Server) bootstrapBeforeSchedulerStart(ctx context.Context) error {
-	log.L().Info("start before scheduler start")
+	log.L().Info("bootstrap before scheduler start")
 	// no need for v1.0.x
 	if s.cfg.V1SourcesPath != "" {
 		return nil

dm/master/scheduler/scheduler.go

@@ -553,7 +553,7 @@ func (s *Scheduler) AddSubTasks(cfgs ...config.SubTaskConfig) error {
 	return nil
 }
 
-// RemoveSubTasks removes the information of one or more subtaks for one task.
+// RemoveSubTasks removes the information of one or more subtasks for one task.
 func (s *Scheduler) RemoveSubTasks(task string, sources ...string) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()

pkg/upgrade/upgrade.go

@@ -98,6 +98,7 @@ func TryUpgrade(cli *clientv3.Client, uctx Context) error {
 
 	// 5. put the current version into etcd.
 	_, err = PutVersion(cli, CurrentVersion)
+	log.L().Info("upgrade cluster version", zap.Any("version", CurrentVersion), zap.Error(err))
 	return err
 }
 

tests/_utils/test_prepare

@@ -80,6 +80,21 @@ function dmctl_stop_task() {
     dmctl_operate_task $task_name stop-task
 }
 
+# shortcut for retryable stop task
+function dmctl_stop_task_with_retry() {
+    task_name=$1
+    master_port=$2
+    for ((k=0; k<10; k++)); do
+        run_dm_ctl $WORK_DIR "127.0.0.1:$master_port" \
+            "stop-task $task_name"
+        run_dm_ctl $WORK_DIR "127.0.0.1:$master_port" \
+            "query-status $task_name" \
+            "task test has no source or not exist" 1 && return 0
+        sleep 1
+    done
+    return 1
+}
+
 # shortcut for pause task on two DM-workers
 function dmctl_pause_task() {
     task_name=$1

tests/ha/run.sh

@@ -167,11 +167,7 @@ function run() {
 	echo "now we will check whether joined masters can work normally"
 
 	# we need some time for cluster to re-elect new available leader
-	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT5" \
-		"stop-task test" \
-		"\"result\": true" 3 \
-		"\"source\": \"$SOURCE_ID1\"" 1 \
-		"\"source\": \"$SOURCE_ID2\"" 1
+	dmctl_stop_task_with_retry "test" $MASTER_PORT5
 
 	run_sql_file $cur/data/db1.increment2.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
 	run_sql_file $cur/data/db2.increment2.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2

tests/tiup/ansible_data/inventory.ini

@@ -5,7 +5,7 @@ dm_master ansible_host=172.28.0.101
 [dm_worker_servers]
 dm-worker1 ansible_host=172.28.0.111  server_id=101 source_id="mysql-replica-01" mysql_host=mysql1 mysql_user=root mysql_password='' mysql_port=3306
 
-dm-worker2 ansible_host=172.28.0.112  server_id=101 source_id="mysql-replica-02" mysql_host=mysql2 mysql_user=root mysql_password='' mysql_port=3306
+dm-worker2 ansible_host=172.28.0.112  server_id=101 source_id="mariadb-replica-02" mysql_host=mariadb2 mysql_user=root mysql_password='' mysql_port=3306
 
 ## Monitoring modules
 [prometheus_servers]

tests/tiup/conf/diff_config.toml

@@ -28,7 +28,7 @@ password = ""
 instance-id = "source-1"
 
 [[source-db]]
-host = "mysql2"
+host = "mariadb2"
 port = 3306
 user = "root"
 password = ""

tests/tiup/conf/diff_config_optimistic.toml

@@ -28,7 +28,7 @@ password = ""
 instance-id = "source-1"
 
 [[source-db]]
-host = "mysql2"
+host = "mariadb2"
 port = 3306
 user = "root"
 password = ""

tests/tiup/conf/source2.yaml

@@ -1,8 +1,8 @@
-source-id: mysql-replica-02
+source-id: mariadb-replica-02
 enable-gtid: true
 enable-relay: true
 from:
-  host: mysql2
+  host: mariadb2
   user: root
   port: 3306
 checker:

tests/tiup/conf/task.yaml

@@ -15,7 +15,7 @@ mysql-instances:
     black-white-list:  "instance"
     route-rules: ["sharding-route-rules-table", "sharding-route-rules-schema"]
 
-  - source-id: "mysql-replica-02"
+  - source-id: "mariadb-replica-02"
     black-white-list:  "instance"
     route-rules: ["sharding-route-rules-table", "sharding-route-rules-schema"]
 

tests/tiup/conf/task_optimistic.yaml

@@ -16,7 +16,7 @@ mysql-instances:
     black-white-list:  "instance"
     route-rules: ["sharding-route-rules-table", "sharding-route-rules-schema"]
 
-  - source-id: "mysql-replica-02"
+  - source-id: "mariadb-replica-02"
     black-white-list:  "instance"
     route-rules: ["sharding-route-rules-table", "sharding-route-rules-schema"]
 

tests/tiup/docker/control/Dockerfile

@@ -1,10 +1,10 @@
 FROM golang:1.13
 
- RUN apt-get -y -q update && \
-     apt-get -y -q install software-properties-common && \
-     apt-get install -qqy \
-         default-mysql-client \
-         psmisc
+RUN apt-get -y -q update && \
+    apt-get -y -q install software-properties-common && \
+    apt-get install -qqy \
+        default-mysql-client \
+        psmisc
 
 ADD bashrc /root/.bashrc
 ADD init.sh /init.sh

tests/tiup/docker/docker-compose.yml

@@ -37,14 +37,14 @@ services:
       MYSQL_ROOT_PASSWORD: ""
       MYSQL_ALLOW_EMPTY_PASSWORD: 1
 
-  mysql2: # upstream MySQL-2
-    container_name: mysql2
-    hostname: mysql2
+  mariadb2: # upstream MariaDB
+    container_name: mariadb2
+    hostname: mariadb2
     networks:
       dm-tiup:
         ipv4_address: 172.28.0.202
-    image: mysql:5.7.32
-    command: --default-authentication-plugin=mysql_native_password --log-bin=/var/lib/mysql/mysql-bin --server-id=1 --binlog-format=ROW --gtid_mode=ON --enforce-gtid-consistency=true
+    image: mariadb:10.5.8
+    command: --log-bin --server-id=1 --binlog-format=ROW
     restart: always
     environment:
       MYSQL_ROOT_PASSWORD: ""

tests/tiup/lib.sh

@@ -31,90 +31,90 @@ function install_sync_diff() {
 function exec_full_stage() {
 	# drop previous data
 	exec_sql mysql1 3306 "DROP DATABASE IF EXISTS $DB1;"
-	exec_sql mysql2 3306 "DROP DATABASE IF EXISTS $DB2;"
+	exec_sql mariadb2 3306 "DROP DATABASE IF EXISTS $DB2;"
 	exec_sql mysql1 3306 "DROP DATABASE IF EXISTS $DB3;"
-	exec_sql mysql2 3306 "DROP DATABASE IF EXISTS $DB4;"
+	exec_sql mariadb2 3306 "DROP DATABASE IF EXISTS $DB4;"
 	exec_sql tidb 4000 "DROP DATABASE IF EXISTS db_target;"
 	exec_sql tidb 4000 "DROP DATABASE IF EXISTS opt_db_target;"
 	exec_sql tidb 4000 "DROP DATABASE IF EXISTS dm_meta;"
 
 	# # prepare full data
 	exec_sql mysql1 3306 "CREATE DATABASE $DB1;"
-	exec_sql mysql2 3306 "CREATE DATABASE $DB2;"
+	exec_sql mariadb2 3306 "CREATE DATABASE $DB2;"
 	exec_sql mysql1 3306 "CREATE TABLE $DB1.$TBL1 (c1 INT PRIMARY KEY, c2 TEXT);"
 	exec_sql mysql1 3306 "CREATE TABLE $DB1.$TBL2 (c1 INT PRIMARY KEY, c2 TEXT);"
-	exec_sql mysql2 3306 "CREATE TABLE $DB2.$TBL2 (c1 INT PRIMARY KEY, c2 TEXT);"
-	exec_sql mysql2 3306 "CREATE TABLE $DB2.$TBL3 (c1 INT PRIMARY KEY, c2 TEXT);"
+	exec_sql mariadb2 3306 "CREATE TABLE $DB2.$TBL2 (c1 INT PRIMARY KEY, c2 TEXT);"
+	exec_sql mariadb2 3306 "CREATE TABLE $DB2.$TBL3 (c1 INT PRIMARY KEY, c2 TEXT);"
 
 	exec_sql mysql1 3306 "INSERT INTO $DB1.$TBL1 (c1, c2) VALUES (1, '1');"
 	exec_sql mysql1 3306 "INSERT INTO $DB1.$TBL2 (c1, c2) VALUES (2, '2');"
-	exec_sql mysql2 3306 "INSERT INTO $DB2.$TBL2 (c1, c2) VALUES (11, '11');"
-	exec_sql mysql2 3306 "INSERT INTO $DB2.$TBL3 (c1, c2) VALUES (12, '12');"
+	exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL2 (c1, c2) VALUES (11, '11');"
+	exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL3 (c1, c2) VALUES (12, '12');"
 
 	# prepare optimsitic full data
 	exec_sql mysql1 3306 "CREATE DATABASE $DB3;"
-	exec_sql mysql2 3306 "CREATE DATABASE $DB4;"
+	exec_sql mariadb2 3306 "CREATE DATABASE $DB4;"
 	exec_sql mysql1 3306 "CREATE TABLE $DB3.$TBL1 (c1 INT PRIMARY KEY, c2 TEXT);"
 	exec_sql mysql1 3306 "CREATE TABLE $DB3.$TBL2 (c1 INT PRIMARY KEY, c2 TEXT);"
-	exec_sql mysql2 3306 "CREATE TABLE $DB4.$TBL2 (c1 INT PRIMARY KEY, c2 TEXT);"
-	exec_sql mysql2 3306 "CREATE TABLE $DB4.$TBL3 (c1 INT PRIMARY KEY, c2 TEXT);"
+	exec_sql mariadb2 3306 "CREATE TABLE $DB4.$TBL2 (c1 INT PRIMARY KEY, c2 TEXT);"
+	exec_sql mariadb2 3306 "CREATE TABLE $DB4.$TBL3 (c1 INT PRIMARY KEY, c2 TEXT);"
 
 	exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL1 (c1, c2) VALUES (1, '1');"
 	exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2) VALUES (2, '2');"
-	exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2) VALUES (11, '11');"
-	exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2) VALUES (12, '12');"
+	exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2) VALUES (11, '11');"
+	exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2) VALUES (12, '12');"
 }
 
 function exec_incremental_stage1() {
 	# prepare incremental data
 	exec_sql mysql1 3306 "INSERT INTO $DB1.$TBL1 (c1, c2) VALUES (101, '101');"
 	exec_sql mysql1 3306 "INSERT INTO $DB1.$TBL2 (c1, c2) VALUES (102, '102');"
-	exec_sql mysql2 3306 "INSERT INTO $DB2.$TBL2 (c1, c2) VALUES (111, '111');"
-	exec_sql mysql2 3306 "INSERT INTO $DB2.$TBL3 (c1, c2) VALUES (112, '112');"
+	exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL2 (c1, c2) VALUES (111, '111');"
+	exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL3 (c1, c2) VALUES (112, '112');"
 
 	# prepare optimistic incremental data
 	exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL1 (c1, c2) VALUES (101, '101');"
 	exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2) VALUES (102, '102');"
-	exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2) VALUES (111, '111');"
-	exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2) VALUES (112, '112');"
+	exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2) VALUES (111, '111');"
+	exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2) VALUES (112, '112');"
 
 	# optimistic shard ddls
 	exec_sql mysql1 3306 "ALTER TABLE $DB3.$TBL1 ADD COLUMN c3 INT;"
 	exec_sql mysql1 3306 "ALTER TABLE $DB3.$TBL2 ADD COLUMN c4 INT;"
-	exec_sql mysql2 3306 "ALTER TABLE $DB4.$TBL2 ADD COLUMN c3 INT;"
-	exec_sql mysql2 3306 "ALTER TABLE $DB4.$TBL3 ADD COLUMN c4 INT;"
+	exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL2 ADD COLUMN c3 INT;"
+	exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL3 ADD COLUMN c4 INT;"
 
 	# prepare optimistic incremental data
 	exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL1 (c1, c2, c3) VALUES (103, '103', 103);"
 	exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c4) VALUES (104, '104', 104);"
-	exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3) VALUES (113, '113', 113);"
-	exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2, c4) VALUES (114, '113', 113);"
+	exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3) VALUES (113, '113', 113);"
+	exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2, c4) VALUES (114, '113', 113);"
 }
 
 function exec_incremental_stage2() {
 	# prepare incremental data
 	exec_sql mysql1 3306 "INSERT INTO $DB1.$TBL1 (c1, c2) VALUES (201, '201');"
 	exec_sql mysql1 3306 "INSERT INTO $DB1.$TBL2 (c1, c2) VALUES (202, '202');"
-	exec_sql mysql2 3306 "INSERT INTO $DB2.$TBL2 (c1, c2) VALUES (211, '211');"
-	exec_sql mysql2 3306 "INSERT INTO $DB2.$TBL3 (c1, c2) VALUES (212, '212');"
+	exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL2 (c1, c2) VALUES (211, '211');"
+	exec_sql mariadb2 3306 "INSERT INTO $DB2.$TBL3 (c1, c2) VALUES (212, '212');"
 
 	# prepare optimistic incremental data
 	exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL1 (c1, c2, c3) VALUES (201, '201', 201);"
 	exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c4) VALUES (202, '202', 202);"
-	exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3) VALUES (211, '211', 211);"
-	exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2, c4) VALUES (212, '212', 212);"
+	exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3) VALUES (211, '211', 211);"
+	exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2, c4) VALUES (212, '212', 212);"
 
 	# optimistic shard ddls
 	exec_sql mysql1 3306 "ALTER TABLE $DB3.$TBL1 ADD COLUMN c4 INT;"
 	exec_sql mysql1 3306 "ALTER TABLE $DB3.$TBL2 ADD COLUMN c3 INT AFTER c2;"
-	exec_sql mysql2 3306 "ALTER TABLE $DB4.$TBL2 ADD COLUMN c4 INT;"
-	exec_sql mysql2 3306 "ALTER TABLE $DB4.$TBL3 ADD COLUMN c3 INT AFTER c2;"
+	exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL2 ADD COLUMN c4 INT;"
+	exec_sql mariadb2 3306 "ALTER TABLE $DB4.$TBL3 ADD COLUMN c3 INT AFTER c2;"
 
 	# prepare optimistic incremental data
 	exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL1 (c1, c2, c3, c4) VALUES (203, '203', 203, 203);"
 	exec_sql mysql1 3306 "INSERT INTO $DB3.$TBL2 (c1, c2, c3, c4) VALUES (204, '204', 204, 204);"
-	exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3, c4) VALUES (213, '213', 213, 213);"
-	exec_sql mysql2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2, c3, c4) VALUES (214, '214', 214, 214);"
+	exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL2 (c1, c2, c3, c4) VALUES (213, '213', 213, 213);"
+	exec_sql mariadb2 3306 "INSERT INTO $DB4.$TBL3 (c1, c2, c3, c4) VALUES (214, '214', 214, 214);"
 }
 
 function patch_nightly_with_tiup_mirror() {

tests/tiup/upgrade-from-v1.sh

@@ -107,7 +107,7 @@ function migrate_in_v2 {
 	exec_incremental_stage2
 
 	echo "check sources"
-	run_dmctl_with_retry $DM_V2_VER "operate-source show" "mysql-replica-01" 1 "mysql-replica-02" 1
+	run_dmctl_with_retry $DM_V2_VER "operate-source show" "mysql-replica-01" 1 "mariadb-replica-02" 1
 	echo "check workers"
 	run_dmctl_with_retry $DM_V2_VER "list-member --worker" "\"stage\": \"bound\"" 2
 

tests/tiup/upgrade-from-v2.sh

@@ -27,9 +27,10 @@ function deploy_previous_v2() {
 function migrate_in_previous_v2() {
 	exec_full_stage
 
-	# v2.0.0 doesn't support relay log
+	# v2.0.0 doesn't implement relay log, and enable-gtid for MariaDB has a bug
 	if [[ "$PRE_VER" == "v2.0.0" ]]; then
 		sed -i "s/enable-relay: true/enable-relay: false/g" $CUR/conf/source2.yaml
+		sed -i "s/enable-gtid: true/enable-gtid: false/g" $CUR/conf/source2.yaml
 	fi
 
 	tiup dmctl:$PRE_VER --master-addr=master1:8261 operate-source create $CUR/conf/source1.yaml
@@ -55,7 +56,7 @@ function migrate_in_v2 {
 	exec_incremental_stage2
 
 	echo "check sources"
-	run_dmctl_with_retry $CUR_VER "operate-source show" "mysql-replica-01" 1 "mysql-replica-02" 1
+	run_dmctl_with_retry $CUR_VER "operate-source show" "mysql-replica-01" 1 "mariadb-replica-02" 1
 	echo "check workers"
 	run_dmctl_with_retry $CUR_VER "list-member --worker" "\"stage\": \"bound\"" 2