dm: add foreign key test for full phase (#7599)

close #7598

dm/loader/lightning.go

@@ -315,6 +315,8 @@ func GetLightningConfig(globalCfg *lcfg.GlobalConfig, subtaskCfg *config.SubTask
 	cfg.TiDB.Vars = map[string]string{
 		// always set transaction mode to optimistic
 		"tidb_txn_mode": "optimistic",
+		// always disable foreign key check when do full sync.
+		"foreign_key_checks": "0",
 	}
 	cfg.Mydumper.SourceID = subtaskCfg.SourceID
 	return cfg, nil

dm/tests/foreign_key/conf/diff_config.toml

@@ -0,0 +1,30 @@
+# diff Configuration.
+
+check-thread-count = 4
+
+export-fix-sql = true
+
+check-struct-only = false
+
+[task]
+    output-dir = "/tmp/ticdc_dm_test/output"
+
+    source-instances = ["mysql1"]
+
+    target-instance = "tidb0"
+
+    target-check-tables = ["foreign_key.?*"]
+
+
+[data-sources]
+[data-sources.mysql1]
+host = "127.0.0.1"
+port = 3306
+user = "root"
+password = "123456"
+
+[data-sources.tidb0]
+host = "127.0.0.1"
+port = 4000
+user = "test"
+password = "123456"

dm/tests/foreign_key/conf/dm-master.toml

@@ -0,0 +1,4 @@
+# Master Configuration.
+master-addr = ":8261"
+advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

dm/tests/foreign_key/conf/dm-task.yaml

@@ -0,0 +1,43 @@
+---
+name: test
+task-mode: all
+is-sharding: false
+meta-schema: "dm_meta"
+# enable-heartbeat: true
+heartbeat-update-interval: 1
+heartbeat-report-interval: 1
+
+target-database:
+  host: "127.0.0.1"
+  port: 4000
+  user: "root"
+  password: ""
+
+mysql-instances:
+  - source-id: "mysql-replica-01"
+    black-white-list:  "instance" # compatible with deprecated config
+    mydumper-config-name: "global"
+    loader-config-name: "global"
+    syncer-config-name: "global"
+
+black-white-list: # compatible with deprecated config
+  instance:
+    do-dbs: ["foreign_key"]
+
+mydumpers:
+  global:
+    threads: 4
+    chunk-filesize: 64
+    skip-tz-utc: true
+    extra-args: ""
+
+loaders:
+  global:
+    pool-size: 16
+    dir: "./dumped_data"
+
+syncers:
+  global:
+    worker-count: 16
+    batch: 100
+    enable-ansi-quotes: true

dm/tests/foreign_key/conf/dm-worker1.toml

@@ -0,0 +1,2 @@
+name = "worker1"
+join = "127.0.0.1:8261"

dm/tests/foreign_key/conf/source1.yaml

@@ -0,0 +1,15 @@
+source-id: mysql-replica-01
+flavor: ''
+enable-gtid: false
+enable-relay: true
+relay-binlog-name: ''
+relay-binlog-gtid: ''
+from:
+  host: 127.0.0.1
+  user: root
+  password: /Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=
+  port: 3306
+checker:
+  check-enable: true
+  backoff-rollback: 5m
+  backoff-max: 5m

dm/tests/foreign_key/data/db1.prepare.sql

@@ -0,0 +1,61 @@
+set @@foreign_key_checks=1;
+drop database if exists `foreign_key`;
+create database `foreign_key`;
+use `foreign_key`;
+
+-- Check foreign key on delete/update cascade.
+create table t1 (id int key);
+create table t2 (id int key, foreign key (id) references t1 (id) on delete cascade on update cascade);
+begin;
+insert into t1 values (1),(2),(3),(4),(5);
+insert into t2 values (1),(2),(3),(4),(5);
+delete from t1 where id in (1,2);
+update t1 set id=id+10 where id =3;
+commit;
+set @@foreign_key_checks=0;
+delete from t1 where id=4;
+set @@foreign_key_checks=1;
+
+-- Check foreign key on delete/update set null.
+create table t3 (id int key);
+create table t4 (id int key, b int, foreign key (b) references t3 (id) on delete set null on update set null);
+begin;
+insert into t3 values (1),(2),(3),(4),(5);
+insert into t4 values (1, 1),(2, 2),(3, 3),(4, 4),(5, 5);
+delete from t3 where id in (1,2);
+update t3 set id=id+10 where id =3;
+commit;
+
+-- Check foreign key on delete/update restrict.
+create table t5 (id int key);
+create table t6 (id int key, foreign key (id) references t5 (id) on delete restrict on update restrict);
+insert into t5 values (1),(2),(3),(4),(5);
+insert into t6 values (1),(2),(3),(4),(5);
+set @@foreign_key_checks=0;
+delete from t5 where id in (1,2);
+update t5 set id=id+10 where id =3;
+delete from t5 where id=4;
+set @@foreign_key_checks=1;
+
+-- Check foreign key on ddl drop table.
+create table t7 (id int key);
+create table t8 (id int key, foreign key (id) references t5 (id) on delete restrict on update restrict);
+drop table t7, t8;
+
+-- Test for cascade delete.
+create table t9 (id int key, name varchar(10), leader int,  index(leader), foreign key (leader) references t9(id) ON DELETE CASCADE);
+insert into t9 values (1, 'boss', null), (10, 'l1_a', 1), (11, 'l1_b', 1), (12, 'l1_c', 1);
+insert into t9 values (100, 'l2_a1', 10), (101, 'l2_a2', 10), (102, 'l2_a3', 10);
+insert into t9 values (110, 'l2_b1', 11), (111, 'l2_b2', 11), (112, 'l2_b3', 11);
+insert into t9 values (120, 'l2_c1', 12), (121, 'l2_c2', 12), (122, 'l2_c3', 12);
+insert into t9 values (1000,'l3_a1', 100);
+delete from t9 where id=1;
+
+-- Test ddl add foreign key.
+create table t10 (id int key, b int, index(b));
+create table t11 (id int key, b int);
+insert into t10 values (1,1),(2,2),(3,3);
+insert into t11 values (1,1),(2,2),(3,3);
+alter table t11 add foreign key (b) references t10(id) on delete cascade on update cascade;
+delete from t10 where id=1;
+update t10 set id=id+10 where id=2;

dm/tests/foreign_key/run.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -eu
+
+cur=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+source $cur/../_utils/test_prepare
+WORK_DIR=$TEST_DIR/$TEST_NAME
+
+function run() {
+	run_sql_tidb "set @@global.tidb_enable_foreign_key=1;"
+	run_sql_tidb "set @@global.foreign_key_checks=1;"
+	run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1
+	# start DM worker and master
+	run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml
+	check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT
+	run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
+	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT
+
+	# operate mysql config to worker
+	cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml
+	sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml
+	dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1
+
+	run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+		"start-task $cur/conf/dm-task.yaml --remove-meta"
+	run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+		"query-status test" \
+		"\"relayCatchUpMaster\": true" 1
+
+	# use sync_diff_inspector to check full dump loader
+	check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
+}
+
+cleanup_data foreign_key
+# also cleanup dm processes in case of last run failed
+cleanup_process $*
+run $*
+cleanup_process $*
+
+echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>"

dm/tests/others_integration_2.txt

@@ -1,3 +1,4 @@
+foreign_key
 load_task
 downstream_more_column
 expression_filter