From f8ea47302edd35464593ae31a5fb45131842e9e3 Mon Sep 17 00:00:00 2001 From: amyangfei Date: Fri, 7 May 2021 15:37:37 +0800 Subject: [PATCH 1/8] tests: add random region merge test --- tests/region_merge/conf/diff_config.toml | 27 +++++++++ tests/region_merge/run.sh | 71 ++++++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 tests/region_merge/conf/diff_config.toml create mode 100644 tests/region_merge/run.sh diff --git a/tests/region_merge/conf/diff_config.toml b/tests/region_merge/conf/diff_config.toml new file mode 100644 index 00000000000..1a4cae161f9 --- /dev/null +++ b/tests/region_merge/conf/diff_config.toml @@ -0,0 +1,27 @@ +# diff Configuration. + +log-level = "info" +chunk-size = 10 +check-thread-count = 4 +sample-percent = 100 +use-rowid = false +use-checksum = true +fix-sql-file = "fix.sql" + +# tables need to check. +[[check-tables]] + schema = "region_merge" + tables = ["~.*"] + +[[source-db]] + host = "127.0.0.1" + port = 4000 + user = "root" + password = "" + instance-id = "source-1" + +[target-db] + host = "127.0.0.1" + port = 3306 + user = "root" + password = "" diff --git a/tests/region_merge/run.sh b/tests/region_merge/run.sh new file mode 100644 index 00000000000..cfe1fb40afe --- /dev/null +++ b/tests/region_merge/run.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +set -e + +CUR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +source $CUR/../_utils/test_prepare +WORK_DIR=$OUT_DIR/$TEST_NAME +CDC_BINARY=cdc.test +SINK_TYPE=$1 + +function split_and_random_merge() { + pd_addr=$1 + scale=$2 + echo "split_and_random_merge scale: $scale..." + pd-ctl --pd=$pd_addr scheduler remove random-merge-scheduler || true + run_sql "SPLIT TABLE region_merge.t1 BETWEEN (-9223372036854775808) AND (9223372036854775807) REGIONS $scale;" ${UP_TIDB_HOST} ${UP_TIDB_PORT} || true + pd-ctl --pd=$pd_addr scheduler add random-merge-scheduler + run_sql "insert into region_merge.t1 values (-9223372036854775808),(0),(1),(9223372036854775807);" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + run_sql "delete from region_merge.t1;" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + # sleep 5s to wait some region merge + sleep 5 +} + +large_scale=(100 200 400 800 1600 3200 6400 12800 25600 51200) +small_scale=(100 200 400 800) +# in CI, we use the small data set +test_scale=( "${small_scale[@]}" ) + +function run() { + rm -rf $WORK_DIR && mkdir -p $WORK_DIR + + start_tidb_cluster --workdir $WORK_DIR --multiple-upstream-pd true + + cd $WORK_DIR + + pd_addr="http://$UP_PD_HOST_1:$UP_PD_PORT_1" + run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY + TOPIC_NAME="ticdc-region-merge-test-$RANDOM" + case $SINK_TYPE in + kafka) SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?partition-num=4&kafka-version=${KAFKA_VERSION}";; + *) SINK_URI="mysql://root@127.0.0.1:3306/";; + esac + run_cdc_cli changefeed create --sink-uri="$SINK_URI" + if [ "$SINK_TYPE" == "kafka" ]; then + run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?partition-num=4&version=${KAFKA_VERSION}" + fi + + # set max_execution_time to 20s, because split region could block even region has been split. + run_sql "SET @@global.MAX_EXECUTION_TIME = 20000;" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + run_sql "CREATE DATABASE region_merge;" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + run_sql "CREATE TABLE region_merge.t1 (id bigint primary key);" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + + # decrease the split-merge-interval to trigger fast region merge + pd-ctl --pd=$pd_addr config set split-merge-interval 1s + pd-ctl --pd=$pd_addr config set merge-schedule-limit 128 + + for scale in "${test_scale[@]}"; do + split_and_random_merge $pd_addr $scale + done + + run_sql "insert into region_merge.t1 values (-9223372036854775808),(0),(1),(9223372036854775807);" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + check_table_exists region_merge.t1 ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} + check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml + + cleanup_process $CDC_BINARY +} + +trap stop_tidb_cluster EXIT +run $* +check_logs $WORK_DIR +echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>" From d83a23dc519533423c68ee348d6ef0aa2c5096d8 Mon Sep 17 00:00:00 2001 From: amyangfei Date: Fri, 7 May 2021 18:15:53 +0800 Subject: [PATCH 2/8] add timeout to pd-ctl command --- tests/region_merge/run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/region_merge/run.sh b/tests/region_merge/run.sh index cfe1fb40afe..6b28d9b1779 100644 --- a/tests/region_merge/run.sh +++ b/tests/region_merge/run.sh @@ -12,9 +12,9 @@ function split_and_random_merge() { pd_addr=$1 scale=$2 echo "split_and_random_merge scale: $scale..." - pd-ctl --pd=$pd_addr scheduler remove random-merge-scheduler || true + timeout --signal=SIGINT 5 pd-ctl --pd=$pd_addr scheduler remove random-merge-scheduler || true run_sql "SPLIT TABLE region_merge.t1 BETWEEN (-9223372036854775808) AND (9223372036854775807) REGIONS $scale;" ${UP_TIDB_HOST} ${UP_TIDB_PORT} || true - pd-ctl --pd=$pd_addr scheduler add random-merge-scheduler + timeout --signal=SIGINT 5 pd-ctl --pd=$pd_addr scheduler add random-merge-scheduler run_sql "insert into region_merge.t1 values (-9223372036854775808),(0),(1),(9223372036854775807);" ${UP_TIDB_HOST} ${UP_TIDB_PORT} run_sql "delete from region_merge.t1;" ${UP_TIDB_HOST} ${UP_TIDB_PORT} # sleep 5s to wait some region merge @@ -51,8 +51,8 @@ function run() { run_sql "CREATE TABLE region_merge.t1 (id bigint primary key);" ${UP_TIDB_HOST} ${UP_TIDB_PORT} # decrease the split-merge-interval to trigger fast region merge - pd-ctl --pd=$pd_addr config set split-merge-interval 1s - pd-ctl --pd=$pd_addr config set merge-schedule-limit 128 + timeout --signal=SIGINT 5 pd-ctl --pd=$pd_addr -d config set split-merge-interval 1s + timeout --signal=SIGINT 5 pd-ctl --pd=$pd_addr -d config set merge-schedule-limit 128 for scale in "${test_scale[@]}"; do split_and_random_merge $pd_addr $scale From 9e65de6a75eacedd4fe68ec316c25326ebbaf7c1 Mon Sep 17 00:00:00 2001 From: amyangfei Date: Fri, 7 May 2021 19:19:27 +0800 Subject: [PATCH 3/8] pd-ctl accept sigint --- tests/region_merge/run.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/region_merge/run.sh b/tests/region_merge/run.sh index 6b28d9b1779..cb5fc528c7f 100644 --- a/tests/region_merge/run.sh +++ b/tests/region_merge/run.sh @@ -14,7 +14,7 @@ function split_and_random_merge() { echo "split_and_random_merge scale: $scale..." timeout --signal=SIGINT 5 pd-ctl --pd=$pd_addr scheduler remove random-merge-scheduler || true run_sql "SPLIT TABLE region_merge.t1 BETWEEN (-9223372036854775808) AND (9223372036854775807) REGIONS $scale;" ${UP_TIDB_HOST} ${UP_TIDB_PORT} || true - timeout --signal=SIGINT 5 pd-ctl --pd=$pd_addr scheduler add random-merge-scheduler + timeout --signal=SIGINT 5 pd-ctl --pd=$pd_addr scheduler add random-merge-scheduler || true run_sql "insert into region_merge.t1 values (-9223372036854775808),(0),(1),(9223372036854775807);" ${UP_TIDB_HOST} ${UP_TIDB_PORT} run_sql "delete from region_merge.t1;" ${UP_TIDB_HOST} ${UP_TIDB_PORT} # sleep 5s to wait some region merge @@ -51,8 +51,8 @@ function run() { run_sql "CREATE TABLE region_merge.t1 (id bigint primary key);" ${UP_TIDB_HOST} ${UP_TIDB_PORT} # decrease the split-merge-interval to trigger fast region merge - timeout --signal=SIGINT 5 pd-ctl --pd=$pd_addr -d config set split-merge-interval 1s - timeout --signal=SIGINT 5 pd-ctl --pd=$pd_addr -d config set merge-schedule-limit 128 + timeout --signal=SIGINT 5 pd-ctl --pd=$pd_addr -d config set split-merge-interval 1s || true + timeout --signal=SIGINT 5 pd-ctl --pd=$pd_addr -d config set merge-schedule-limit 128 || true for scale in "${test_scale[@]}"; do split_and_random_merge $pd_addr $scale From bc9979400ca9538002b0f62148b92a42960d1e83 Mon Sep 17 00:00:00 2001 From: amyangfei Date: Fri, 7 May 2021 21:25:43 +0800 Subject: [PATCH 4/8] add some log --- tests/region_merge/run.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/region_merge/run.sh b/tests/region_merge/run.sh index cb5fc528c7f..1898a2543a3 100644 --- a/tests/region_merge/run.sh +++ b/tests/region_merge/run.sh @@ -11,10 +11,11 @@ SINK_TYPE=$1 function split_and_random_merge() { pd_addr=$1 scale=$2 - echo "split_and_random_merge scale: $scale..." - timeout --signal=SIGINT 5 pd-ctl --pd=$pd_addr scheduler remove random-merge-scheduler || true + echo "split_and_random_merge scale: $scale" + timeout --signal=SIGINT 3 pd-ctl --pd=$pd_addr scheduler remove random-merge-scheduler || true run_sql "SPLIT TABLE region_merge.t1 BETWEEN (-9223372036854775808) AND (9223372036854775807) REGIONS $scale;" ${UP_TIDB_HOST} ${UP_TIDB_PORT} || true - timeout --signal=SIGINT 5 pd-ctl --pd=$pd_addr scheduler add random-merge-scheduler || true + run_sql "SELECT count(distinct region_id) from tikv_region_status where db_name = 'region_merge' and table_name = 't1';" && cat $OUT_DIR/sql_res.region_merge.txt + timeout --signal=SIGINT 3 pd-ctl --pd=$pd_addr scheduler add random-merge-scheduler || true run_sql "insert into region_merge.t1 values (-9223372036854775808),(0),(1),(9223372036854775807);" ${UP_TIDB_HOST} ${UP_TIDB_PORT} run_sql "delete from region_merge.t1;" ${UP_TIDB_HOST} ${UP_TIDB_PORT} # sleep 5s to wait some region merge @@ -51,8 +52,8 @@ function run() { run_sql "CREATE TABLE region_merge.t1 (id bigint primary key);" ${UP_TIDB_HOST} ${UP_TIDB_PORT} # decrease the split-merge-interval to trigger fast region merge - timeout --signal=SIGINT 5 pd-ctl --pd=$pd_addr -d config set split-merge-interval 1s || true - timeout --signal=SIGINT 5 pd-ctl --pd=$pd_addr -d config set merge-schedule-limit 128 || true + timeout --signal=SIGINT 3 pd-ctl --pd=$pd_addr -d config set split-merge-interval 1s || true + timeout --signal=SIGINT 3 pd-ctl --pd=$pd_addr -d config set merge-schedule-limit 128 || true for scale in "${test_scale[@]}"; do split_and_random_merge $pd_addr $scale From 3cd3cae526a796b69b63c34e7d9cd85747acd01e Mon Sep 17 00:00:00 2001 From: amyangfei Date: Fri, 7 May 2021 21:44:04 +0800 Subject: [PATCH 5/8] fix no database selected --- tests/region_merge/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/region_merge/run.sh b/tests/region_merge/run.sh index 1898a2543a3..0f96b609e85 100644 --- a/tests/region_merge/run.sh +++ b/tests/region_merge/run.sh @@ -14,7 +14,7 @@ function split_and_random_merge() { echo "split_and_random_merge scale: $scale" timeout --signal=SIGINT 3 pd-ctl --pd=$pd_addr scheduler remove random-merge-scheduler || true run_sql "SPLIT TABLE region_merge.t1 BETWEEN (-9223372036854775808) AND (9223372036854775807) REGIONS $scale;" ${UP_TIDB_HOST} ${UP_TIDB_PORT} || true - run_sql "SELECT count(distinct region_id) from tikv_region_status where db_name = 'region_merge' and table_name = 't1';" && cat $OUT_DIR/sql_res.region_merge.txt + run_sql "SELECT count(distinct region_id) from information_schema.tikv_region_status where db_name = 'region_merge' and table_name = 't1';" && cat $OUT_DIR/sql_res.region_merge.txt timeout --signal=SIGINT 3 pd-ctl --pd=$pd_addr scheduler add random-merge-scheduler || true run_sql "insert into region_merge.t1 values (-9223372036854775808),(0),(1),(9223372036854775807);" ${UP_TIDB_HOST} ${UP_TIDB_PORT} run_sql "delete from region_merge.t1;" ${UP_TIDB_HOST} ${UP_TIDB_PORT} From e2db222ff74a7ca35ad5b0d1b8b76760c3f6b0ef Mon Sep 17 00:00:00 2001 From: amyangfei Date: Fri, 7 May 2021 22:30:52 +0800 Subject: [PATCH 6/8] use pd server config instead of pd-ctl control --- tests/_utils/start_tidb_cluster_impl | 13 ++++++++++++- tests/region_merge/conf/pd_config.toml | 18 ++++++++++++++++++ tests/region_merge/run.sh | 11 +++-------- 3 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 tests/region_merge/conf/pd_config.toml diff --git a/tests/_utils/start_tidb_cluster_impl b/tests/_utils/start_tidb_cluster_impl index a5e23b58ad6..762d5015291 100755 --- a/tests/_utils/start_tidb_cluster_impl +++ b/tests/_utils/start_tidb_cluster_impl @@ -8,6 +8,7 @@ set -e OUT_DIR= tidb_config= +pd_config= multiple_upstream_pd= while [[ ${1} ]]; do @@ -20,6 +21,10 @@ while [[ ${1} ]]; do tidb_config=${2} shift ;; + --pd-config) + pd_config=${2} + shift + ;; --multiple-upstream-pd) multiple_upstream_pd=${2} shift @@ -42,12 +47,18 @@ stop_tidb_cluster cd $OUT_DIR && echo "start tidb cluster in $OUT_DIR" -cat - >"$OUT_DIR/pd-config.toml" < $OUT_DIR/pd-config.toml +else + cat - >"$OUT_DIR/pd-config.toml" < Date: Fri, 7 May 2021 22:41:47 +0800 Subject: [PATCH 7/8] fix pd config --- tests/_utils/start_tidb_cluster | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/_utils/start_tidb_cluster b/tests/_utils/start_tidb_cluster index a308179efa4..b659c3ddb04 100755 --- a/tests/_utils/start_tidb_cluster +++ b/tests/_utils/start_tidb_cluster @@ -8,6 +8,7 @@ set -e OUT_DIR= tidb_config= +pd_config= retry_times=3 multiple_upstream_pd="false" @@ -21,6 +22,10 @@ while [[ ${1} ]]; do tidb_config=${2} shift ;; + --pd-config) + pd_config=${2} + shift + ;; --retry) retry_times=${2} shift @@ -51,6 +56,8 @@ do if [[ "$tidb_config" != "" ]]; then start_tidb_cluster_impl --workdir ${OUT_DIR} --multiple-upstream-pd ${multiple_upstream_pd} --tidb-config ${tidb_config} + elif [[ "$pd_config" != "" ]]; then + start_tidb_cluster_impl --workdir ${OUT_DIR} --multiple-upstream-pd ${multiple_upstream_pd} --pd-config ${pd_config} else start_tidb_cluster_impl --workdir ${OUT_DIR} --multiple-upstream-pd ${multiple_upstream_pd} fi From 4da7de45c40b7d0a2dcf9ea53bbff490d6a9e28d Mon Sep 17 00:00:00 2001 From: amyangfei Date: Fri, 7 May 2021 23:14:46 +0800 Subject: [PATCH 8/8] decrease ci run time --- tests/region_merge/run.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/region_merge/run.sh b/tests/region_merge/run.sh index 99f975cf607..863fe45f6ea 100644 --- a/tests/region_merge/run.sh +++ b/tests/region_merge/run.sh @@ -22,7 +22,7 @@ function split_and_random_merge() { } large_scale=(100 200 400 800 1600 3200 6400 12800 25600 51200) -small_scale=(100 200 400 800) +small_scale=(100 200 400) # in CI, we use the small data set test_scale=( "${small_scale[@]}" ) @@ -45,8 +45,8 @@ function run() { run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?partition-num=4&version=${KAFKA_VERSION}" fi - # set max_execution_time to 20s, because split region could block even region has been split. - run_sql "SET @@global.MAX_EXECUTION_TIME = 20000;" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + # set max_execution_time to 15s, because split region could block even region has been split. + run_sql "SET @@global.MAX_EXECUTION_TIME = 15000;" ${UP_TIDB_HOST} ${UP_TIDB_PORT} run_sql "CREATE DATABASE region_merge;" ${UP_TIDB_HOST} ${UP_TIDB_PORT} run_sql "CREATE TABLE region_merge.t1 (id bigint primary key);" ${UP_TIDB_HOST} ${UP_TIDB_PORT}