Skip to content
This repository has been archived by the owner on Nov 24, 2023. It is now read-only.

test: add online ddl test in integration test #84

Merged
merged 12 commits into from
Mar 19, 2019
Merged

test: add online ddl test in integration test #84

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
b0910c3
test: intergration test script tuning
amyangfei Mar 15, 2019
8318a7b
test: add online ddl test
amyangfei Mar 15, 2019
40ba707
*: fix mysql port
amyangfei Mar 16, 2019
2debe90
*: simplify sql file in online ddl test
amyangfei Mar 16, 2019
132e93a
test: add pt-osc test. refine scripts
amyangfei Mar 18, 2019
3a0eb43
test: enable both gh-ost and pt-osc in online ddl test
amyangfei Mar 18, 2019
ea0a039
test: fix cleanup target db
amyangfei Mar 18, 2019
6cabb4c
*: fix parameter sequence in run_sql_file
amyangfei Mar 18, 2019
7d13afd
test: update test readme
amyangfei Mar 18, 2019
e9d5aad
test: remove unused interface in online_ddl.
amyangfei Mar 18, 2019
1d5e6ac
Update tests/README.md
csuzhangxc Mar 19, 2019
8747c6e
*: DDL typo
amyangfei Mar 18, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion tests/_utils/run_sql_file
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
#!/bin/sh
# parameter 1: sql file
# parameter 3: host
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Err must keep the list in order 😕

# parameter 2: port

set -eu

echo "[$(date)] Executing SQL: $1" > "$TEST_DIR/sql_res.$TEST_NAME.txt"
mysql -uroot -h127.0.0.1 -P$2 --default-character-set utf8 -vv < "$1" >> "$TEST_DIR/sql_res.$TEST_NAME.txt"
mysql -uroot -h$2 -P$3 --default-character-set utf8 -vv < "$1" >> "$TEST_DIR/sql_res.$TEST_NAME.txt"
28 changes: 17 additions & 11 deletions tests/_utils/run_sql_file_online_ddl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,18 @@
#!/bin/bash
# parameter 1: sql file
# parameter 2: port
# parameter 3: db
# parameter 4: online ddl tool, pt-osc or gh-ost
# parameter 2: host
# parameter 3: port
# parameter 4: db
# parameter 5: online ddl tool, pt or gh-ost

set -eu

sql_file=$1
port=$2
schema=$3
host=$2
port=$3
schema=$4
ghost_bin=${GHOST_BINARY:-gh-ost}
ptosc_bin=${PTOSC_BINARY:-pt-online-schema-change}

echo "[$(date)] Executing SQL: $sql_file" > "$TEST_DIR/sql_res.$TEST_NAME.txt"

Expand All @@ -24,19 +28,21 @@ do
# if we have test MySQL in container and port mapping, these two ports
# may different. So we cheat gh-ost that we are running on aliyun rds,
# on which will disable the port check.
if [ "$4" == "gh-ost" ]; then
gh-ost --user=root --host=127.0.0.1 --port=$port \
if [ "$5" == "gh-ost" ]; then
$ghost_bin --user=root --host=$host --port=$port \
--database=$schema --table=$table --alter="$alter" \
--serve-socket-file="$TEST_DIR/gh-ost.$schema.$table.$port.sock" \
--allow-on-master --allow-master-master --initially-drop-ghost-table \
--initially-drop-old-table -ok-to-drop-table -aliyun-rds -execute \
>> $TEST_DIR/gh-ost.log
elif [ "$4" == "pt-osc" ]; then
pt-online-schema-change --user=root --host=127.0.0.1 --port=$port --alter="$alter" D=$schame,t=$table --print --execute
elif [ "$5" == "pt" ]; then
$ptosc_bin --user=root --host=$host --port=$port \
--alter="$alter" D=$schema,t=$table \
--recursion-method=none --print --execute
else
mysql -uroot -h127.0.0.1 -P$port --default-character-set utf8 -E -e "$use $schema; $line" >> "$TEST_DIR/sql_res.$TEST_NAME.txt"
mysql -uroot -h$host -P$port --default-character-set utf8 -E -e "use $schema; $line" >> "$TEST_DIR/sql_res.$TEST_NAME.txt"
fi
else
mysql -uroot -h127.0.0.1 -P$port --default-character-set utf8 -E -e "use $schema; $line" >> "$TEST_DIR/sql_res.$TEST_NAME.txt"
mysql -uroot -h$host -P$port --default-character-set utf8 -E -e "use $schema; $line" >> "$TEST_DIR/sql_res.$TEST_NAME.txt"
fi
done <"$sql_file"
2 changes: 2 additions & 0 deletions tests/_utils/test_prepare
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
MYSQL_HOST1=${MYSQL_HOST1:-127.0.0.1}
MYSQL_HOST2=${MYSQL_HOST2:-127.0.0.1}
MYSQL_PORT1=${MYSQL_PORT1:-3306}
MYSQL_PORT2=${MYSQL_PORT2:-3307}
TIDB_PORT=4000
Expand Down
8 changes: 4 additions & 4 deletions tests/all_mode/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ source $cur/../_utils/test_prepare
WORK_DIR=$TEST_DIR/$TEST_NAME

function run() {
run_sql_file $cur/data/db1.prepare.sql $MYSQL_PORT1
run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1
check_contains 'Query OK, 2 rows affected'
run_sql_file $cur/data/db2.prepare.sql $MYSQL_PORT2
run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2
check_contains 'Query OK, 3 rows affected'

run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
Expand All @@ -27,8 +27,8 @@ function run() {
# use sync_diff_inspector to check full dump loader
check_sync_diff $WORK_DIR $cur/conf/diff_config.toml

run_sql_file $cur/data/db1.increment.sql $MYSQL_PORT1
run_sql_file $cur/data/db2.increment.sql $MYSQL_PORT2
run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1
run_sql_file $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2

# use sync_diff_inspector to check data now!
check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
Expand Down
2 changes: 1 addition & 1 deletion tests/online_ddl/conf/dm-task.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ meta-schema: "dm_meta"
remove-meta: false
disable-heartbeat: true
timezone: "Asia/Shanghai"
online-ddl-scheme: gh-ost
online-ddl-scheme: online-ddl-scheme-placeholder

target-database:
host: "127.0.0.1"
Expand Down
47 changes: 32 additions & 15 deletions tests/online_ddl/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@ set -eu

cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
source $cur/../_utils/test_prepare
WORK_DIR=$TEST_DIR/$TEST_NAME
ONLINE_DDL_ENABLE=${ONLINE_DDL_ENABLE:-true}
BASE_TEST_NAME=$TEST_NAME

function run() {
run_sql_file $cur/data/db1.prepare.sql $MYSQL_PORT1
function real_run() {
online_ddl_scheme=$1
run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1
check_contains 'Query OK, 2 rows affected'
run_sql_file $cur/data/db2.prepare.sql $MYSQL_PORT2
run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2
check_contains 'Query OK, 3 rows affected'

run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
Expand All @@ -22,27 +24,42 @@ function run() {

cd $cur && GO111MODULE=on go build -o bin/dmctl && cd -
# start DM task only
$cur/bin/dmctl "$cur/conf/dm-task.yaml"
cp $cur/conf/dm-task.yaml $WORK_DIR/dm-task-${online_ddl_scheme}.yaml
sed -i "s/online-ddl-scheme-placeholder/${online_ddl_scheme}/g" $WORK_DIR/dm-task-${online_ddl_scheme}.yaml
$cur/bin/dmctl "$WORK_DIR/dm-task-${online_ddl_scheme}.yaml"

# TODO: check sharding partition id
# use sync_diff_inspector to check full dump loader
check_sync_diff $WORK_DIR $cur/conf/diff_config.toml

run_sql_file_online_ddl $cur/data/db1.increment.sql $MYSQL_PORT1 $TEST_NAME gh-ost
run_sql_file_online_ddl $cur/data/db2.increment.sql $MYSQL_PORT2 $TEST_NAME gh-ost
run_sql_file_online_ddl $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 online_ddl $online_ddl_scheme
run_sql_file_online_ddl $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2 online_ddl $online_ddl_scheme

# TODO: check sharding partition id
# use sync_diff_inspector to check data now!
check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
}

cleanup1 $*
# also cleanup dm processes in case of last run failed
cleanup2 $*
run $*
cleanup2 $*
function run() {
online_ddl_scheme=$1
TEST_NAME=${BASE_TEST_NAME}_$online_ddl_scheme
WORK_DIR=$TEST_DIR/$TEST_NAME

cleanup1 $*
# also cleanup dm processes in case of last run failed
cleanup2 $*
real_run $*
cleanup2 $*

wait_process_exit dm-master.test
wait_process_exit dm-worker.test
wait_process_exit dm-master.test
wait_process_exit dm-worker.test

echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>"
}

echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>"
if [ "$ONLINE_DDL_ENABLE" == true ]; then
# run gh-ost
run pt
else
echo "[$(date)] <<<<<< skip online ddl test! >>>>>>"
fi
8 changes: 4 additions & 4 deletions tests/sharding/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ source $cur/../_utils/test_prepare
WORK_DIR=$TEST_DIR/$TEST_NAME

function run() {
run_sql_file $cur/data/db1.prepare.sql $MYSQL_PORT1
run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1
check_contains 'Query OK, 2 rows affected'
run_sql_file $cur/data/db2.prepare.sql $MYSQL_PORT2
run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2
check_contains 'Query OK, 3 rows affected'

run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
Expand All @@ -28,8 +28,8 @@ function run() {
# use sync_diff_inspector to check full dump loader
check_sync_diff $WORK_DIR $cur/conf/diff_config.toml

run_sql_file $cur/data/db1.increment.sql $MYSQL_PORT1
run_sql_file $cur/data/db2.increment.sql $MYSQL_PORT2
run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1
run_sql_file $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2

# TODO: check sharding partition id
# use sync_diff_inspector to check data now!
Expand Down