test: add online ddl test in integration test

syncer/ghost.go

@@ -14,7 +14,6 @@
 package syncer
 
 import (
-	"fmt"
 	"strings"
 
 	"github.com/pingcap/errors"
@@ -127,16 +126,6 @@ func (g *Ghost) Apply(tables []*filter.Table, statement string, stmt ast.StmtNod
 	return nil, schema, table, nil
 }
 
-// InOnlineDDL implements interface
-func (g *Ghost) InOnlineDDL(schema, table string) bool {
-	if g == nil {
-		return false
-	}
-
-	ghostInfo := g.storge.Get(schema, table)
-	return ghostInfo != nil
-}
-
 // Finish implements interface
 func (g *Ghost) Finish(schema, table string) error {
 	if g == nil {
@@ -172,25 +161,6 @@ func (g *Ghost) RealName(schema, table string) (string, string) {
 	return schema, table
 }
 
-// GhostName implements interface
-func (g *Ghost) GhostName(schema, table string) (string, string) {
-	tp := g.TableType(table)
-	if tp == ghostTable {
-		return schema, table
-	}
-
-	if tp == trashTable {
-		table = table[1 : len(table)-4]
-	}
-
-	return schema, fmt.Sprintf("_%s_gho", table)
-}
-
-// SchemeName implements interface
-func (g *Ghost) SchemeName() string {
-	return config.GHOST
-}
-
 // Clear clears online ddl information
 func (g *Ghost) Clear() error {
 	return errors.Trace(g.storge.Clear())

syncer/online_ddl.go

@@ -43,18 +43,12 @@ type OnlinePlugin interface {
 	// * apply online ddl on real table
 	// returns sqls, replaced/self schema, repliaced/slef table, error
 	Apply(tables []*filter.Table, statement string, stmt ast.StmtNode) ([]string, string, string, error)
-	// InOnlineDDL returns true if an online ddl is unresolved
-	InOnlineDDL(schema, table string) bool
 	// Finish would delete online ddl from memory and storage
 	Finish(schema, table string) error
 	// TableType returns ghhost/real table
 	TableType(table string) TableType
 	// RealName returns real table name that removed ghost suffix and handled by table router
 	RealName(schema, table string) (string, string)
-	// GhostName returns ghost table name of a table
-	GhostName(schema, table string) (string, string)
-	// SchemaName returns scheme name (gh-ost/pt)
-	SchemeName() string
 	// Clear clears all online information
 	Clear() error
 	// Close closes online ddl plugin

syncer/pt_osc.go

@@ -14,7 +14,6 @@
 package syncer
 
 import (
-	"fmt"
 	"strings"
 
 	"github.com/pingcap/errors"
@@ -127,16 +126,6 @@ func (p *PT) Apply(tables []*filter.Table, statement string, stmt ast.StmtNode)
 	return nil, schema, table, nil
 }
 
-// InOnlineDDL implements interface
-func (p *PT) InOnlineDDL(schema, table string) bool {
-	if p == nil {
-		return false
-	}
-
-	ghostInfo := p.storge.Get(schema, table)
-	return ghostInfo != nil
-}
-
 // Finish implements interface
 func (p *PT) Finish(schema, table string) error {
 	if p == nil {
@@ -173,26 +162,6 @@ func (p *PT) RealName(schema, table string) (string, string) {
 	return schema, table
 }
 
-// GhostName implements interface
-func (p *PT) GhostName(schema, table string) (string, string) {
-	tp := p.TableType(table)
-	if tp == ghostTable {
-		return schema, table
-	}
-
-	if tp == trashTable {
-		table = strings.TrimLeft(table, "_")
-		table = table[:len(table)-4]
-	}
-
-	return schema, fmt.Sprintf("_%s_new", table)
-}
-
-// SchemeName implements interface
-func (p *PT) SchemeName() string {
-	return config.PT
-}
-
 // Clear clears online ddl information
 func (p *PT) Clear() error {
 	return errors.Trace(p.storge.Clear())

tests/README.md

@@ -1,13 +1,15 @@
 
 ## Preparations
 
-1. The following five executables must be copied or generated or linked into these locations, `mydumper` and `sync_diff_inspector` can be downloaded from [tidb-enterprise-tools-latest-linux-amd64](http://download.pingcap.org/tidb-enterprise-tools-latest-linux-amd64.tar.gz):
+1. The following executables must be copied or generated or linked into these locations, `mydumper` and `sync_diff_inspector` can be downloaded from [tidb-enterprise-tools-latest-linux-amd64](http://download.pingcap.org/tidb-enterprise-tools-latest-linux-amd64.tar.gz):
 
     * `bin/tidb-server`
     * `bin/sync_diff_inspector`
     * `bin/mydumper`
     * `bin/dm-master.test` # generated by `make dm_integration_test_build`
     * `bin/dm-worker.test` # generated by `make dm_integration_test_build`
+    * [gh-ost](https://github.com/github/gh-ost) # must be added to path, or you can `export GHOST_BINARY=/path/to/gh-ost-binary`
+    * [pt-online-schema-change](https://www.percona.com/doc/percona-toolkit/LATEST/pt-online-schema-change.html) # must be added to path, or you can `export PTOSC_BINARY=/path/to/pt-osc-binary`
 
 2. The following programs must be installed:
 
@@ -33,7 +35,9 @@
     1. Check that all required executables exist.
     2. Execute `tests/run.sh`
 
-    > If want to run one integration test case only, just pass the CASE parameter, such as `make integration_test CASE=sharding`
+    > If want to run one integration test case only, just pass the CASE parameter, such as `make integration_test CASE=sharding`.
+
+    > The online DDL test using pt-osc doesn't work if the upstream MySQL has different connect port and bind port (often caused by port forwarding via NAT). In this case, you must specify the real IP and port of MySQL. Otherwise you can skip online DDL test by `export ONLINE_DDL_ENABLE=false`.
 
 4. After executing the tests, run `make coverage` to get a coverage report at `/tmp/dm_test/all_cov.html`.
 

tests/_utils/run_sql_file

@@ -1,8 +1,9 @@
 #!/bin/sh
 # parameter 1: sql file
-# parameter 2: port
+# parameter 2: host
+# parameter 3: port
 
 set -eu
 
 echo "[$(date)] Executing SQL: $1" > "$TEST_DIR/sql_res.$TEST_NAME.txt"
-mysql -uroot -h127.0.0.1 -P$2 --default-character-set utf8 -vv < "$1" >> "$TEST_DIR/sql_res.$TEST_NAME.txt"
+mysql -uroot -h$2 -P$3 --default-character-set utf8 -vv < "$1" >> "$TEST_DIR/sql_res.$TEST_NAME.txt"

tests/_utils/run_sql_file_online_ddl

@@ -0,0 +1,49 @@
+#!/bin/bash
+# parameter 1: sql file
+# parameter 2: host
+# parameter 3: port
+# parameter 4: db
+# parameter 5: online ddl tool, pt or gh-ost
+
+set -eu
+
+sql_file=$1
+host=$2
+port=$3
+schema=$4
+ghost_bin=${GHOST_BINARY:-gh-ost}
+ptosc_bin=${PTOSC_BINARY:-pt-online-schema-change}
+
+echo "[$(date)] Executing SQL: $sql_file" > "$TEST_DIR/sql_res.$TEST_NAME.txt"
+
+# we use lower case `alter table` in test sql, if want case insensetive,
+# just set `shopt -s nocasematch`
+ddl_regex="^alter table.*"
+while IFS= read -r line
+do
+    if [[ "$line" =~ $ddl_regex ]]; then
+        table=$(echo $line | cut -d " " -f3)
+        alter=$(echo $line | cut -d " " -f4-)
+        # gh-ost check connection port whether equals to `select @@global.port`.
+        # if we have test MySQL in container and port mapping, these two ports
+        # may different. So we cheat gh-ost that we are running on aliyun rds,
+        # on which will disable the port check.
+        if [ "$5" == "gh-ost" ]; then
+            $ghost_bin --user=root --host=$host --port=$port \
+                --database=$schema --table=$table --alter="$alter" \
+                --serve-socket-file="$TEST_DIR/gh-ost.$schema.$table.$port.sock" \
+                --allow-on-master --allow-master-master --initially-drop-ghost-table \
+                --initially-drop-old-table -ok-to-drop-table -aliyun-rds -execute \
+                >> $TEST_DIR/gh-ost.log
+        elif [ "$5" == "pt" ]; then
+            $ptosc_bin --user=root --host=$host --port=$port \
+                --alter="$alter" D=$schema,t=$table \
+                 --recursion-method=none --print --execute \
+                 >> $TEST_DIR/pt-osc.log
+        else
+            mysql -uroot -h$host -P$port --default-character-set utf8 -E -e "use $schema; $line" >> "$TEST_DIR/sql_res.$TEST_NAME.txt"
+        fi
+    else
+        mysql -uroot -h$host -P$port --default-character-set utf8 -E -e "use $schema; $line" >> "$TEST_DIR/sql_res.$TEST_NAME.txt"
+    fi
+done <"$sql_file"

tests/_utils/test_prepare

@@ -0,0 +1,23 @@
+MYSQL_HOST1=${MYSQL_HOST1:-127.0.0.1}
+MYSQL_HOST2=${MYSQL_HOST2:-127.0.0.1}
+MYSQL_PORT1=${MYSQL_PORT1:-3306}
+MYSQL_PORT2=${MYSQL_PORT2:-3307}
+TIDB_PORT=4000
+MASTER_PORT=8261
+WORKER1_PORT=8262
+WORKER2_PORT=8263
+
+# we do clean staff at beginning of each run, so we can keep logs of the latset run
+function cleanup1() {
+    rm -rf $WORK_DIR
+    mkdir $WORK_DIR
+    for target_db in "$@"; do
+        run_sql "drop database if exists ${target_db}" $TIDB_PORT
+    done
+    run_sql "drop database if exists dm_meta" $TIDB_PORT
+}
+
+function cleanup2() {
+    pkill -hup dm-worker.test 2>/dev/null || true
+    pkill -hup dm-master.test 2>/dev/null || true
+}

tests/all_mode/run.sh

@@ -3,32 +3,13 @@
 set -eu
 
 cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-PWD=$(pwd)
-DB1_PORT=3306
-DB2_PORT=3307
-TIDB_PORT=4000
-MASTER_PORT=8261
-WORKER1_PORT=8262
-WORKER2_PORT=8263
-WORK_DIR=$TEST_DIR/all_mode
-
-# we do clean staff at beginning of each run, so we can keep logs of the latset run
-function cleanup1() {
-    rm -rf $WORK_DIR
-    mkdir $WORK_DIR
-    run_sql "drop database if exists all_mode" $TIDB_PORT
-    run_sql "drop database if exists dm_meta" $TIDB_PORT
-}
-
-function cleanup2() {
-    pkill -hup dm-worker.test 2>/dev/null || true
-    pkill -hup dm-master.test 2>/dev/null || true
-}
+source $cur/../_utils/test_prepare
+WORK_DIR=$TEST_DIR/$TEST_NAME
 
 function run() {
-    run_sql_file $cur/data/db1.prepare.sql $DB1_PORT
+    run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1
     check_contains 'Query OK, 2 rows affected'
-    run_sql_file $cur/data/db2.prepare.sql $DB2_PORT
+    run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2
     check_contains 'Query OK, 3 rows affected'
 
     run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
@@ -46,14 +27,14 @@ function run() {
     # use sync_diff_inspector to check full dump loader
     check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
 
-    run_sql_file $cur/data/db1.increment.sql $DB1_PORT
-    run_sql_file $cur/data/db2.increment.sql $DB2_PORT
+    run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1
+    run_sql_file $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2
 
     # use sync_diff_inspector to check data now!
     check_sync_diff $WORK_DIR $cur/conf/diff_config.toml
 }
 
-cleanup1 $*
+cleanup1 all_mode
 # also cleanup dm processes in case of last run failed
 cleanup2 $*
 run $*

tests/online_ddl/conf/diff_config.toml

@@ -0,0 +1,57 @@
+# diff Configuration.
+
+log-level = "info"
+
+chunk-size = 1000
+
+check-thread-count = 4
+
+sample-percent = 100
+
+use-rowid = false
+
+use-checksum = true
+
+fix-sql-file = "fix.sql"
+
+# tables need to check.
+[[check-tables]]
+schema = "online_ddl"
+tables = ["t_target"]
+
+[[table-config]]
+schema = "online_ddl"
+table = "t_target"
+ignore-columns = ["id"]
+is-online_ddl = true
+index-field = "uid"
+
+[[table-config.source-tables]]
+instance-id = "source-1"
+schema = "online_ddl"
+table  = "~t.*"
+
+[[table-config.source-tables]]
+instance-id = "source-2"
+schema = "online_ddl"
+table  = "~t.*"
+
+[[source-db]]
+host = "127.0.0.1"
+port = 3306
+user = "root"
+password = ""
+instance-id = "source-1"
+
+[[source-db]]
+host = "127.0.0.1"
+port = 3307
+user = "root"
+password = ""
+instance-id = "source-2"
+
+[target-db]
+host = "127.0.0.1"
+port = 4000
+user = "root"
+password = ""

tests/online_ddl/conf/dm-master.toml

@@ -0,0 +1,9 @@
+# Master Configuration.
+
+[[deploy]]
+source-id = "mysql-replica-01"
+dm-worker = "127.0.0.1:8262"
+
+[[deploy]]
+source-id = "mysql-replica-02"
+dm-worker = "127.0.0.1:8263"