tests: add e2e test

tests/_utils/check_sync_diff

@@ -0,0 +1,34 @@
+#!/bin/bash
+# parameter 1: config file for sync_diff_inspector
+# parameter 2: max check times
+
+conf=$1
+if [ $# -ge 2 ]; then
+    check_time=$2
+else
+    check_time=10
+fi
+
+LOG=$DUMPLING_OUTPUT_DIR/sync_diff_inspector.log
+
+i=0
+while [ $i -lt $check_time ]
+do
+    bin/sync_diff_inspector --config=$conf >> $LOG 2>&1
+    ret=$?
+    if [ "$ret" == 0 ]; then
+        echo "check diff successfully"
+        break
+    fi
+    ((i++))
+    echo "check diff failed $i-th time, retry later"
+    sleep 2
+done
+
+if [ $i -ge $check_time ]; then
+    echo "check diff failed at last"
+    # show \n and other blanks
+    printf "$(cat $LOG)\n"
+    exit 1
+fi
+cd $PWD

tests/_utils/run_lightning

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -eu
+
+echo "[$(date)] Executing bin/tidb-lightning..."
+
+conf=$1
+
+bin/tidb-lightning -c $1

tests/_utils/run_services

@@ -0,0 +1,65 @@
+#!/bin/sh
+
+set -eu
+
+PD_ADDR="127.0.0.1:2379"
+TIDB_IP="127.0.0.1"
+TIDB_PORT="4000"
+TIDB_ADDR="127.0.0.1:4000"
+TIDB_STATUS_ADDR="127.0.0.1:10080"
+
+stop_services() {
+    killall -9 tidb-server || true
+
+    find "$DUMPLING_TEST_DIR" -maxdepth 1 -not -path "$DUMPLING_TEST_DIR" -not -name "*.log" | xargs rm -r || true
+}
+
+start_services() {
+    stop_services
+    echo "Starting TiDB..."
+
+    echo "Ensure mysql is started..."
+    i=0
+    while ! run_sql 'select 0 limit 0' > /dev/null; do
+       i=$((i+1))
+       if [ "$i" -gt 10 ]; then
+          echo 'Failed to ping MySQL Server'
+          exit 1
+       fi
+       sleep 3
+    done
+
+    echo "Starting PD..."
+    bin/pd-server \
+        --client-urls "http://$PD_ADDR" \
+        --log-file "$DUMPLING_TEST_DIR/pd.log" \
+        --data-dir "$DUMPLING_TEST_DIR/pd" &
+    # wait until PD is online...
+    i=0
+    while ! curl -o /dev/null -sf "http://$PD_ADDR/pd/api/v1/version"; do
+       i=$((i+1))
+       if [ "$i" -gt 10 ]; then
+          echo 'Failed to start PD'
+          exit 1
+       fi
+       sleep 3
+    done
+
+    bin/tidb-server \
+        -P 4000 \
+        --status 10080 \
+        --store mocktikv \
+        --log-file "$DUMPLING_TEST_DIR/tidb.log" &
+
+    echo "Verifying TiDB is started..."
+    i=0
+    while ! curl -o /dev/null -sf "http://$TIDB_IP:10080/status"; do
+        i=$((i+1))
+        if [ "$i" -gt 10 ]; then
+            echo 'Failed to start TiDB'
+            exit 1
+        fi
+        sleep 3
+    done
+}
+

tests/e2e/conf/diff_config.toml

@@ -0,0 +1,42 @@
+# diff Configuration.
+
+log-level = "info"
+
+chunk-size = 1000
+
+check-thread-count = 4
+
+sample-percent = 100
+
+use-rowid = false
+
+use-checksum = true
+
+fix-sql-file = "fix.sql"
+
+# tables need to check.
+[[check-tables]]
+schema = "e2e"
+tables = ["~t.*"]
+
+[[table-config]]
+schema = "e2e"
+table = "t"
+
+[[table-config.source-tables]]
+instance-id = "source-1"
+schema = "e2e"
+table  = "t"
+
+[[source-db]]
+host = "127.0.0.1"
+port = 3306
+user = "root"
+password = ""
+instance-id = "source-1"
+
+[target-db]
+host = "127.0.0.1"
+port = 4000
+user = "root"
+password = ""

tests/e2e/conf/lightning.toml

@@ -0,0 +1,22 @@
+### tidb-lightning config
+
+[lightning]
+server-mode = false
+level = "error"
+check-requirements = false
+file = "/tmp/dumpling_test_result/tidb-lightning.og"
+
+[tikv-importer]
+backend="tidb"
+on-duplicate = "error"
+
+[mydumper]
+data-source-dir = "/tmp/dumpling_test_result/sql_res.e2e"
+
+[tidb]
+host = "127.0.0.1"
+port = 4000
+user = "root"
+password = ""
+status-port = 10080
+pd-addr = "127.0.0.1:2379"

tests/e2e/run.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+set -eu
+cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+
+DB_NAME="e2e"
+TABLE_NAME="t"
+
+# drop database on tidb
+export DUMPLING_TEST_PORT=4000
+run_sql "drop database if exists $DB_NAME;"
+
+# drop database on mysql
+export DUMPLING_TEST_PORT=3306
+run_sql "drop database if exists $DB_NAME;"
+
+# build data on mysql
+run_sql "create database $DB_NAME;"
+run_sql "create table $DB_NAME.$TABLE_NAME (a int(255));"
+
+# insert 100 records
+i=0; while [ $i -lt 100 ]; do
+  run_sql "insert into $DB_NAME.$TABLE_NAME values (\"$i\");"
+  i=$(( i + 1 ))
+done
+
+# dumping
+export DUMPLING_TEST_DATABASE=$DB_NAME
+run_dumpling
+
+# use lightning import data to tidb
+run_lightning $cur/conf/lightning.toml
+
+# check mysql and tidb data
+check_sync_diff $cur/conf/diff_config.toml
+
+

tests/run.sh

@@ -18,20 +18,17 @@ set -eu
 
 mkdir -p "$DUMPLING_TEST_DIR"
 PATH="tests/_utils:$PATH"
+cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source $cur/_utils/run_services
 
-test_connection() {
-  i=0
-  while ! run_sql 'select 0 limit 0' > /dev/null; do
-      i=$((i+1))
-      if [ "$i" -gt 10 ]; then
-          echo 'Failed to ping MySQL Server'
-          exit 1
-      fi
-      sleep 3
-  done
-}
-
-test_connection
+file_should_exist bin/pd-server
+file_should_exist bin/tidb-server
+file_should_exist bin/tidb-lightning
+file_should_exist bin/dumpling
+file_should_exist bin/sync_diff_inspector
+
+trap stop_services EXIT
+start_services
 
 for script in tests/*/run.sh; do
     echo "Running test $script..."