diff --git a/.github/workflows/binlog-999999.yml b/.github/workflows/binlog-999999.yml new file mode 100644 index 0000000000..8d50ad62dd --- /dev/null +++ b/.github/workflows/binlog-999999.yml @@ -0,0 +1,100 @@ +name: Test binlog 999999 + +on: + push: + branches: + - master + - release-2.0 + pull_request: + branches: + - master + - release-2.0 + +jobs: + test-binlog-999999: + name: Test binlog 999999 + runs-on: ubuntu-18.04 + + steps: + - name: Set up Go 1.13 + uses: actions/setup-go@v2 + with: + go-version: 1.13 + + - name: Check out code + uses: actions/checkout@v2 + + - name: Cache go modules + uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-dm-${{ hashFiles('**/go.sum') }} + + - name: Cache tools + uses: actions/cache@v2 + with: + path: | + **/tools + key: ${{ runner.os }}-dm-tools-${{ hashFiles('**/tools/go.sum') }} + + - name: Build DM binary + run: make dm_integration_test_build + + - name: Setup CI environment + run: | + docker-compose -f ./tests/binlog_999999/docker-compose.yml up -d + curl http://download.pingcap.org/tidb-enterprise-tools-nightly-linux-amd64.tar.gz | tar xz + mv tidb-enterprise-tools-nightly-linux-amd64/bin/sync_diff_inspector bin/ + curl http://download.pingcap.org/tidb-nightly-linux-amd64.tar.gz | tar xz + mv tidb-nightly-linux-amd64/bin/tidb-server bin/ + + - name: change binlog sequence number to 999998 + run: | + while ! mysqladmin -h127.0.0.1 -P3306 -p123456 -uroot ping --connect-timeout=1 > /dev/null 2>&1 ; do + echo "wait mysql" + sleep 1 + done + while ! mysqladmin -h127.0.0.1 -P3307 -p123456 -uroot ping --connect-timeout=1 > /dev/null 2>&1 ; do + echo "wait mysql" + sleep 1 + done + + echo "reset master;" | mysql -uroot -h127.0.0.1 -P3306 -p123456 + echo "reset master;" | mysql -uroot -h127.0.0.1 -P3307 -p123456 + docker-compose -f ./tests/binlog_999999/docker-compose.yml down + + sudo mv /tmp/mysql1-data/mysql-bin.000001 /tmp/mysql1-data/mysql-bin.999998 + sudo bash -c "echo './mysql-bin.999998' > /tmp/mysql1-data/mysql-bin.index" + sudo mv /tmp/mysql2-data/mysql-bin.000001 /tmp/mysql2-data/mysql-bin.999998 + sudo bash -c "echo './mysql-bin.999998' > /tmp/mysql2-data/mysql-bin.index" + docker-compose -f ./tests/binlog_999999/docker-compose.yml up -d + + - name: Run test cases + run: | + RESET_MASTER=false make integration_test CASE=incremental_mode + echo "show binary logs;" | mysql -uroot -h127.0.0.1 -P3306 -p123456 | grep -q "mysql-bin.1000000" + echo "show binary logs;" | mysql -uroot -h127.0.0.1 -P3307 -p123456 | grep -q "mysql-bin.1000000" + + - name: Copy logs to hack permission + if: ${{ always() }} + run: | + mkdir ./logs + sudo cp -r -L /tmp/dm_test/incremental_mode/master/log ./logs/master + sudo cp -r -L /tmp/dm_test/incremental_mode/worker1/log ./logs/worker1 + sudo cp -r -L /tmp/dm_test/incremental_mode/worker2/log ./logs/worker2 + sudo chown -R runner ./logs + + # Update logs as artifact seems not stable, so we set `continue-on-error: true` here. + - name: Upload logs + continue-on-error: true + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: test-binlog-999999 + path: | + ./logs + + # Debug via SSH if previous steps failed + - name: Set up tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v2 \ No newline at end of file diff --git a/go.mod b/go.mod index 2bb12489c0..f37b036d89 100644 --- a/go.mod +++ b/go.mod @@ -49,3 +49,5 @@ require ( ) go 1.13 + +replace github.com/siddontang/go-mysql v1.1.1-0.20200824131207-0c5789dd0bd3 => github.com/lance6716/go-mysql v1.1.1-0.20210303100354-b0e44c2c5623 diff --git a/go.sum b/go.sum index f06d78554c..de74ec2645 100644 --- a/go.sum +++ b/go.sum @@ -599,6 +599,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lance6716/go-mysql v1.1.1-0.20210303100354-b0e44c2c5623 h1:HesfLUZvjUaluKJzKetJ6odasdH/WQ/Tc8UO1oAs0WI= +github.com/lance6716/go-mysql v1.1.1-0.20210303100354-b0e44c2c5623/go.mod h1:+W4RCzesQDI11HvIkaDjS8yM36SpAnGNQ7jmTLn5BnU= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v0.0.0-20181016162627-9eb73efc1fcc/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -1071,8 +1073,6 @@ github.com/siddontang/go-log v0.0.0-20190221022429-1e957dd83bed h1:KMgQoLJGCq1Io github.com/siddontang/go-log v0.0.0-20190221022429-1e957dd83bed/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4= github.com/siddontang/go-mysql v0.0.0-20200222075837-12e89848f047 h1:boyJ8EgQN/aC3grvx8QUoJrptt7RvneezSJSCbW25a4= github.com/siddontang/go-mysql v0.0.0-20200222075837-12e89848f047/go.mod h1:+W4RCzesQDI11HvIkaDjS8yM36SpAnGNQ7jmTLn5BnU= -github.com/siddontang/go-mysql v1.1.1-0.20200824131207-0c5789dd0bd3 h1:t4iXtvFWrw0yfgf4Bm/KJtf4jOSnuXgs93ML+yLslSc= -github.com/siddontang/go-mysql v1.1.1-0.20200824131207-0c5789dd0bd3/go.mod h1:+W4RCzesQDI11HvIkaDjS8yM36SpAnGNQ7jmTLn5BnU= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= diff --git a/pkg/binlog/filename.go b/pkg/binlog/filename.go index f70451566c..83d26412b1 100644 --- a/pkg/binlog/filename.go +++ b/pkg/binlog/filename.go @@ -35,17 +35,17 @@ type Filename struct { // LessThan checks whether this filename < other filename. func (f Filename) LessThan(other Filename) bool { - return f.BaseName == other.BaseName && f.Seq < other.Seq + return f.BaseName == other.BaseName && f.SeqInt64 < other.SeqInt64 } // GreaterThanOrEqualTo checks whether this filename >= other filename. func (f Filename) GreaterThanOrEqualTo(other Filename) bool { - return f.BaseName == other.BaseName && f.Seq >= other.Seq + return f.BaseName == other.BaseName && f.SeqInt64 >= other.SeqInt64 } // GreaterThan checks whether this filename > other filename. func (f Filename) GreaterThan(other Filename) bool { - return f.BaseName == other.BaseName && f.Seq > other.Seq + return f.BaseName == other.BaseName && f.SeqInt64 > other.SeqInt64 } // ParseFilename parses a string representation binlog filename into a `Filename`. diff --git a/pkg/binlog/filename_test.go b/pkg/binlog/filename_test.go index beba71a5a2..4234e864a3 100644 --- a/pkg/binlog/filename_test.go +++ b/pkg/binlog/filename_test.go @@ -26,18 +26,22 @@ func (t *testFilenameSuite) TestFilenameCmp(c *C) { f1 := Filename{ BaseName: "mysql-bin", Seq: "000001", + SeqInt64: 1, } f2 := Filename{ BaseName: "mysql-bin", Seq: "000002", + SeqInt64: 2, } f3 := Filename{ BaseName: "mysql-bin", Seq: "000001", // == f1 + SeqInt64: 1, } f4 := Filename{ BaseName: "bin-mysq", // diff BaseName Seq: "000001", + SeqInt64: 1, } c.Assert(f1.LessThan(f2), IsTrue) diff --git a/relay/writer/file.go b/relay/writer/file.go index a74679047c..6d069922a0 100644 --- a/relay/writer/file.go +++ b/relay/writer/file.go @@ -241,7 +241,7 @@ func (w *FileWriter) handleRotateEvent(ev *replication.BinlogEvent) (result Resu if err == nil { // update binlog filename if needed nextFile := string(rotateEv.NextLogName) - if nextFile > currFile { + if gmysql.CompareBinlogFileName(nextFile, currFile) == 1 { // record the next filename, but not create it. // even it's a fake RotateEvent, we still need to record it, // because if we do not specify the filename when creating the writer (like Auto-Position), diff --git a/tests/binlog_999999/docker-compose.yml b/tests/binlog_999999/docker-compose.yml new file mode 100644 index 0000000000..80468800af --- /dev/null +++ b/tests/binlog_999999/docker-compose.yml @@ -0,0 +1,43 @@ +version: "3.7" + +networks: + db-networks: + ipam: # for static IP + config: + - subnet: "172.28.128.0/24" + +# re-usable extension fields, https://docs.docker.com/compose/compose-file/#extension-fields. +x-node: &default-node + privileged: true + restart: always + environment: + MYSQL_ROOT_PASSWORD: "123456" + +services: + mysql1: # upstream MySQL-1 + <<: *default-node + container_name: mysql1 + hostname: mysql1 + ports: + - "3306:3306" + networks: + db-networks: + ipv4_address: 172.28.128.4 + image: mysql:5.7.32 + command: --default-authentication-plugin=mysql_native_password --log-bin=/var/lib/mysql/mysql-bin --server-id=1 --binlog-format=ROW --gtid_mode=ON --enforce-gtid-consistency=true + volumes: + - /tmp/mysql1-data:/var/lib/mysql + + mysql2: # upstream MySQL-2 + <<: *default-node + container_name: mysql2 + hostname: mysql2 + ports: + - "3307:3306" + networks: + db-networks: + ipv4_address: 172.28.128.5 + image: mysql:8.0.23 + command: --default-authentication-plugin=mysql_native_password --log-bin=/var/lib/mysql/mysql-bin --server-id=1 --binlog-format=ROW --gtid_mode=ON --enforce-gtid-consistency=true + volumes: + - /tmp/mysql2-data:/var/lib/mysql \ No newline at end of file diff --git a/tests/incremental_mode/data/db1.increment.sql b/tests/incremental_mode/data/db1.increment.sql index 13112bc38f..f5d1d0b4a5 100644 --- a/tests/incremental_mode/data/db1.increment.sql +++ b/tests/incremental_mode/data/db1.increment.sql @@ -2,6 +2,7 @@ use incremental_mode; insert into t1 (id, name) values (3, 'Eddard Stark'); update t1 set name = 'Arya Stark' where id = 1; update t1 set name = 'Catelyn Stark' where name = 'catelyn'; +flush logs; -- test multi column index with generated column alter table t1 add column info json; @@ -11,6 +12,7 @@ insert into t1 (id, name, info) values (4, 'gentest', '{"id": 123}'); insert into t1 (id, name, info) values (5, 'gentest', '{"id": 124}'); update t1 set info = '{"id": 120}' where id = 1; update t1 set info = '{"id": 121}' where id = 2; +flush logs; update t1 set info = '{"id": 122}' where id = 3; -- test genColumnCache is reset after ddl diff --git a/tests/incremental_mode/data/db2.increment.sql b/tests/incremental_mode/data/db2.increment.sql index 457960d31f..be1cdad5b6 100644 --- a/tests/incremental_mode/data/db2.increment.sql +++ b/tests/incremental_mode/data/db2.increment.sql @@ -1,2 +1,4 @@ use incremental_mode; +flush logs; +flush logs; delete from t2 where name = 'Sansa';