Merge branch 'master' into fetch-table

.github/workflows/binlog-999999.yml

@@ -14,10 +14,10 @@ jobs:
     runs-on: ubuntu-18.04
 
     steps:
-      - name: Set up Go 1.13
+      - name: Set up Go 1.16
         uses: actions/setup-go@v2
         with:
-          go-version: 1.13
+          go-version: 1.16
 
       - name: Check out code
         uses: actions/checkout@v2

.github/workflows/chaos-mesh.yml

@@ -40,11 +40,11 @@ jobs:
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
-    # Set up Go for building DM, now it's v1.13
-    - name: Set up Go 1.13
+    # Set up Go for building DM, now it's v1.16
+    - name: Set up Go 1.16
       uses: actions/setup-go@v2
       with:
-        go-version: 1.13
+        go-version: 1.16
     - name: Print Go version
       run: go version
 

.github/workflows/upgrade-via-tiup.yml

@@ -12,7 +12,7 @@ on:
       fromVer:
         dmVer: 'v1.0.6'
       toVer:
-        dmVer: 'nightly'  
+        dmVer: 'nightly'
 
 jobs:
   from_v1:
@@ -22,16 +22,15 @@ jobs:
       working-directory: ${{ github.workspace }}/go/src/github.com/pingcap/dm
 
     steps:
-    - name: Set up Go 1.13
+    - name: Set up Go 1.16
       uses: actions/setup-go@v2
       with:
-        go-version: 1.13
-
+        go-version: 1.16
     - name: Check out code
       uses: actions/checkout@v2
       with:
         path: go/src/github.com/pingcap/dm
-    
+
     - name: Setup containers
       working-directory: ${{ env.working-directory }}
       run: |
@@ -69,13 +68,12 @@ jobs:
         matrix:
           previous_v2: ["v2.0.0", "v2.0.1", "v2.0.3"]
 
-    steps:    
+    steps:
 
-    - name: Set up Go 1.13
+    - name: Set up Go 1.16
       uses: actions/setup-go@v2
       with:
-        go-version: 1.13
-
+        go-version: 1.16
     - name: Check out code
       uses: actions/checkout@v2
       with:
@@ -85,7 +83,7 @@ jobs:
       if: ${{ github.ref != 'refs/heads/master' }}
       working-directory: ${{ env.working-directory }}
       run: make build nolint=true
-    
+
     - name: Package files
       if: ${{ github.ref != 'refs/heads/master' }}
       run: |
@@ -176,10 +174,10 @@ jobs:
       working-directory: ${{ github.workspace }}/go/src/github.com/pingcap/dm
 
     steps:
-    - name: Set up Go 1.13
+    - name: Set up Go 1.16
       uses: actions/setup-go@v2
       with:
-        go-version: 1.13
+        go-version: 1.16
 
     - name: Check out code
       uses: actions/checkout@v2

.github/workflows/upstream-switch.yml

@@ -14,10 +14,10 @@ jobs:
     runs-on: ubuntu-18.04
 
     steps:
-    - name: Set up Go 1.13
+    - name: Set up Go 1.16
       uses: actions/setup-go@v2
       with:
-        go-version: 1.13
+        go-version: 1.16
 
     - name: Check out code
       uses: actions/checkout@v2

CONTRIBUTING.md

@@ -16,7 +16,7 @@ See [Step by Step Contribute Example](step_by_step_contribute.md) for a more det
 
 Developing DM requires:
 
-* [Go 1.13+](http://golang.org/doc/code.html)
+* [Go 1.16+](http://golang.org/doc/code.html)
 * An internet connection to download the dependencies
 
 Simply run `make` to build the program.

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13-alpine as builder
+FROM golang:1.16-alpine as builder
 MAINTAINER siddontang
 
 RUN apk add --no-cache \

README.md

@@ -83,4 +83,8 @@ DM is under the Apache 2.0 license. See the [LICENSE](./LICENSE) file for detail
     - [English](https://pingcap.com/blog/)
     - [简体中文](https://pingcap.com/blog-cn/)
 
-- [TiDB Monthly](https://pingcap.com/weekly/)
+- TiDB case studies
+
+    - [English](https://pingcap.com/case-studies/)
+    - [简体中文](https://pingcap.com/cases-cn/)
+

_utils/terror_gen/errors_release.txt

@@ -319,6 +319,7 @@ ErrSyncerOperatorNotExist,[code=36065:class=sync-unit:scope=internal:level=low],
 ErrSyncerReplaceEventNotExist,[code=36066:class=sync-unit:scope=internal:level=high], "Message: replace event not exist, location: %s"
 ErrSyncerParseDDL,[code=36067:class=sync-unit:scope=internal:level=high], "Message: parse DDL: %s, Workaround: Please confirm your DDL statement is correct and needed. For TiDB compatible DDL, see https://docs.pingcap.com/tidb/stable/mysql-compatibility#ddl. You can use `handle-error` command to skip or replace the DDL or add a binlog filter rule to ignore it if the DDL is not needed."
 ErrSyncerUnsupportedStmt,[code=36068:class=sync-unit:scope=internal:level=high], "Message: `%s` statement not supported in %s mode"
+ErrSyncerGetEvent,[code=36069:class=sync-unit:scope=upstream:level=high], "Message: get binlog event error: %v, Workaround: Please check if the binlog file could be parsed by `mysqlbinlog`."
 ErrMasterSQLOpNilRequest,[code=38001:class=dm-master:scope=internal:level=medium], "Message: nil request not valid"
 ErrMasterSQLOpNotSupport,[code=38002:class=dm-master:scope=internal:level=medium], "Message: op %s not supported"
 ErrMasterSQLOpWithoutSharding,[code=38003:class=dm-master:scope=internal:level=medium], "Message: operate request without --sharding specified not valid"

dm/ctl/common/config.go

@@ -18,6 +18,7 @@ import (
 	"flag"
 	"fmt"
 	"net"
+	"os"
 	"strings"
 	"time"
 
@@ -149,8 +150,12 @@ func (c *Config) Parse(arguments []string) (finish bool, err error) {
 		return false, errors.Errorf("'%s' is an invalid flag", c.FlagSet.Arg(0))
 	}
 
+	// try get master Addr from env "DM_MASTER_ADDR" if this flag is empty.
 	if c.MasterAddr == "" {
-		return false, errors.Errorf("--master-addr not provided, use `dmtcl --help` to see help messages")
+		c.MasterAddr = os.Getenv("DM_MASTER_ADDR")
+	}
+	if c.MasterAddr == "" {
+		return false, errors.Errorf("--master-addr not provided, this parameter is required when interacting with the dm-master, you can also use environment variable 'DM_MASTER_ADDR' to specify the value. Use `dmtcl --help` to see more help messages")
 	}
 
 	return false, errors.Trace(c.adjust())

dm/portal/frontend/yarn.lock

@@ -3561,9 +3561,9 @@ dns-equal@^1.0.0:
   integrity sha1-s55/HabrCnW6nBcySzR1PEfgZU0=
 
 dns-packet@^1.3.1:
-  version "1.3.1"
-  resolved "https://registry.yarnpkg.com/dns-packet/-/dns-packet-1.3.1.tgz#12aa426981075be500b910eedcd0b47dd7deda5a"
-  integrity sha512-0UxfQkMhYAUaZI+xrNZOz/as5KgDU0M/fQ9b6SpkyLbk3GEswDi6PADJVaYJradtRVsRIlF1zLyOodbcTCDzUg==
+  version "1.3.4"
+  resolved "https://registry.yarnpkg.com/dns-packet/-/dns-packet-1.3.4.tgz#e3455065824a2507ba886c55a89963bb107dec6f"
+  integrity sha512-BQ6F4vycLXBvdrJZ6S3gZewt6rcrks9KBgM9vrhW+knGRqc8uEdT7fuCwloc7nny5xNoMJ17HGH0R/6fpo8ECA==
   dependencies:
     ip "^1.1.0"
     safe-buffer "^5.0.1"
@@ -9469,11 +9469,16 @@ rxjs@^6.4.0:
   dependencies:
     tslib "^1.9.0"
 
-safe-buffer@5.1.2, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
+safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
   integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
 
+safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2:
+  version "5.2.1"
+  resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
+  integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
+
 safe-regex@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/safe-regex/-/safe-regex-1.1.0.tgz#40a3669f3b077d1e943d44629e157dd48023bf2e"

errors.toml

@@ -1924,6 +1924,12 @@ description = ""
 workaround = ""
 tags = ["internal", "high"]
 
+[error.DM-sync-unit-36069]
+message = "get binlog event error: %v"
+description = ""
+workaround = "Please check if the binlog file could be parsed by `mysqlbinlog`."
+tags = ["upstream", "high"]
+
 [error.DM-dm-master-38001]
 message = "nil request not valid"
 description = ""

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/coreos/go-semver v0.3.0
 	github.com/docker/go-units v0.4.0
 	github.com/dustin/go-humanize v1.0.0
-	github.com/go-mysql-org/go-mysql v1.1.3-0.20210420093540-a51d9dec4f96
+	github.com/go-mysql-org/go-mysql v1.1.3-0.20210519094347-5ec8d511f5be
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/gogo/gateway v1.1.0
 	github.com/gogo/protobuf v1.3.2
@@ -47,4 +47,4 @@ require (
 	gopkg.in/yaml.v2 v2.4.0
 )
 
-go 1.13
+go 1.16

go.sum

@@ -165,7 +165,6 @@ github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMe
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
-github.com/codahale/hdrhistogram v0.9.0 h1:9GjrtRI+mLEFPtTfR/AZhcxp+Ii8NZYWq5104FbZQY0=
 github.com/codahale/hdrhistogram v0.9.0/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c=
 github.com/coocood/badger v1.5.1-0.20200515070411-e02af0688441/go.mod h1:klY8SfH2lNZ/23/SIxwHoJw+T6wYGB12YPCF9MUoiu0=
@@ -271,7 +270,6 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu
 github.com/frankban/quicktest v1.4.1/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ=
 github.com/frankban/quicktest v1.11.1 h1:stwUsXhUGliQs9t0ZS39BWCltFdOHgABiIlihop8AD4=
 github.com/frankban/quicktest v1.11.1/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsouza/fake-gcs-server v1.15.0/go.mod h1:HNxAJ/+FY/XSsxuwz8iIYdp2GtMmPbJ8WQjjGMxd6Qk=
 github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw=
@@ -303,8 +301,8 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-mysql-org/go-mysql v1.1.3-0.20210420093540-a51d9dec4f96 h1:+0HKUBVZNwqh+k20hbdiFoZQ7neXijUm54G/B2cQVgM=
-github.com/go-mysql-org/go-mysql v1.1.3-0.20210420093540-a51d9dec4f96/go.mod h1:3lFZKf7l95Qo70+3XB2WpiSf9wu2s3na3geLMaIIrqQ=
+github.com/go-mysql-org/go-mysql v1.1.3-0.20210519094347-5ec8d511f5be h1:12z4p/lldNvXStdf2qPICQ3eyTC0B0kw32X8ysx40Yk=
+github.com/go-mysql-org/go-mysql v1.1.3-0.20210519094347-5ec8d511f5be/go.mod h1:3lFZKf7l95Qo70+3XB2WpiSf9wu2s3na3geLMaIIrqQ=
 github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
 github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
@@ -1094,7 +1092,6 @@ github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx
 github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457 h1:tBbuFCtyJNKT+BFAv6qjvTFpVdy97IYNaBwGUXifIUs=
 github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457/go.mod h1:pheqtXeHQFzxJk45lRQ0UIGIivKnLXvialZSFWs81A8=
 github.com/xitongsys/parquet-go-source v0.0.0-20190524061010-2b72cbee77d5/go.mod h1:xxCx7Wpym/3QCo6JhujJX51dzSXrwmb0oH6FQb39SEA=
-github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 h1:a742S4V5A15F93smuVxA60LQWsrCnN8bKeWDBARU1/k=
 github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE=
 github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xo/dburl v0.0.0-20191219122722-3cca8608d645/go.mod h1:A47W3pdWONaZmXuLZgfKLAVgUY0qvfTRM5vVDKS40S4=

pkg/terror/error_list.go

@@ -412,6 +412,7 @@ const (
 	codeSyncerReplaceEventNotExist
 	codeSyncerParseDDL
 	codeSyncerUnsupportedStmt
+	codeSyncerGetEvent
 )
 
 // DM-master error code.
@@ -1005,6 +1006,7 @@ var (
 	ErrSyncerReplaceEventNotExist           = New(codeSyncerReplaceEventNotExist, ClassSyncUnit, ScopeInternal, LevelHigh, "replace event not exist, location: %s", "")
 	ErrSyncerParseDDL                       = New(codeSyncerParseDDL, ClassSyncUnit, ScopeInternal, LevelHigh, "parse DDL: %s", "Please confirm your DDL statement is correct and needed. For TiDB compatible DDL, see https://docs.pingcap.com/tidb/stable/mysql-compatibility#ddl. You can use `handle-error` command to skip or replace the DDL or add a binlog filter rule to ignore it if the DDL is not needed.")
 	ErrSyncerUnsupportedStmt                = New(codeSyncerUnsupportedStmt, ClassSyncUnit, ScopeInternal, LevelHigh, "`%s` statement not supported in %s mode", "")
+	ErrSyncerGetEvent                       = New(codeSyncerGetEvent, ClassSyncUnit, ScopeUpstream, LevelHigh, "get binlog event error: %v", "Please check if the binlog file could be parsed by `mysqlbinlog`.")
 
 	// DM-master error.
 	ErrMasterSQLOpNilRequest        = New(codeMasterSQLOpNilRequest, ClassDMMaster, ScopeInternal, LevelMedium, "nil request not valid", "")

step_by_step_contribute.md

@@ -30,7 +30,7 @@ git remote -v # Confirms that your remote makes sense
 
     > **Note:**
     >
-    > DM uses [`Go Modules`](https://github.com/golang/go/wiki/Modules) to manage dependencies. The version of Go should be **1.13** or above.
+    > DM uses [`Go Modules`](https://github.com/golang/go/wiki/Modules) to manage dependencies. The version of Go should be **1.16** or above.
 
     You'll need `GOPATH` defined, and `PATH` modified to access your Go binaries. A
     common setup is the following but you could always google a setup for your own

syncer/streamer_controller.go

@@ -15,6 +15,7 @@ package syncer
 
 import (
 	"context"
+	"errors"
 	"strings"
 	"sync"
 	"time"
@@ -249,6 +250,9 @@ func (c *StreamerController) GetEvent(tctx *tcontext.Context) (event *replicatio
 
 	event, err = streamer.GetEvent(ctx)
 	cancel()
+	failpoint.Inject("GetEventError", func() {
+		err = errors.New("go-mysql returned an error")
+	})
 	if err != nil {
 		if err != context.Canceled && err != context.DeadlineExceeded {
 			c.Lock()

syncer/syncer.go

@@ -1313,6 +1313,7 @@ func (s *Syncer) Run(ctx context.Context) (err error) {
 		}
 
 		if err != nil {
+			// TODO: wrap the error with terror, and attach binlog position
 			tctx.L().Error("fail to fetch binlog", log.ShortError(err))
 
 			if isConnectionRefusedError(err) {
@@ -1338,7 +1339,7 @@ func (s *Syncer) Run(ctx context.Context) (err error) {
 				continue
 			}
 
-			return err
+			return terror.ErrSyncerGetEvent.Generate(err)
 		}
 
 		// time duration for reading an event from relay log or upstream master.

tests/dmctl_command/run.sh

@@ -86,6 +86,12 @@ function run() {
 	# query status with command mode
 	$PWD/bin/dmctl.test DEVEL --master-addr=:$MASTER_PORT query-status >$WORK_DIR/query-status.log
 
+	# use DM_MASTER_ADDR env
+	export DM_MASTER_ADDR="127.0.0.1:$MASTER_PORT"
+	run_dm_ctl_with_retry $WORK_DIR "" \
+		"query-status test" \
+		"worker1" 1
+
 	running_task=$(grep -r Running $WORK_DIR/query-status.log | wc -l | xargs)
 
 	if [ "$running_task" != 1 ]; then