Merge remote-tracking branch 'upstream/master' into sink-manager-mana…

…ge-checkpoint-per-table

# Conflicts:
#	cdc/processor/processor_test.go

.codecov.yml

@@ -24,7 +24,7 @@ parsers:
       macro: no
 
 comment:
-  layout: "header, diff"
+  layout: "header, flags, diff"
   behavior: default
   require_changes: no
 
@@ -38,3 +38,16 @@ ignore:
   - "*.md"
   - "docs/.*"
   - "testing_utils/.*"
+
+flags:
+  cdc:
+    carryforward: true
+    paths:
+      - "cdc/"
+      - "cmd/"
+      - "pkg/"
+
+  dm:
+    carryforward: true
+    paths:
+      - "dm/"

.github/workflows/check_and_build.yaml

@@ -4,77 +4,56 @@ on:
   push:
     branches:
       - master
-      - 'release-[0-9].[0-9]*'
+      - "release-[0-9].[0-9]*"
   pull_request:
     branches:
       - master
-      - 'release-[0-9].[0-9]*'
+      - "release-[0-9].[0-9]*"
 
 # See: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency.
 concurrency:
   group: ${{ github.ref }}-${{ github.workflow }}
   cancel-in-progress: true
 
 jobs:
-
   docker_build:
     name: Docker Build
     runs-on: ubuntu-latest
     steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      - name: Build-cdc
+        run: docker build -f ./deployments/ticdc/docker/Dockerfile .
 
-    - name: Build
-      run: docker build -f ./deployments/ticdc/docker/Dockerfile .
+      - name: Build-dm
+        run: docker build -f ./dm/Dockerfile .
 
-  make_check_build_mac:
-    name: Make Check & Build On Mac OS
+  mac_build:
+    name: Mac OS Build
     runs-on: macos-latest
     steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
-
-    - name: Setup Go environment
-      uses: actions/setup-go@v2
-      with:
-        go-version: '1.16.4'
-
-    - name: Cache Tools
-      id: cache-tools
-      uses: actions/cache@v2
-      with:
-        path: tools/bin
-        key: macos-latest-ticdc-tools-${{ hashFiles('tools/check/go.sum') }}
-    - name: Check
-      run: make check
-
-    - name: Build
-      run: make build
-
-  make_check_build_linux:
-    name: Make Check & Build On Linux
-    runs-on: [x64]
-    steps:
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
-
-    - name: Setup Go environment
-      uses: actions/setup-go@v2
-      with:
-        go-version: '1.16.4'
+      - name: Setup Go environment
+        uses: actions/setup-go@v2
+        with:
+          go-version: "1.16.4"
 
-    - name: Check
-      run: make check
+      - name: Cache Tools
+        id: cache-tools
+        uses: actions/cache@v2
+        with:
+          path: tools/bin
+          key: macos-latest-ticdc-tools-${{ hashFiles('tools/check/go.sum') }}
 
-    - name: Build
-      run: make build
+      - name: Build
+        run: make build
 
   arm_build:
     runs-on: [ARM64]
-    name: Make Build
+    name: Arm Build
     strategy:
       fail-fast: false
       matrix:
@@ -86,7 +65,7 @@ jobs:
       - name: Setup Go environment
         uses: actions/setup-go@v2
         with:
-          go-version: '1.16.4'
+          go-version: "1.16.4"
 
       - name: Build
         run: make build

Makefile

@@ -129,6 +129,7 @@ unit_test_in_verify_ci: check_failpoint_ctl tools/bin/gotestsum tools/bin/gocov
 	|| { $(FAILPOINT_DISABLE); exit 1; }
 	tools/bin/gocov convert "$(TEST_DIR)/cov.unit.out" | tools/bin/gocov-xml > cdc-coverage.xml
 	$(FAILPOINT_DISABLE)
+	@bash <(curl -s https://codecov.io/bash) -F cdc -f $(TEST_DIR)/cov.unit.out -t $(TICDC_CODECOV_TOKEN)
 
 leak_test: check_failpoint_ctl
 	$(FAILPOINT_ENABLE)
@@ -222,12 +223,8 @@ endif
 
 unit_test_coverage:
 	grep -vE ".*.pb.go|$(CDC_PKG)/testing_utils/.*|$(CDC_PKG)/cdc/kv/testing.go|$(CDC_PKG)/cdc/sink/simple_mysql_tester.go|.*.__failpoint_binding__.go" "$(TEST_DIR)/cov.unit.out" > "$(TEST_DIR)/unit_cov.out"
-ifeq ("$(JenkinsCI)", "1")
-	@bash <(curl -s https://codecov.io/bash) -f $(TEST_DIR)/unit_cov.out -t $(CODECOV_TOKEN)
-else
 	go tool cover -html "$(TEST_DIR)/unit_cov.out" -o "$(TEST_DIR)/unit_cov.html"
 	go tool cover -func="$(TEST_DIR)/unit_cov.out"
-endif
 
 data-flow-diagram: docs/data-flow.dot
 	dot -Tsvg docs/data-flow.dot > docs/data-flow.svg
@@ -289,6 +286,7 @@ dm_unit_test_in_verify_ci: check_failpoint_ctl tools/bin/gotestsum tools/bin/goc
 	|| { $(FAILPOINT_DISABLE); exit 1; }
 	tools/bin/gocov convert "$(DM_TEST_DIR)/cov.unit_test.out" | tools/bin/gocov-xml > dm-coverage.xml
 	$(FAILPOINT_DISABLE)
+	@bash <(curl -s https://codecov.io/bash) -F dm -f $(DM_TEST_DIR)/cov.unit_test.out -t $(TICDC_CODECOV_TOKEN)
 
 dm_integration_test_build: check_failpoint_ctl
 	$(FAILPOINT_ENABLE)
@@ -358,13 +356,9 @@ dm_coverage: tools/bin/gocovmerge tools/bin/goveralls
 	find "$(DM_TEST_DIR)" -type f -name "cov.*.dmctl.*.out" -exec sed -i "s/mode: count/mode: atomic/g" {} \;
 	tools/bin/gocovmerge "$(DM_TEST_DIR)"/cov.* | grep -vE ".*.pb.go|.*.pb.gw.go|.*.__failpoint_binding__.go|.*debug-tools.*|.*portal.*|.*chaos.*" > "$(DM_TEST_DIR)/all_cov.out"
 	tools/bin/gocovmerge "$(DM_TEST_DIR)"/cov.unit_test*.out | grep -vE ".*.pb.go|.*.pb.gw.go|.*.__failpoint_binding__.go|.*debug-tools.*|.*portal.*|.*chaos.*" > $(DM_TEST_DIR)/unit_test.out
-ifeq ("$(JenkinsCI)", "1")
-	@bash <(curl -s https://codecov.io/bash) -f $(DM_TEST_DIR)/unit_test.out -t $(CODECOV_TOKEN)
-	tools/bin/goveralls -coverprofile=$(DM_TEST_DIR)/all_cov.out -service=jenkins-ci -repotoken $(COVERALLS_TOKEN)
-else
 	go tool cover -html "$(DM_TEST_DIR)/all_cov.out" -o "$(DM_TEST_DIR)/all_cov.html"
 	go tool cover -html "$(DM_TEST_DIR)/unit_test.out" -o "$(DM_TEST_DIR)/unit_test_cov.html"
-endif
+
 
 tools/bin/failpoint-ctl: tools/check/go.mod
 	cd tools/check && $(GO) build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl

cdc/model/owner.go

@@ -78,9 +78,9 @@ func (t AdminJobType) IsStopState() bool {
 // TaskPosition records the process information of a capture
 type TaskPosition struct {
 	// The maximum event CommitTs that has been synchronized. This is updated by corresponding processor.
-	CheckPointTs uint64 `json:"checkpoint-ts"`
+	CheckPointTs uint64 `json:"checkpoint-ts"` // Deprecated
 	// The event that satisfies CommitTs <= ResolvedTs can be synchronized. This is updated by corresponding processor.
-	ResolvedTs uint64 `json:"resolved-ts"`
+	ResolvedTs uint64 `json:"resolved-ts"` // Deprecated
 	// The count of events were synchronized. This is updated by corresponding processor.
 	Count uint64 `json:"count"`
 	// Error when error happens
@@ -231,11 +231,9 @@ func (i *TableReplicaInfo) Clone() *TableReplicaInfo {
 type TaskStatus struct {
 	// Table information list, containing tables that processor should process, updated by ownrer, processor is read only.
 	Tables       map[TableID]*TableReplicaInfo `json:"tables"`
-	Operation    map[TableID]*TableOperation   `json:"operation"`
+	Operation    map[TableID]*TableOperation   `json:"operation"` // Deprecated
 	AdminJobType AdminJobType                  `json:"admin-job-type"`
 	ModRevision  int64                         `json:"-"`
-	// true means Operation record has been changed
-	Dirty bool `json:"-"`
 }
 
 // String implements fmt.Stringer interface.