Merge branch 'master' into history_hot_regions_command

.github/CODEOWNERS

@@ -0,0 +1,5 @@
+# Require review from domain experts when the PR modified significant config files.
+/server/config/config.go @tikv/pd-configuration-reviewer
+/server/schedulers/hot_region_config.go @tikv/pd-configuration-reviewer
+/conf/config.toml @tikv/pd-configuration-reviewer
+/metrics/grafana/pd.json @tikv/pd-configuration-reviewer

.github/ISSUE_TEMPLATE/feature-request.md

@@ -1,7 +1,7 @@
 ---
 name: "🚀 Feature Request"
 about: I have a suggestion
-labels: type/enhancement
+labels: type/feature-request
 ---
 
 ## Feature Request

.github/workflows/check.yaml

@@ -17,9 +17,9 @@ jobs:
             ~/.cache/go-build
             **/.tools
             **/.dashboard_asset_cache
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          key: ${{ runner.os }}-golang-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            ${{ runner.os }}-go-
+            ${{ runner.os }}-golang
       - name: Make Check
         run: |
           make build

.github/workflows/pd-tests.yaml

@@ -1,3 +1,4 @@
+name: PD Tests
 on:
   push:
     branches:
@@ -9,9 +10,6 @@ on:
       - master
       - release-4.0
       - release-5.*
-
-name: PD Tests
-
 jobs:
   chunks:
     runs-on: ubuntu-latest
@@ -43,14 +41,16 @@ jobs:
         id: packages-units
         env:
           WORKER_ID: ${{ matrix.worker_id }}
+        # github.com/tikv/pd/tests/server/tso is the integration test of TSO, which will take a long time,
+        # will be run independently in the TSO Function Test.
         run: |
-          go list ./... |grep -v -E  "github.com/tikv/pd/tests/client|github.com/tikv/pd/server/api" > packages.list
+          go list ./... | grep -v -E  "github.com/tikv/pd/server/api|github.com/tikv/pd/tests/client|github.com/tikv/pd/tests/server/tso" > packages.list
           total=$(expr ${{ strategy.job-total }} - 1)
           echo  "Dispatched ${total} normal chunks"
           split packages.list -n r/${total} packages_unit_ -a 1 --numeric-suffixes=1
-          echo  "Dispatched 2 special task to the last chunk(the special test takes a long time)"
-          echo github.com/tikv/pd/tests/client > packages_unit_${{ strategy.job-total }}
-          echo github.com/tikv/pd/server/api >> packages_unit_${{ strategy.job-total }}
+          echo  "Dispatched 2 special task to the last chunk (the special tests take a long time)"
+          echo github.com/tikv/pd/server/api > packages_unit_${{ strategy.job-total }}
+          echo github.com/tikv/pd/tests/client >> packages_unit_${{ strategy.job-total }}
           packages="{$(cat packages_unit_${WORKER_ID} |tr "\n" ",")}"
           echo "This worker will test the chunk - ${packages}"
           echo "::set-output name=list::${packages}"

.github/workflows/tso-consistency-test.yaml

@@ -0,0 +1,17 @@
+name: TSO Consistency Test
+on:
+  # Only run when the new code is merged into master.
+  push:
+    branches:
+      - master
+jobs:
+  tso-consistency-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v2
+        with:
+          go-version: 1.16
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Make TSO Consistency Test
+        run: make test-tso-consistency

.github/workflows/tso-function-test.yaml

@@ -0,0 +1,19 @@
+name: TSO Function Test
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+jobs:
+  tso-function-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v2
+        with:
+          go-version: 1.16
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Make TSO Function Test
+        run: make test-tso-function

.golangci.yml

@@ -4,5 +4,30 @@ linters:
   enable:
     - misspell
     - stylecheck
+    - gocritic
+    - unparam
+    - unconvert
+    - whitespace
+    - makezero
+    - gosec
   disable:
     - errcheck
+linters-settings:
+  gocritic:
+    # Which checks should be enabled; can't be combined with 'disabled-checks';
+    # See https://go-critic.github.io/overview#checks-overview
+    # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run`
+    # By default list of stable checks is used.
+    enabled-checks:
+    # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty
+    disabled-checks:
+      - regexpMust
+      - appendAssign
+      - exitAfterDefer
+      - ifElseChain
+  gosec:
+    # To specify a set of rules to explicitly exclude.
+    # Available rules: https://github.com/securego/gosec#available-rules
+    excludes:
+      - G402
+      - G404

Makefile

@@ -4,6 +4,8 @@ TEST_PKGS := $(shell find . -iname "*_test.go" -exec dirname {} \; | \
                      sort -u | sed -e "s/^\./github.com\/tikv\/pd/")
 INTEGRATION_TEST_PKGS := $(shell find . -iname "*_test.go" -exec dirname {} \; | \
                      sort -u | sed -e "s/^\./github.com\/tikv\/pd/" | grep -E "tests")
+TSO_INTEGRATION_TEST_PKGS := $(shell find . -iname "*_test.go" -exec dirname {} \; | \
+                     sort -u | sed -e "s/^\./github.com\/tikv\/pd/" | grep -E "server/tso")
 BASIC_TEST_PKGS := $(filter-out $(INTEGRATION_TEST_PKGS),$(TEST_PKGS))
 
 PACKAGES := go list ./...
@@ -97,6 +99,10 @@ PD_SERVER_DEP :=
 ifneq ($(SWAGGER), 0)
 	PD_SERVER_DEP += swagger-spec
 endif
+ifneq ($(DASHBOARD_DISTRIBUTION_DIR),)
+	BUILD_TAGS += dashboard_distro
+	PD_SERVER_DEP += dashboard-replace-distro-info
+endif
 PD_SERVER_DEP += dashboard-ui
 
 pd-server: export GO111MODULE=on
@@ -111,7 +117,7 @@ pd-server-basic:
 install-go-tools: export GO111MODULE=on
 install-go-tools:
 	@mkdir -p $(GO_TOOLS_BIN_PATH)
-	@which golangci-lint >/dev/null 2>&1 || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GO_TOOLS_BIN_PATH) v1.38.0
+	@which golangci-lint >/dev/null 2>&1 || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GO_TOOLS_BIN_PATH) v1.43.0
 	@grep '_' tools.go | sed 's/"//g' | awk '{print $$2}' | xargs go install
 
 swagger-spec: export GO111MODULE=on
@@ -125,6 +131,10 @@ dashboard-ui: export GO111MODULE=on
 dashboard-ui:
 	./scripts/embed-dashboard-ui.sh
 
+dashboard-replace-distro-info:
+	rm -f pkg/dashboard/distro/distro_info.go
+	cp $(DASHBOARD_DISTRIBUTION_DIR)/distro_info.go pkg/dashboard/distro/distro_info.go
+
 # Tools
 pd-ctl: export GO111MODULE=on
 pd-ctl:
@@ -143,20 +153,21 @@ pd-heartbeat-bench:
 	CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-heartbeat-bench tools/pd-heartbeat-bench/main.go
 
 test: install-go-tools
-	# testing...
+	# testing all pkgs...
 	@$(DEADLOCK_ENABLE)
 	@$(FAILPOINT_ENABLE)
-	CGO_ENABLED=1 GO111MODULE=on go test -race -cover $(TEST_PKGS) || { $(FAILPOINT_DISABLE); $(DEADLOCK_DISABLE); exit 1; }
+	CGO_ENABLED=1 GO111MODULE=on go test -tags tso_function_test -timeout 20m -race -cover $(TEST_PKGS) || { $(FAILPOINT_DISABLE); $(DEADLOCK_DISABLE); exit 1; }
 	@$(FAILPOINT_DISABLE)
 	@$(DEADLOCK_DISABLE)
 
 basic-test:
+	# testing basic pkgs...
 	@$(FAILPOINT_ENABLE)
 	GO111MODULE=on go test $(BASIC_TEST_PKGS) || { $(FAILPOINT_DISABLE); exit 1; }
 	@$(FAILPOINT_DISABLE)
 
 test-with-cover: install-go-tools dashboard-ui
-	# testing...
+	# testing all pkgs (expect TSO consistency test) with converage...
 	@$(FAILPOINT_ENABLE)
 	for PKG in $(TEST_PKGS); do\
 		set -euo pipefail;\
@@ -165,6 +176,22 @@ test-with-cover: install-go-tools dashboard-ui
 	done
 	@$(FAILPOINT_DISABLE)
 
+test-tso-function: install-go-tools dashboard-ui
+	# testing TSO function...
+	@$(DEADLOCK_ENABLE)
+	@$(FAILPOINT_ENABLE)
+	CGO_ENABLED=1 GO111MODULE=on go test -race -tags tso_function_test $(TSO_INTEGRATION_TEST_PKGS) || { $(FAILPOINT_DISABLE); $(DEADLOCK_DISABLE); exit 1; }
+	@$(FAILPOINT_DISABLE)
+	@$(DEADLOCK_DISABLE)
+
+test-tso-consistency: install-go-tools dashboard-ui
+	# testing TSO consistency...
+	@$(DEADLOCK_ENABLE)
+	@$(FAILPOINT_ENABLE)
+	CGO_ENABLED=1 GO111MODULE=on go test -race -tags tso_consistency_test $(TSO_INTEGRATION_TEST_PKGS) || { $(FAILPOINT_DISABLE); $(DEADLOCK_DISABLE); exit 1; }
+	@$(FAILPOINT_DISABLE)
+	@$(DEADLOCK_DISABLE)
+
 check: install-go-tools check-all check-plugin errdoc check-testing-t docker-build-test
 
 check-all: static lint tidy
@@ -220,6 +247,7 @@ clean-test:
 	rm -rf /tmp/test_pd*
 	rm -rf /tmp/pd-tests*
 	rm -rf /tmp/test_etcd*
+	go clean -testcache
 
 clean-build:
 	# Cleaning building files...

README.md

@@ -120,9 +120,4 @@ docker run -d -p 2379:2379 -p 2380:2380 --name pd tikv/pd \
 
 ### Cluster
 
-PD is a component in TiDB project, you must run it with TiDB and TiKV together, see
-[TiDB-Ansible](https://pingcap.com/docs/dev/how-to/deploy/orchestrated/ansible/#deploy-tidb-using-ansible)
-to learn how to set up the cluster and run them.
-
-You can also use [Docker](https://pingcap.com/docs/dev/how-to/deploy/orchestrated/docker/#deploy-tidb-using-docker)
-to run the cluster.
+As a component of TiKV project, PD needs to run with TiKV to work. The cluster can also include TiDB to provide SQL services. You can refer [Deploy a TiDB Cluster Using TiUP](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) or [TiDB in Kubernetes Documentation](https://docs.pingcap.com/tidb-in-kubernetes/stable) for detailed instructions to deploy a cluster.