Merge branch 'master' into history_hot_region_config

.github/workflows/pd-tests.yaml

@@ -1,3 +1,4 @@
+name: PD Tests
 on:
   push:
     branches:
@@ -9,9 +10,6 @@ on:
       - master
       - release-4.0
       - release-5.*
-
-name: PD Tests
-
 jobs:
   chunks:
     runs-on: ubuntu-latest
@@ -43,14 +41,16 @@ jobs:
         id: packages-units
         env:
           WORKER_ID: ${{ matrix.worker_id }}
+        # github.com/tikv/pd/tests/server/tso is the integration test of TSO, which will take a long time,
+        # will be run independently in the TSO Function Test.
         run: |
-          go list ./... |grep -v -E  "github.com/tikv/pd/tests/client|github.com/tikv/pd/server/api" > packages.list
+          go list ./... | grep -v -E  "github.com/tikv/pd/server/api|github.com/tikv/pd/tests/client|github.com/tikv/pd/tests/server/tso" > packages.list
           total=$(expr ${{ strategy.job-total }} - 1)
           echo  "Dispatched ${total} normal chunks"
           split packages.list -n r/${total} packages_unit_ -a 1 --numeric-suffixes=1
-          echo  "Dispatched 2 special task to the last chunk(the special test takes a long time)"
-          echo github.com/tikv/pd/tests/client > packages_unit_${{ strategy.job-total }}
-          echo github.com/tikv/pd/server/api >> packages_unit_${{ strategy.job-total }}
+          echo  "Dispatched 2 special task to the last chunk (the special tests take a long time)"
+          echo github.com/tikv/pd/server/api > packages_unit_${{ strategy.job-total }}
+          echo github.com/tikv/pd/tests/client >> packages_unit_${{ strategy.job-total }}
           packages="{$(cat packages_unit_${WORKER_ID} |tr "\n" ",")}"
           echo "This worker will test the chunk - ${packages}"
           echo "::set-output name=list::${packages}"

.github/workflows/tso-consistency-test.yaml

@@ -0,0 +1,17 @@
+name: TSO Consistency Test
+on:
+  # Only run when the new code is merged into master.
+  push:
+    branches:
+      - master
+jobs:
+  tso-consistency-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v2
+        with:
+          go-version: 1.16
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Make TSO Consistency Test
+        run: make test-tso-consistency

.github/workflows/tso-function-test.yaml

@@ -0,0 +1,19 @@
+name: TSO Function Test
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+jobs:
+  tso-function-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v2
+        with:
+          go-version: 1.16
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Make TSO Function Test
+        run: make test-tso-function

Makefile

@@ -4,6 +4,8 @@ TEST_PKGS := $(shell find . -iname "*_test.go" -exec dirname {} \; | \
                      sort -u | sed -e "s/^\./github.com\/tikv\/pd/")
 INTEGRATION_TEST_PKGS := $(shell find . -iname "*_test.go" -exec dirname {} \; | \
                      sort -u | sed -e "s/^\./github.com\/tikv\/pd/" | grep -E "tests")
+TSO_INTEGRATION_TEST_PKGS := $(shell find . -iname "*_test.go" -exec dirname {} \; | \
+                     sort -u | sed -e "s/^\./github.com\/tikv\/pd/" | grep -E "server/tso")
 BASIC_TEST_PKGS := $(filter-out $(INTEGRATION_TEST_PKGS),$(TEST_PKGS))
 
 PACKAGES := go list ./...
@@ -97,6 +99,10 @@ PD_SERVER_DEP :=
 ifneq ($(SWAGGER), 0)
 	PD_SERVER_DEP += swagger-spec
 endif
+ifneq ($(DASHBOARD_DISTRIBUTION_DIR),)
+	BUILD_TAGS += dashboard_distro
+	PD_SERVER_DEP += dashboard-replace-distro-info
+endif
 PD_SERVER_DEP += dashboard-ui
 
 pd-server: export GO111MODULE=on
@@ -125,6 +131,10 @@ dashboard-ui: export GO111MODULE=on
 dashboard-ui:
 	./scripts/embed-dashboard-ui.sh
 
+dashboard-replace-distro-info:
+	rm -f pkg/dashboard/distro/distro_info.go
+	cp $(DASHBOARD_DISTRIBUTION_DIR)/distro_info.go pkg/dashboard/distro/distro_info.go
+
 # Tools
 pd-ctl: export GO111MODULE=on
 pd-ctl:
@@ -143,20 +153,21 @@ pd-heartbeat-bench:
 	CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-heartbeat-bench tools/pd-heartbeat-bench/main.go
 
 test: install-go-tools
-	# testing...
+	# testing all pkgs...
 	@$(DEADLOCK_ENABLE)
 	@$(FAILPOINT_ENABLE)
-	CGO_ENABLED=1 GO111MODULE=on go test -race -cover $(TEST_PKGS) || { $(FAILPOINT_DISABLE); $(DEADLOCK_DISABLE); exit 1; }
+	CGO_ENABLED=1 GO111MODULE=on go test -tags tso_function_test -timeout 20m -race -cover $(TEST_PKGS) || { $(FAILPOINT_DISABLE); $(DEADLOCK_DISABLE); exit 1; }
 	@$(FAILPOINT_DISABLE)
 	@$(DEADLOCK_DISABLE)
 
 basic-test:
+	# testing basic pkgs...
 	@$(FAILPOINT_ENABLE)
 	GO111MODULE=on go test $(BASIC_TEST_PKGS) || { $(FAILPOINT_DISABLE); exit 1; }
 	@$(FAILPOINT_DISABLE)
 
 test-with-cover: install-go-tools dashboard-ui
-	# testing...
+	# testing all pkgs (expect TSO consistency test) with converage...
 	@$(FAILPOINT_ENABLE)
 	for PKG in $(TEST_PKGS); do\
 		set -euo pipefail;\
@@ -165,6 +176,22 @@ test-with-cover: install-go-tools dashboard-ui
 	done
 	@$(FAILPOINT_DISABLE)
 
+test-tso-function: install-go-tools dashboard-ui
+	# testing TSO function...
+	@$(DEADLOCK_ENABLE)
+	@$(FAILPOINT_ENABLE)
+	CGO_ENABLED=1 GO111MODULE=on go test -race -tags tso_function_test $(TSO_INTEGRATION_TEST_PKGS) || { $(FAILPOINT_DISABLE); $(DEADLOCK_DISABLE); exit 1; }
+	@$(FAILPOINT_DISABLE)
+	@$(DEADLOCK_DISABLE)
+
+test-tso-consistency: install-go-tools dashboard-ui
+	# testing TSO consistency...
+	@$(DEADLOCK_ENABLE)
+	@$(FAILPOINT_ENABLE)
+	CGO_ENABLED=1 GO111MODULE=on go test -race -tags tso_consistency_test $(TSO_INTEGRATION_TEST_PKGS) || { $(FAILPOINT_DISABLE); $(DEADLOCK_DISABLE); exit 1; }
+	@$(FAILPOINT_DISABLE)
+	@$(DEADLOCK_DISABLE)
+
 check: install-go-tools check-all check-plugin errdoc check-testing-t docker-build-test
 
 check-all: static lint tidy
@@ -220,6 +247,7 @@ clean-test:
 	rm -rf /tmp/test_pd*
 	rm -rf /tmp/pd-tests*
 	rm -rf /tmp/test_etcd*
+	go clean -testcache
 
 clean-build:
 	# Cleaning building files...

client/base_client.go

@@ -8,6 +8,7 @@
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 

client/client.go

@@ -8,6 +8,7 @@
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 

client/client_test.go

@@ -8,6 +8,7 @@
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 

client/metrics.go

@@ -8,6 +8,7 @@
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 

cmd/pd-server/main.go

@@ -8,6 +8,7 @@
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 

errors.toml

@@ -91,6 +91,11 @@ error = '''
 store %v is paused for leader transfer
 '''
 
+["PD:core:ErrSlowStoreEvicted"]
+error = '''
+store %v is evited as a slow store
+'''
+
 ["PD:core:ErrStoreDestroyed"]
 error = '''
 store %v has been physically destroyed
@@ -471,6 +476,16 @@ error = '''
 failed to unmarshal proto
 '''
 
+["PD:region:ErrRegionRuleContent"]
+error = '''
+invalid region rule content, %s
+'''
+
+["PD:region:ErrRegionRuleNotFound"]
+error = '''
+region label rule not found for id %s
+'''
+
 ["PD:schedule:ErrCreateOperator"]
 error = '''
 unable to create operator, %s
@@ -596,11 +611,6 @@ error = '''
 get local allocator failed, %s
 '''
 
-["PD:tso:ErrInvalidTimestamp"]
-error = '''
-invalid timestamp
-'''
-
 ["PD:tso:ErrLogicOverflow"]
 error = '''
 logic part overflow

go.mod

@@ -28,10 +28,10 @@ require (
 	github.com/pingcap/errcode v0.3.0
 	github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3
 	github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce
-	github.com/pingcap/kvproto v0.0.0-20210604082642-dda0a102bc6a
-	github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4
-	github.com/pingcap/sysutil v0.0.0-20210315073920-cc0985d983a3
-	github.com/pingcap/tidb-dashboard v0.0.0-20210716172320-2226872e3296
+	github.com/pingcap/kvproto v0.0.0-20210819164333-bd5706b9d9f2
+	github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7
+	github.com/pingcap/sysutil v0.0.0-20210730114356-fcd8a63f68c5
+	github.com/pingcap/tidb-dashboard v0.0.0-20210826074103-29034af68525
 	github.com/prometheus/client_golang v1.1.0
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.6.0
@@ -50,6 +50,7 @@ require (
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738
 	go.uber.org/goleak v1.1.10
 	go.uber.org/zap v1.16.0
+	golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e // indirect
 	golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064
 	google.golang.org/grpc v1.26.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0

go.sum

@@ -317,16 +317,18 @@ github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce h1:Y1kCxlCtlPTMt
 github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk=
 github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
 github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
-github.com/pingcap/kvproto v0.0.0-20210604082642-dda0a102bc6a h1:1vLA91bNdTnIDd3dwZ5VhwjVB5TLHzS23a47sNoYUlw=
-github.com/pingcap/kvproto v0.0.0-20210604082642-dda0a102bc6a/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
+github.com/pingcap/kvproto v0.0.0-20210819164333-bd5706b9d9f2 h1:iCXDtZrssbW8mX8WoMQ5TapBOdoV+GF2wGR+QRdcyCs=
+github.com/pingcap/kvproto v0.0.0-20210819164333-bd5706b9d9f2/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
 github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
-github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 h1:ERrF0fTuIOnwfGbt71Ji3DKbOEaP189tjym50u8gpC8=
 github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
-github.com/pingcap/sysutil v0.0.0-20210315073920-cc0985d983a3 h1:A9KL9R+lWSVPH8IqUuH1QSTRJ5FGoY1bT2IcfPKsWD8=
+github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7 h1:k2BbABz9+TNpYRwsCCFS8pEEnFVOdbgEjL/kTlLuzZQ=
+github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM=
 github.com/pingcap/sysutil v0.0.0-20210315073920-cc0985d983a3/go.mod h1:tckvA041UWP+NqYzrJ3fMgC/Hw9wnmQ/tUkp/JaHly8=
-github.com/pingcap/tidb-dashboard v0.0.0-20210716172320-2226872e3296 h1:kTH6Jyn8XVoFJNxT3UF4eiZMxDbyfsSXkAtSk9jLGr4=
-github.com/pingcap/tidb-dashboard v0.0.0-20210716172320-2226872e3296/go.mod h1:OCXbZTBTIMRcIt0jFsuCakZP+goYRv6IjawKbwLS2TQ=
+github.com/pingcap/sysutil v0.0.0-20210730114356-fcd8a63f68c5 h1:7rvAtZe/ZUzOKzgriNPQoBNvleJXBk4z7L3Z47+tS98=
+github.com/pingcap/sysutil v0.0.0-20210730114356-fcd8a63f68c5/go.mod h1:XsOaV712rUk63aOEKYP9PhXTIE3FMNHmC2r1wX5wElY=
+github.com/pingcap/tidb-dashboard v0.0.0-20210826074103-29034af68525 h1:hJDAzcVGfVsUx90/JHXPgvHggxsiuYZJ6FKkLY6Vo3M=
+github.com/pingcap/tidb-dashboard v0.0.0-20210826074103-29034af68525/go.mod h1:OCXbZTBTIMRcIt0jFsuCakZP+goYRv6IjawKbwLS2TQ=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -561,8 +563,9 @@ golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 h1:Bli41pIlzTzf3KEY06n+xnzK/BESIg2ze4Pgfh/aI8c=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e h1:XMgFehsDnnLGtjvjOfqWSUzt0alpTR1RSEuznObga2c=
+golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

metrics/alertmanager/pd.rules.yml

@@ -2,12 +2,12 @@ groups:
 - name: alert.rules
   rules:
   - alert: PD_cluster_offline_tikv_nums
-    expr: (sum ( pd_cluster_status{type="store_down_count"} ) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)
+    expr: (sum(pd_cluster_status{type="store_down_count"}) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)
     for: 1m
     labels:
       env: ENV_LABELS_ENV
       level: emergency
-      expr:  (sum ( pd_cluster_status{type="store_down_count"} ) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)
+      expr:  (sum(pd_cluster_status{type="store_down_count"}) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)
     annotations:
       description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
       value: '{{ $value }}'
@@ -38,24 +38,24 @@ groups:
       summary: PD_miss_peer_region_count
 
   - alert: PD_cluster_lost_connect_tikv_nums
-    expr: (sum ( pd_cluster_status{type="store_disconnected_count"} ) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)
+    expr: (sum(pd_cluster_status{type="store_disconnected_count"}) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)
     for: 1m
     labels:
       env: ENV_LABELS_ENV
       level: warning
-      expr:  (sum ( pd_cluster_status{type="store_disconnected_count"} ) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)
+      expr:  (sum(pd_cluster_status{type="store_disconnected_count"}) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)
     annotations:
       description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
       value: '{{ $value }}'
       summary: PD_cluster_lost_connect_tikv_nums
 
   - alert: PD_cluster_low_space
-    expr: (sum ( pd_cluster_status{type="store_low_space_count"} ) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)
+    expr: (sum(pd_cluster_status{type="store_low_space_count"}) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)
     for: 1m
     labels:
       env: ENV_LABELS_ENV
       level: warning
-      expr:  (sum ( pd_cluster_status{type="store_low_space_count"} ) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)
+      expr:  (sum(pd_cluster_status{type="store_low_space_count"}) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)
     annotations:
       description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
       value: '{{ $value }}'
@@ -97,25 +97,13 @@ groups:
       value: '{{ $value }}'
       summary: PD_down_peer_region_nums
 
-  - alert: PD_incorrect_namespace_region_count
-    expr: (sum(pd_regions_status{type="incorrect_namespace_region_count"}) by (instance)  > 100) and (sum(etcd_server_is_leader) by (instance) > 0)
-    for: 1m
-    labels:
-      env: ENV_LABELS_ENV
-      level: warning
-      expr: (sum(pd_regions_status{type="incorrect_namespace_region_count"}) by (instance)  > 100) and (sum(etcd_server_is_leader) by (instance) > 0)
-    annotations:
-      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
-      value: '{{ $value }}'
-      summary: PD_incorrect_namespace_region_count
-
   - alert: PD_pending_peer_region_count
-    expr: (sum(pd_regions_status{type="pending_peer_region_count"}) by (instance)  > 100) and (sum(etcd_server_is_leader) by (instance) > 0)
+    expr: (sum(pd_regions_status{type="pending-peer-region-count"}) by (instance)  > 100) and (sum(etcd_server_is_leader) by (instance) > 0)
     for: 1m
     labels:
       env: ENV_LABELS_ENV
       level: warning
-      expr:  (sum(pd_regions_status{type="pending_peer_region_count"}) by (instance)  > 100) and (sum(etcd_server_is_leader) by (instance) > 0)
+      expr:  (sum(pd_regions_status{type="pending-peer-region-count"}) by (instance)  > 100) and (sum(etcd_server_is_leader) by (instance) > 0)
     annotations:
       description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
       value: '{{ $value }}'
@@ -180,3 +168,15 @@ groups:
       description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
       value: '{{ $value }}'
       summary: PD server has been restarted
+
+  - alert: PD_cluster_slow_tikv_nums
+    expr: (sum(pd_cluster_status{type="store_slow_count"}) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)
+    for: 1m
+    labels:
+      env: ENV_LABELS_ENV
+      level: critical
+      expr:  (sum(pd_cluster_status{type="store_slow_count"}) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)
+    annotations:
+      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
+      value: '{{ $value }}'
+      summary: PD_cluster_slow_tikv_nums