From cd60925897337d790469cc8293f0cbb3a2bdcb36 Mon Sep 17 00:00:00 2001 From: ti-srebot <66930949+ti-srebot@users.noreply.github.com> Date: Fri, 29 Apr 2022 12:44:52 +0800 Subject: [PATCH] metrics: Support multi k8s in grafana dashboards (#32546) (#32953) close pingcap/tidb#32593 --- br/metrics/grafana/br.json | 112 ++++---- br/metrics/grafana/lightning.json | 84 +++--- metrics/grafana/overview.json | 220 ++++++++-------- metrics/grafana/tidb.json | 408 ++++++++++++++++-------------- metrics/grafana/tidb_runtime.json | 101 ++++---- metrics/grafana/tidb_summary.json | 127 ++++++---- 6 files changed, 586 insertions(+), 466 deletions(-) diff --git a/br/metrics/grafana/br.json b/br/metrics/grafana/br.json index 5a2d0ad5d771f..3b48947f0eab1 100644 --- a/br/metrics/grafana/br.json +++ b/br/metrics/grafana/br.json @@ -123,7 +123,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[1m])) by (instance)", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[1m])) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -133,7 +133,7 @@ "step": 4 }, { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"backup_endpoint\"}[1m])) by (instance)", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"backup_endpoint\"}[1m])) by (instance)", "format": "time_series", "hide": true, "intervalFactor": 2, @@ -143,7 +143,7 @@ "step": 4 }, { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[1m])) by (instance, tid) > 0", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[1m])) by (instance, tid) > 0", "format": "time_series", "hide": true, "intervalFactor": 2, @@ -153,7 +153,7 @@ "step": 4 }, { - "expr": "sum(tikv_backup_thread_pool_size{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)", + "expr": "sum(tikv_backup_thread_pool_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)", "format": "time_series", "hide": true, "intervalFactor": 2, @@ -253,7 +253,7 @@ ], "targets": [ { - "expr": "sum(tikv_backup_thread_pool_size{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)", + "expr": "sum(tikv_backup_thread_pool_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -308,7 +308,7 @@ "steppedLine": false, "targets": [ { - "expr": "delta(tikv_backup_error_counter{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", + "expr": "delta(tikv_backup_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -401,7 +401,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "max(rate(tikv_backup_range_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", cf=\"write\"}[1m])) by (le)", + "expr": "max(rate(tikv_backup_range_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", cf=\"write\"}[1m])) by (le)", "format": "heatmap", "instant": false, "intervalFactor": 2, @@ -478,7 +478,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "max(rate(tikv_backup_range_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", cf=\"default\"}[1m])) by (le)", + "expr": "max(rate(tikv_backup_range_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", cf=\"default\"}[1m])) by (le)", "format": "heatmap", "instant": false, "intervalFactor": 2, @@ -560,7 +560,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_backup_range_size_bytes_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tikv_backup_range_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -570,7 +570,7 @@ "step": 4 }, { - "expr": "rate(tikv_backup_range_size_bytes_sum{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "rate(tikv_backup_range_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -666,7 +666,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "max(rate(tikv_backup_range_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[1m])) by (le)", + "expr": "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[1m])) by (le)", "format": "heatmap", "instant": false, "intervalFactor": 2, @@ -743,7 +743,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "max(rate(tikv_backup_range_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"scan\"}[1m])) by (le)", + "expr": "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"scan\"}[1m])) by (le)", "format": "heatmap", "instant": false, "intervalFactor": 2, @@ -820,7 +820,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "max(rate(tikv_backup_range_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"save.*\"}[1m])) by (le)", + "expr": "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"save.*\"}[1m])) by (le)", "format": "heatmap", "instant": false, "intervalFactor": 2, @@ -897,7 +897,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_backup_range_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}} - 99%", @@ -906,7 +906,7 @@ "step": 4 }, { - "expr": "histogram_quantile(0.95, sum(rate(tikv_backup_range_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.95, sum(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}} - 95%", @@ -914,7 +914,7 @@ "step": 4 }, { - "expr": "sum(rate(tikv_backup_range_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type) / sum(rate(tikv_backup_range_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", + "expr": "sum(rate(tikv_backup_range_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type) / sum(rate(tikv_backup_range_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}} - avg", @@ -1008,7 +1008,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "max(rate(tikv_external_storage_create_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le)", + "expr": "max(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le)", "format": "heatmap", "instant": false, "intervalFactor": 2, @@ -1089,14 +1089,14 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tikv_external_storage_create_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le,type))", + "expr": "histogram_quantile(1, sum(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le,type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}-100%", "refId": "E" }, { - "expr": "histogram_quantile(0.99, sum(rate(tikv_external_storage_create_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le,type))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le,type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}-99%", @@ -1195,14 +1195,14 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[1m])) by (le,req))", + "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[1m])) by (le,req))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{req}}-100%", "refId": "E" }, { - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[1m])) by (le,req))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[1m])) by (le,req))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{req}}-99%", @@ -1300,7 +1300,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(node_disk_io_time_seconds_total{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - {{device}}", @@ -1400,7 +1400,7 @@ }, "lines": true, "linewidth": 1, - "links": [], + "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -1417,7 +1417,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1427,7 +1427,7 @@ "step": 4 }, { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance, tid) > 0", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance, tid) > 0", "format": "time_series", "hide": true, "intervalFactor": 2, @@ -1437,7 +1437,7 @@ "step": 4 }, { - "expr": "count(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)", + "expr": "count(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)", "format": "time_series", "hide": true, "intervalFactor": 2, @@ -1537,7 +1537,7 @@ ], "targets": [ { - "expr": "count(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)", + "expr": "count(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -1593,7 +1593,7 @@ "steppedLine": false, "targets": [ { - "expr": "delta(tikv_import_error_counter{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", + "expr": "delta(tikv_import_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1681,7 +1681,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_import_rpc_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, request))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, request))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1777,7 +1777,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(node_disk_io_time_seconds_total{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - {{device}}", @@ -1872,7 +1872,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "max(rate(tikv_import_rpc_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", request=~\"download|write\"}[1m])) by (le)", + "expr": "max(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", request=~\"download|write\"}[1m])) by (le)", "format": "heatmap", "instant": false, "intervalFactor": 2, @@ -1949,7 +1949,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "max(rate(tikv_import_download_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"queue\"}[1m])) by (le)", + "expr": "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"queue\"}[1m])) by (le)", "format": "heatmap", "instant": false, "intervalFactor": 2, @@ -2026,7 +2026,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "max(rate(tikv_import_download_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"read\"}[1m])) by (le)", + "expr": "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"read\"}[1m])) by (le)", "format": "heatmap", "instant": false, "intervalFactor": 2, @@ -2103,7 +2103,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "max(rate(tikv_import_download_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"rewrite\"}[1m])) by (le)", + "expr": "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"rewrite\"}[1m])) by (le)", "format": "heatmap", "instant": false, "intervalFactor": 2, @@ -2180,7 +2180,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "max(rate(tikv_import_rpc_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", request=~\"ingest\"}[1m])) by (le)", + "expr": "max(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", request=~\"ingest\"}[1m])) by (le)", "format": "heatmap", "instant": false, "intervalFactor": 2, @@ -2257,7 +2257,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "max(rate(tikv_import_ingest_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ingest\"}[1m])) by (le)", + "expr": "max(rate(tikv_import_ingest_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ingest\"}[1m])) by (le)", "format": "heatmap", "instant": false, "intervalFactor": 2, @@ -2334,7 +2334,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "max(rate(tikv_import_ingest_byte{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le)", + "expr": "max(rate(tikv_import_ingest_byte{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le)", "format": "heatmap", "instant": false, "intervalFactor": 2, @@ -2406,14 +2406,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_import_download_bytes_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", + "expr": "sum(rate(tikv_import_download_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "total", "refId": "A" }, { - "expr": "rate(tikv_import_download_bytes_sum{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "rate(tikv_import_download_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", @@ -2513,7 +2513,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_raftstore_region_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}) by (instance)", + "expr": "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -2522,7 +2522,7 @@ "step": 10 }, { - "expr": "delta(tikv_raftstore_region_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[30s]) < -10", + "expr": "delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[30s]) < -10", "format": "time_series", "hide": true, "intervalFactor": 2, @@ -2619,7 +2619,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_raftstore_region_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)", + "expr": "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2717,14 +2717,14 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[1m])) by (le,req))", + "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[1m])) by (le,req))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{req}}-100%", "refId": "E" }, { - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[1m])) by (le,req))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[1m])) by (le,req))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{req}}-99%", @@ -2822,7 +2822,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(node_disk_io_time_seconds_total{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - {{device}}", @@ -2884,6 +2884,26 @@ "tags": [], "templating": { "list": [ + { + "allValue": null, + "current": { }, + "datasource": "${DS_TEST-CLUSTER}", + "hide": 2, + "includeAll": false, + "label": "K8s-cluster", + "multi": false, + "name": "k8s_cluster", + "options": [ ], + "query": "label_values(tikv_engine_size_bytes, k8s_cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": null, "current": { @@ -2897,7 +2917,7 @@ "options": [ ], - "query": "label_values(tikv_engine_size_bytes, tidb_cluster)", + "query": "label_values(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster)", "refresh": 2, "regex": "", "sort": 1, diff --git a/br/metrics/grafana/lightning.json b/br/metrics/grafana/lightning.json index d842f9f523a4c..624fc3e91c953 100644 --- a/br/metrics/grafana/lightning.json +++ b/br/metrics/grafana/lightning.json @@ -100,14 +100,14 @@ "steppedLine": false, "targets": [ { - "expr": "rate(tikv_import_write_chunk_bytes_sum{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "rate(tikv_import_write_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "write from lightning", "refId": "B" }, { - "expr": "sum(rate(tikv_import_upload_chunk_bytes_sum{tidb_cluster=\"$tidb_cluster\"}[1m]))", + "expr": "sum(rate(tikv_import_upload_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "upload to tikv", @@ -182,7 +182,7 @@ "steppedLine": false, "targets": [ { - "expr": "1/rate(lightning_chunks{tidb_cluster=\"$tidb_cluster\", state=\"finished\"}[1m]) ", + "expr": "1/rate(lightning_chunks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", state=\"finished\"}[1m]) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -294,7 +294,7 @@ "tableColumn": "", "targets": [ { - "expr": "lightning_chunks{tidb_cluster=\"$tidb_cluster\", state=\"finished\"} / ignoring(state) lightning_chunks{tidb_cluster=\"$tidb_cluster\", state=\"estimated\"}", + "expr": "lightning_chunks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", state=\"finished\"} / ignoring(state) lightning_chunks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", state=\"estimated\"}", "format": "time_series", "instant": false, "intervalFactor": 2, @@ -370,7 +370,7 @@ "tableColumn": "", "targets": [ { - "expr": "lightning_tables{tidb_cluster=\"$tidb_cluster\", state=\"completed\"} / ignoring(state) lightning_tables{tidb_cluster=\"$tidb_cluster\", state=\"pending\"}", + "expr": "lightning_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", state=\"completed\"} / ignoring(state) lightning_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", state=\"pending\"}", "format": "time_series", "instant": false, "intervalFactor": 1, @@ -454,7 +454,7 @@ ], "targets": [ { - "expr": "lightning_tables{tidb_cluster=\"$tidb_cluster\", result=\"failure\"}", + "expr": "lightning_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", result=\"failure\"}", "format": "time_series", "instant": false, "intervalFactor": 2, @@ -510,14 +510,14 @@ "steppedLine": false, "targets": [ { - "expr": "process_resident_memory_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tikv-importer\"}", + "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tikv-importer\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "importer RSS", "refId": "A" }, { - "expr": "go_memstats_heap_inuse_bytes{tidb_cluster=\"$tidb_cluster\", job=\"lightning\"}", + "expr": "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"lightning\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "lightning heap", @@ -592,7 +592,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_goroutines{tidb_cluster=\"$tidb_cluster\", job=\"lightning\"}", + "expr": "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"lightning\"}", "format": "time_series", "instant": false, "intervalFactor": 2, @@ -671,14 +671,14 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", job=\"lightning\"}[30s])*100", + "expr": "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"lightning\"}[30s])*100", "format": "time_series", "intervalFactor": 2, "legendFormat": "Lightning", "refId": "A" }, { - "expr": "rate(process_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", job=\"tikv-importer\"}[30s])*100", + "expr": "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tikv-importer\"}[30s])*100", "format": "time_series", "intervalFactor": 2, "legendFormat": "Importer", @@ -766,7 +766,7 @@ "steppedLine": true, "targets": [ { - "expr": "lightning_idle_workers{tidb_cluster=\"$tidb_cluster\"}", + "expr": "lightning_idle_workers{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{name}}", @@ -842,21 +842,21 @@ "steppedLine": true, "targets": [ { - "expr": "lightning_kv_encoder{tidb_cluster=\"$tidb_cluster\", type=\"open\"} - ignoring(type) lightning_kv_encoder{tidb_cluster=\"$tidb_cluster\", type=\"closed\"}", + "expr": "lightning_kv_encoder{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"open\"} - ignoring(type) lightning_kv_encoder{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"closed\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "KV Encoder", "refId": "A" }, { - "expr": "lightning_importer_engine{tidb_cluster=\"$tidb_cluster\", type=\"open\"} - ignoring(type) lightning_importer_engine{tidb_cluster=\"$tidb_cluster\", type=\"closed\"}", + "expr": "lightning_importer_engine{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"open\"} - ignoring(type) lightning_importer_engine{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"closed\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Importer Engines (via Lightning)", "refId": "B" }, { - "expr": "tikv_import_rpc_duration_count{tidb_cluster=\"$tidb_cluster\", request=\"open_engine\",result=\"ok\"} - ignoring(request) tikv_import_rpc_duration_count{tidb_cluster=\"$tidb_cluster\", request=\"close_engine\",result=\"ok\"}", + "expr": "tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", request=\"open_engine\",result=\"ok\"} - ignoring(request) tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", request=\"close_engine\",result=\"ok\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Importer Engines (via Importer)", @@ -958,7 +958,7 @@ ], "targets": [ { - "expr": "min(tikv_config_rocksdb{tidb_cluster=\"$tidb_cluster\", name=\"hard_pending_compaction_bytes_limit\"}) by (instance)", + "expr": "min(tikv_config_rocksdb{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=\"hard_pending_compaction_bytes_limit\"}) by (instance)", "format": "time_series", "instant": false, "intervalFactor": 2, @@ -1014,14 +1014,14 @@ "steppedLine": false, "targets": [ { - "expr": "rate(lightning_chunk_parser_read_block_seconds_sum{tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(lightning_chunk_parser_read_block_seconds_count{tidb_cluster=\"$tidb_cluster\"}[30s])", + "expr": "rate(lightning_chunk_parser_read_block_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(lightning_chunk_parser_read_block_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "read block", "refId": "A" }, { - "expr": "rate(lightning_apply_worker_seconds_sum{tidb_cluster=\"$tidb_cluster\", name = \"io\"}[30s]) /rate(lightning_apply_worker_seconds_count{tidb_cluster=\"$tidb_cluster\", name = \"io\"}[30s]) ", + "expr": "rate(lightning_apply_worker_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name = \"io\"}[30s]) /rate(lightning_apply_worker_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name = \"io\"}[30s]) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "apply worker", @@ -1096,14 +1096,14 @@ "steppedLine": false, "targets": [ { - "expr": "rate(lightning_row_encode_seconds_sum{tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(lightning_row_encode_seconds_count{tidb_cluster=\"$tidb_cluster\"}[30s])", + "expr": "rate(lightning_row_encode_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(lightning_row_encode_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "row encode", "refId": "A" }, { - "expr": "rate(lightning_block_deliver_seconds_sum{tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(lightning_block_deliver_seconds_count{tidb_cluster=\"$tidb_cluster\"}[30s])", + "expr": "rate(lightning_block_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(lightning_block_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "block deliver", @@ -1190,14 +1190,14 @@ "steppedLine": false, "targets": [ { - "expr": "rate(lightning_block_deliver_bytes_sum{tidb_cluster=\"$tidb_cluster\"}[30s])", + "expr": "rate(lightning_block_deliver_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{kind}} deliver rate", "refId": "B" }, { - "expr": "sum(rate(lightning_block_deliver_bytes_sum{tidb_cluster=\"$tidb_cluster\"}[30s]))", + "expr": "sum(rate(lightning_block_deliver_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "total deliver rate", @@ -1272,21 +1272,21 @@ "steppedLine": false, "targets": [ { - "expr": "lightning_row_read_bytes_sum{tidb_cluster=\"$tidb_cluster\"}", + "expr": "lightning_row_read_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "parser read size", "refId": "A" }, { - "expr": "lightning_block_deliver_bytes_sum{tidb_cluster=\"$tidb_cluster\"}", + "expr": "lightning_block_deliver_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{kind}} deliver size", "refId": "B" }, { - "expr": "pd_cluster_status{tidb_cluster=\"$tidb_cluster\", type=\"storage_size\"} / ignoring(type) pd_config_status{tidb_cluster=\"$tidb_cluster\", type=\"max_replicas\"}", + "expr": "pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"storage_size\"} / ignoring(type) pd_config_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"max_replicas\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "storage_size / replicas", @@ -1373,14 +1373,14 @@ "steppedLine": false, "targets": [ { - "expr": "rate(tikv_import_range_delivery_duration_sum{tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_range_delivery_duration_count{tidb_cluster=\"$tidb_cluster\"}[30s])", + "expr": "rate(tikv_import_range_delivery_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_range_delivery_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "range deliver", "refId": "A" }, { - "expr": "rate(tikv_import_sst_delivery_duration_sum{tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_sst_delivery_duration_count{tidb_cluster=\"$tidb_cluster\"}[30s])", + "expr": "rate(tikv_import_sst_delivery_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_sst_delivery_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "SST file deliver", @@ -1461,14 +1461,14 @@ "steppedLine": false, "targets": [ { - "expr": "rate(tikv_import_split_sst_duration_sum{tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_split_sst_duration_count{tidb_cluster=\"$tidb_cluster\"}[30s])", + "expr": "rate(tikv_import_split_sst_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_split_sst_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "Split SST", "refId": "C" }, { - "expr": "rate(tikv_import_sst_upload_duration_sum{tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_sst_upload_duration_count{tidb_cluster=\"$tidb_cluster\"}[30s])", + "expr": "rate(tikv_import_sst_upload_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_sst_upload_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1476,7 +1476,7 @@ "refId": "D" }, { - "expr": "rate(tikv_import_sst_ingest_duration_sum{tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_sst_ingest_duration_count{tidb_cluster=\"$tidb_cluster\"}[30s])", + "expr": "rate(tikv_import_sst_ingest_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_sst_ingest_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "instant": false, "intervalFactor": 2, @@ -1484,7 +1484,7 @@ "refId": "E" }, { - "expr": "rate(tikv_import_sst_chunk_bytes_sum{tidb_cluster=\"$tidb_cluster\"}[30s])", + "expr": "rate(tikv_import_sst_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "SST size", @@ -1541,6 +1541,26 @@ "tags": [], "templating": { "list": [ + { + "allValue": null, + "current": { }, + "datasource": "${DS_TEST-CLUSTER}", + "hide": 2, + "includeAll": false, + "label": "K8s-cluster", + "multi": false, + "name": "k8s_cluster", + "options": [ ], + "query": "label_values(lightning_chunks, k8s_cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": null, "current": { @@ -1554,7 +1574,7 @@ "options": [ ], - "query": "label_values(lightning_chunks, tidb_cluster)", + "query": "label_values(lightning_chunks{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", "refresh": 2, "regex": "", "sort": 1, diff --git a/metrics/grafana/overview.json b/metrics/grafana/overview.json index ebbfb72ea361c..4017ff43a8126 100644 --- a/metrics/grafana/overview.json +++ b/metrics/grafana/overview.json @@ -153,91 +153,91 @@ ], "targets": [ { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"tidb\"} == 1)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"tidb\"} == 1)", "format": "time_series", "intervalFactor": 2, "legendFormat": "TiDB", "refId": "A" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"pd\"} == 1)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"pd\"} == 1)", "format": "time_series", "intervalFactor": 2, "legendFormat": "PD", "refId": "B" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"tikv\"} == 1)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"tikv\"} == 1)", "format": "time_series", "intervalFactor": 2, "legendFormat": "TiKV", "refId": "C" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"tiflash\"} == 1)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"tiflash\"} == 1)", "format": "time_series", "intervalFactor": 2, "legendFormat": "TiFlash", "refId": "D" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"pump\"} == 1)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"pump\"} == 1)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Pump", "refId": "E" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"drainer\"} == 1)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"drainer\"} == 1)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Drainer", "refId": "F" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"kafka\"} == 1)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"kafka\"} == 1)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Kafka", "refId": "G" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"zookeeper\"} == 1)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"zookeeper\"} == 1)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Zookeeper", "refId": "H" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"node_exporter\"} == 1)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"node_exporter\"} == 1)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Node_exporter", "refId": "I" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"blackbox_exporter\"} == 1)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"blackbox_exporter\"} == 1)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Blackbox_exporter", "refId": "J" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"grafana\"} == 1)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"grafana\"} == 1)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Grafana", "refId": "K" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", job=\"blackbox_exporter_http\"} == 1)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"blackbox_exporter_http\"} == 1)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Pushgateway", "refId": "L" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"kafka_exporter\"} == 1)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"kafka_exporter\"} == 1)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Kafka_exporter", @@ -331,91 +331,91 @@ ], "targets": [ { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"tidb\"} == 0)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"tidb\"} == 0)", "format": "time_series", "intervalFactor": 2, "legendFormat": "TiDB", "refId": "A" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"pd\"} == 0)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"pd\"} == 0)", "format": "time_series", "intervalFactor": 2, "legendFormat": "PD", "refId": "B" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"tikv\"} == 0)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"tikv\"} == 0)", "format": "time_series", "intervalFactor": 2, "legendFormat": "TiKV", "refId": "C" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"tiflash\"} == 0)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"tiflash\"} == 0)", "format": "time_series", "intervalFactor": 2, "legendFormat": "TiFlash", "refId": "D" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"pump\"} == 0)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"pump\"} == 0)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Pump", "refId": "E" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"drainer\"} == 0)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"drainer\"} == 0)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Drainer", "refId": "F" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"kafka\"} == 0)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"kafka\"} == 0)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Kafka", "refId": "G" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"zookeeper\"} == 0)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"zookeeper\"} == 0)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Zookeeper", "refId": "H" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"node_exporter\"} == 0)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"node_exporter\"} == 0)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Node_exporter", "refId": "I" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"blackbox_exporter\"} == 0)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"blackbox_exporter\"} == 0)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Blackbox_exporter", "refId": "J" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"grafana\"} == 0)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"grafana\"} == 0)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Grafana", "refId": "K" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", job=\"blackbox_exporter_http\"} == 0)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"blackbox_exporter_http\"} == 0)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Pushgateway", "refId": "L" }, { - "expr": "\ncount(probe_success{tidb_cluster=\"$tidb_cluster\", group=\"kafka_exporter\"} == 0)", + "expr": "\ncount(probe_success{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", group=\"kafka_exporter\"} == 0)", "format": "time_series", "intervalFactor": 2, "legendFormat": "Kafka_exporter", @@ -497,7 +497,7 @@ "tableColumn": "", "targets": [ { - "expr": "pd_tso_role{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", dc=\"global\"}", + "expr": "pd_tso_role{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", dc=\"global\"}", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -594,7 +594,7 @@ "tableColumn": "", "targets": [ { - "expr": "pd_cluster_status{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",type=\"storage_capacity\"}", + "expr": "pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",type=\"storage_capacity\"}", "format": "time_series", "intervalFactor": 2, "refId": "A", @@ -679,7 +679,7 @@ "tableColumn": "", "targets": [ { - "expr": "pd_cluster_status{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",type=\"storage_size\"}", + "expr": "pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",type=\"storage_size\"}", "intervalFactor": 2, "refId": "A", "step": 60 @@ -759,7 +759,7 @@ "tableColumn": "", "targets": [ { - "expr": "pd_cluster_status{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"leader_count\"}", + "expr": "pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"leader_count\"}", "intervalFactor": 2, "refId": "A", "step": 60 @@ -839,7 +839,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(pd_cluster_status{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_up_count\"})", + "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_up_count\"})", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -912,7 +912,7 @@ ], "targets": [ { - "expr": "sum(pd_cluster_status{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_disconnected_count\"})", + "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_disconnected_count\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Disconnect Stores", @@ -920,7 +920,7 @@ "step": 20 }, { - "expr": "sum(pd_cluster_status{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_unhealth_count\"})", + "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_unhealth_count\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Unhealth Stores", @@ -928,7 +928,7 @@ "step": 20 }, { - "expr": "sum(pd_cluster_status{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_low_space_count\"})", + "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_low_space_count\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "LowSpace Stores", @@ -936,7 +936,7 @@ "step": 20 }, { - "expr": "sum(pd_cluster_status{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_down_count\"})", + "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_down_count\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Down Stores", @@ -944,7 +944,7 @@ "step": 20 }, { - "expr": "sum(pd_cluster_status{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_offline_count\"})", + "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_offline_count\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Offline Stores", @@ -952,7 +952,7 @@ "step": 20 }, { - "expr": "sum(pd_cluster_status{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_tombstone_count\"})", + "expr": "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"store_tombstone_count\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Tombstone Stores", @@ -1012,7 +1012,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}[5m])) by (grpc_method, le))", + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}[5m])) by (grpc_method, le))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1106,7 +1106,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.98, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[30s])) by (type, le))", + "expr": "histogram_quantile(0.98, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (type, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}-98%", @@ -1114,7 +1114,7 @@ "step": 10 }, { - "expr": "avg(rate(pd_client_request_handle_requests_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\"}[30s])) by (type) / avg(rate(pd_client_request_handle_requests_duration_seconds_count{tidb_cluster=\"$tidb_cluster\"}[30s])) by (type)", + "expr": "avg(rate(pd_client_request_handle_requests_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (type) / avg(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (type)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1204,14 +1204,14 @@ "steppedLine": false, "targets": [ { - "expr": "pd_regions_status{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}", + "expr": "pd_regions_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "A" }, { - "expr": "sum(pd_regions_status{tidb_cluster=\"$tidb_cluster\"}) by (instance, type)", + "expr": "sum(pd_regions_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance, type)", "format": "time_series", "hide": true, "intervalFactor": 2, @@ -1302,7 +1302,7 @@ "steppedLine": false, "targets": [ { - "expr": "pd_hotspot_status{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",type=\"hot_write_region_as_leader\"}", + "expr": "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",type=\"hot_write_region_as_leader\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "store-{{store}}", @@ -1393,7 +1393,7 @@ "steppedLine": false, "targets": [ { - "expr": "pd_hotspot_status{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",type=\"hot_read_region_as_leader\"}", + "expr": "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\",type=\"hot_read_region_as_leader\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "store-{{store}}", @@ -1483,7 +1483,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(pd_scheduler_region_heartbeat{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"report\", status=\"ok\"}[1m])) by (store)", + "expr": "sum(delta(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", type=\"report\", status=\"ok\"}[1m])) by (store)", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1575,7 +1575,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(pd_scheduler_region_heartbeat_latency_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[5m])) by (store, le))", + "expr": "histogram_quantile(0.99, sum(rate(pd_scheduler_region_heartbeat_latency_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (store, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "store-{{store}}", @@ -1686,7 +1686,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_executor_statement_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -1775,7 +1775,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "999", @@ -1783,7 +1783,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 3, "legendFormat": "99", @@ -1791,14 +1791,14 @@ "step": 15 }, { - "expr": "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95", "refId": "C" }, { - "expr": "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "80", @@ -1887,7 +1887,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(tidb_server_query_total{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{type}} {{result}}", @@ -1977,7 +1977,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tidb_server_execute_error_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(increase(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": " {{type}}", @@ -2073,7 +2073,7 @@ "steppedLine": true, "targets": [ { - "expr": "tidb_server_connections{tidb_cluster=\"$tidb_cluster\"}", + "expr": "tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2081,7 +2081,7 @@ "step": 10 }, { - "expr": "sum(tidb_server_connections{tidb_cluster=\"$tidb_cluster\"})", + "expr": "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "total", @@ -2172,7 +2172,7 @@ "steppedLine": false, "targets": [ { - "expr": "process_resident_memory_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", + "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "process-{{instance}}", @@ -2180,7 +2180,7 @@ "step": 10 }, { - "expr": "go_memstats_heap_inuse_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", + "expr": "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", "legendFormat": "HeapInuse-{{instance}}", "format": "time_series", "intervalFactor": 2, @@ -2266,7 +2266,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_session_transaction_duration_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -2351,21 +2351,21 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95", "refId": "B" }, { - "expr": "histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "80", @@ -2454,7 +2454,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -2545,7 +2545,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -2633,7 +2633,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m]))", + "expr": "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "cmd", @@ -2641,7 +2641,7 @@ "step": 10 }, { - "expr": "sum(rate(pd_client_request_handle_requests_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m]))", + "expr": "sum(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "request", @@ -2731,7 +2731,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", + "expr": "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "999", @@ -2739,14 +2739,14 @@ "step": 10 }, { - "expr": "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "B" }, { - "expr": "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "90", @@ -2836,7 +2836,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_region_err_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_tikvclient_region_err_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -2927,7 +2927,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_lock_resolver_actions_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_tikvclient_lock_resolver_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -3022,7 +3022,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -3114,7 +3114,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_backoff_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_tikvclient_backoff_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -3226,7 +3226,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_raftstore_region_count{tidb_cluster=\"$tidb_cluster\", type=\"leader\"}) by (instance)", + "expr": "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"leader\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -3235,7 +3235,7 @@ "step": 10 }, { - "expr": "delta(tikv_raftstore_region_count{tidb_cluster=\"$tidb_cluster\", type=\"leader\"}[30s]) < -10", + "expr": "delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"leader\"}[30s]) < -10", "format": "time_series", "hide": true, "intervalFactor": 2, @@ -3329,7 +3329,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_raftstore_region_count{tidb_cluster=\"$tidb_cluster\", type=\"region\"}) by (instance)", + "expr": "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"region\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -3419,7 +3419,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", job=\"tikv\"}[1m])) by (instance)", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tikv\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -3506,7 +3506,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(process_resident_memory_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tikv\"}) by (instance)", + "expr": "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tikv\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -3595,7 +3595,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_engine_size_bytes{tidb_cluster=\"$tidb_cluster\"}) by (instance)", + "expr": "sum(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -3684,7 +3684,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_engine_size_bytes{tidb_cluster=\"$tidb_cluster\"}) by (type)", + "expr": "sum(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -3778,7 +3778,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_channel_full_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance, type)", + "expr": "sum(rate(tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance, type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - {{type}}", @@ -3877,7 +3877,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_server_report_failure_msg_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type,instance,store_id)", + "expr": "sum(rate(tikv_server_report_failure_msg_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type,instance,store_id)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - {{type}} - to - {{store_id}}", @@ -3968,7 +3968,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_scheduler_contex_total{tidb_cluster=\"$tidb_cluster\"}) by (instance)", + "expr": "sum(tikv_scheduler_contex_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -4058,7 +4058,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_executor_count{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tikv_coprocessor_executor_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -4149,7 +4149,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,req))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,req))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{req}}-99%", @@ -4157,7 +4157,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,req))", + "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,req))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{req}}-95%", @@ -4165,7 +4165,7 @@ "step": 10 }, { - "expr": " sum(rate(tikv_coprocessor_request_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", req=\"select\"}[1m])) / sum(rate(tikv_coprocessor_request_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", req=\"select\"}[1m]))", + "expr": " sum(rate(tikv_coprocessor_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", req=\"select\"}[1m])) / sum(rate(tikv_coprocessor_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", req=\"select\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "select-avg", @@ -4173,7 +4173,7 @@ "step": 10 }, { - "expr": " sum(rate(tikv_coprocessor_request_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", req=\"index\"}[1m])) / sum(rate(tikv_coprocessor_request_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", req=\"index\"}[1m]))", + "expr": " sum(rate(tikv_coprocessor_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", req=\"index\"}[1m])) / sum(rate(tikv_coprocessor_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", req=\"index\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "index-avg", @@ -4262,7 +4262,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", name=~\"raftstore_.*\"}[1m])) by (instance)", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"raftstore_.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -4352,7 +4352,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", name=~\"cop_.*\"}[1m])) by (instance)", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"cop_.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -4491,7 +4491,7 @@ ], "targets": [ { - "expr": "count(node_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", mode=\"user\"}) by (instance)", + "expr": "count(node_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", mode=\"user\"}) by (instance)", "format": "table", "instant": true, "intervalFactor": 2, @@ -4603,7 +4603,7 @@ ], "targets": [ { - "expr": "node_memory_MemTotal_bytes{tidb_cluster=\"$tidb_cluster\"}", + "expr": "node_memory_MemTotal_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "table", "instant": true, "intervalFactor": 2, @@ -4657,7 +4657,7 @@ "steppedLine": false, "targets": [ { - "expr": "100 - avg by (instance) (irate(node_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", mode=\"idle\"}[1m]) ) * 100", + "expr": "100 - avg by (instance) (irate(node_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", mode=\"idle\"}[1m]) ) * 100", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -4747,7 +4747,7 @@ "steppedLine": false, "targets": [ { - "expr": "node_load1{tidb_cluster=\"$tidb_cluster\"}", + "expr": "node_load1{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -4834,7 +4834,7 @@ "steppedLine": false, "targets": [ { - "expr": "node_memory_MemAvailable_bytes{tidb_cluster=\"$tidb_cluster\"}", + "expr": "node_memory_MemAvailable_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ instance }}", @@ -4922,14 +4922,14 @@ "steppedLine": false, "targets": [ { - "expr": "irate(node_network_receive_bytes_total{tidb_cluster=\"$tidb_cluster\", device!=\"lo\"}[5m])", + "expr": "irate(node_network_receive_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", device!=\"lo\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "Inbound: {{instance}}-{{device}}", "refId": "A" }, { - "expr": "irate(node_network_transmit_bytes_total{tidb_cluster=\"$tidb_cluster\", device!=\"lo\"}[5m])", + "expr": "irate(node_network_transmit_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", device!=\"lo\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "Outbound: {{instance}}-{{device}}", @@ -5018,7 +5018,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(node_netstat_Tcp_RetransSegs{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "irate(node_netstat_Tcp_RetransSegs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - TCPSlowStartRetrans", @@ -5108,7 +5108,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(node_disk_io_time_seconds_total{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "irate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - {{device}}", @@ -5169,6 +5169,26 @@ "tags": [], "templating": { "list": [ + { + "allValue": null, + "current": {}, + "datasource": "${DS_TEST-CLUSTER}", + "hide": 2, + "includeAll": false, + "label": "K8s-cluster", + "multi": false, + "name": "k8s_cluster", + "options": [], + "query": "label_values(pd_cluster_status, k8s_cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": null, "current": {}, @@ -5179,7 +5199,7 @@ "multi": false, "name": "tidb_cluster", "options": [], - "query": "label_values(pd_cluster_status, tidb_cluster)", + "query": "label_values(pd_cluster_status{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -5200,7 +5220,7 @@ "multi": false, "name": "instance", "options": [], - "query": "label_values(pd_cluster_status{tidb_cluster=\"$tidb_cluster\"}, instance)", + "query": "label_values(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", "refresh": 1, "regex": "", "skipUrlSync": false, diff --git a/metrics/grafana/tidb.json b/metrics/grafana/tidb.json index b32a37fae8916..00ecaae15db85 100644 --- a/metrics/grafana/tidb.json +++ b/metrics/grafana/tidb.json @@ -107,28 +107,28 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "999", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "B" }, { - "expr": "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95", "refId": "C" }, { - "expr": "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "80", @@ -225,7 +225,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_server_query_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (result)", + "expr": "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (result)", "format": "time_series", "instant": false, "intervalFactor": 2, @@ -234,7 +234,7 @@ "step": 60 }, { - "expr": "sum(rate(tidb_server_query_total{tidb_cluster=\"$tidb_cluster\", result=\"OK\"}[1m] offset 1d))", + "expr": "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", result=\"OK\"}[1m] offset 1d))", "format": "time_series", "hide": true, "instant": false, @@ -244,7 +244,7 @@ "step": 90 }, { - "expr": "sum(tidb_server_connections{tidb_cluster=\"$tidb_cluster\"}) * sum(rate(tidb_server_handle_query_duration_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m])) / sum(rate(tidb_server_handle_query_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\"}[1m]))", + "expr": "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) * sum(rate(tidb_server_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) / sum(rate(tidb_server_handle_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "hide": true, "instant": false, @@ -344,7 +344,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_executor_statement_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -352,7 +352,7 @@ "step": 30 }, { - "expr": "sum(rate(tidb_executor_statement_total{tidb_cluster=\"$tidb_cluster\"}[1m]))", + "expr": "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "total", @@ -455,7 +455,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(tidb_server_query_total{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{type}} {{result}}", @@ -549,7 +549,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tidb_server_execute_error_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type, instance)", + "expr": "sum(increase(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": " {{type}}-{{instance}}", @@ -635,21 +635,21 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_process_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,sql_type))", + "expr": "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_process_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,sql_type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "all_proc_{{sql_type}}", "refId": "A" }, { - "expr": "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_cop_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,sql_type))", + "expr": "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_cop_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,sql_type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "all_cop_proc_{{sql_type}}", "refId": "B" }, { - "expr": "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_wait_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,sql_type))", + "expr": "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,sql_type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "all_cop_wait_{{sql_type}}", @@ -735,7 +735,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", in_txn='1'}[1m])) by (le,in_txn))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", in_txn='1'}[1m])) by (le,in_txn))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -743,7 +743,7 @@ "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", in_txn='0'}[1m])) by (le,in_txn))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", in_txn='0'}[1m])) by (le,in_txn))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -751,7 +751,7 @@ "refId": "B" }, { - "expr": "histogram_quantile(0.90, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", in_txn='1'}[1m])) by (le,in_txn))", + "expr": "histogram_quantile(0.90, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", in_txn='1'}[1m])) by (le,in_txn))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -759,7 +759,7 @@ "refId": "C" }, { - "expr": "histogram_quantile(0.90, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", in_txn='0'}[1m])) by (le,in_txn))", + "expr": "histogram_quantile(0.90, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", in_txn='0'}[1m])) by (le,in_txn))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -767,7 +767,7 @@ "refId": "D" }, { - "expr": "histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", in_txn='1'}[1m])) by (le,in_txn))", + "expr": "histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", in_txn='1'}[1m])) by (le,in_txn))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -775,7 +775,7 @@ "refId": "E" }, { - "expr": "histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", in_txn='0'}[1m])) by (le,in_txn))", + "expr": "histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", in_txn='0'}[1m])) by (le,in_txn))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -862,7 +862,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,sql_type))", + "expr": "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,sql_type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{sql_type}}", @@ -948,7 +948,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,sql_type))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,sql_type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{sql_type}}", @@ -1034,7 +1034,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,sql_type))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,sql_type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{sql_type}}", @@ -1120,7 +1120,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,sql_type))", + "expr": "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,sql_type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{sql_type}}", @@ -1226,7 +1226,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1322,7 +1322,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ instance }}", @@ -1417,7 +1417,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1510,7 +1510,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1603,7 +1603,7 @@ "steppedLine": false, "targets": [ { - "expr": "increase(tidb_server_execute_error_total{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "increase(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}} @ {{instance}}", @@ -1694,7 +1694,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_session_restricted_sql_total{tidb_cluster=\"$tidb_cluster\"}[30s]))", + "expr": "sum(rate(tidb_session_restricted_sql_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -1810,7 +1810,7 @@ "steppedLine": false, "targets": [ { - "expr": "(time() - process_start_time_seconds{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"})", + "expr": "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"})", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1907,14 +1907,14 @@ "steppedLine": false, "targets": [ { - "expr": "process_resident_memory_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", + "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "process-{{instance}}", "refId": "A" }, { - "expr": "go_memstats_heap_sys_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", + "expr": "go_memstats_heap_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -1922,14 +1922,14 @@ "refId": "B" }, { - "expr": "go_memstats_heap_inuse_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", + "expr": "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "HeapInuse-{{instance}}", "refId": "C" }, { - "expr": "go_memstats_heap_alloc_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", + "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -1937,7 +1937,7 @@ "refId": "D" }, { - "expr": "go_memstats_heap_idle_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", + "expr": "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -1945,14 +1945,14 @@ "refId": "E" }, { - "expr": "go_memstats_heap_released_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", + "expr": "go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", "hide": true, "interval": "", "legendFormat": "HeapReleased-{{instance}}", "refId": "F" }, { - "expr": "go_memstats_next_gc_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", + "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", "hide": true, "interval": "", "legendFormat": "GCTrigger-{{instance}}", @@ -2056,7 +2056,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(process_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}[30s])", + "expr": "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}[30s])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -2065,7 +2065,7 @@ "step": 40 }, { - "expr": "tidb_server_maxprocs{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", + "expr": "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", "legendFormat": "limit-{{instance}}", "refId": "B" } @@ -2161,7 +2161,7 @@ "steppedLine": false, "targets": [ { - "expr": "tidb_server_connections{tidb_cluster=\"$tidb_cluster\"}", + "expr": "tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2169,7 +2169,7 @@ "step": 40 }, { - "expr": "sum(tidb_server_connections{tidb_cluster=\"$tidb_cluster\"})", + "expr": "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "total", @@ -2259,7 +2259,7 @@ "steppedLine": false, "targets": [ { - "expr": "process_open_fds{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", + "expr": "process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2356,7 +2356,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tidb_server_disconnection_total{tidb_cluster=\"$tidb_cluster\"}) by (instance, result)", + "expr": "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance, result)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{result}}", @@ -2446,7 +2446,7 @@ "steppedLine": false, "targets": [ { - "expr": " go_goroutines{tidb_cluster=\"$tidb_cluster\", job=~\"tidb.*\"}", + "expr": " go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=~\"tidb.*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2534,7 +2534,7 @@ "steppedLine": false, "targets": [ { - "expr": "increase(tidb_server_event_total{tidb_cluster=\"$tidb_cluster\"}[10m])", + "expr": "increase(tidb_server_event_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[10m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-server {{type}}", @@ -2631,7 +2631,7 @@ "steppedLine": false, "targets": [ { - "expr": "tidb_server_prepared_stmts{tidb_cluster=\"$tidb_cluster\"}", + "expr": "tidb_server_prepared_stmts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2639,7 +2639,7 @@ "step": 40 }, { - "expr": "sum(tidb_server_prepared_stmts{tidb_cluster=\"$tidb_cluster\"})", + "expr": "sum(tidb_server_prepared_stmts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "total", @@ -2729,7 +2729,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tidb_monitor_keep_alive_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", + "expr": "sum(increase(tidb_monitor_keep_alive_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2817,7 +2817,7 @@ "steppedLine": false, "targets": [ { - "expr": "increase(tidb_server_panic_total{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "increase(tidb_server_panic_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -2825,7 +2825,7 @@ "refId": "A" }, { - "expr": "increase(tidb_server_critical_error_total{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "increase(tidb_server_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -2914,7 +2914,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tidb_monitor_time_jump_back_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", + "expr": "sum(increase(tidb_monitor_time_jump_back_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -3000,7 +3000,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_server_get_token_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_server_get_token_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99", @@ -3088,7 +3088,7 @@ "steppedLine": false, "targets": [ { - "expr": "tidb_server_critical_error_total{tidb_cluster=\"$tidb_cluster\"}", + "expr": "tidb_server_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -3180,14 +3180,14 @@ "steppedLine": false, "targets": [ { - "expr": "rate(tidb_server_packet_io_bytes_sum{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "rate(tidb_server_packet_io_bytes_sum{k8s_cluster=\"k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}-rate", "refId": "A" }, { - "expr": "tidb_server_packet_io_bytes_sum{tidb_cluster=\"$tidb_cluster\"}", + "expr": "tidb_server_packet_io_bytes_sum{k8s_cluster=\"k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}-total", @@ -3275,7 +3275,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tidb_server_handshake_error_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", + "expr": "sum(increase(tidb_server_handshake_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -3386,7 +3386,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_session_transaction_duration_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type, txn_mode)", + "expr": "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type, txn_mode)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}-{{txn_mode}}", @@ -3480,21 +3480,21 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, txn_mode))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, txn_mode))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99-{{txn_mode}}", "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, txn_mode))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, txn_mode))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95-{{txn_mode}}", "refId": "B" }, { - "expr": "histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, txn_mode))", + "expr": "histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, txn_mode))", "format": "time_series", "intervalFactor": 2, "legendFormat": "80-{{txn_mode}}", @@ -3720,7 +3720,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_session_retry_error_total{tidb_cluster=\"$tidb_cluster\"}[30s])) by (type, sql_type)", + "expr": "sum(rate(tidb_session_retry_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (type, sql_type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}-{{sql_type}}", @@ -3814,21 +3814,21 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95", "refId": "B" }, { - "expr": "histogram_quantile(0.80, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.80, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "80", @@ -3923,7 +3923,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", type=\"commit\"}[1m])) by (instance)", + "expr": "sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"commit\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -4017,21 +4017,21 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99-{{type}}", "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95-{{type}}", "refId": "B" }, { - "expr": "histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "80-{{type}}", @@ -4200,14 +4200,14 @@ "steppedLine": false, "targets": [ { - "expr": "rate(tidb_tikvclient_txn_write_kv_num_sum{tidb_cluster=\"$tidb_cluster\"}[30s])", + "expr": "rate(tidb_tikvclient_txn_write_kv_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-rate", "refId": "A" }, { - "expr": "tidb_tikvclient_txn_write_kv_num_sum{tidb_cluster=\"$tidb_cluster\"}", + "expr": "tidb_tikvclient_txn_write_kv_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-sum", @@ -4440,7 +4440,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "80-{{type}}", @@ -4448,14 +4448,14 @@ "step": 40 }, { - "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 1, "legendFormat": "95-{{type}}", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99-{{type}}", @@ -4558,7 +4558,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(tidb_tikvclient_txn_write_size_bytes_sum{tidb_cluster=\"$tidb_cluster\"}[30s])", + "expr": "rate(tidb_tikvclient_txn_write_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-rate", @@ -4566,7 +4566,7 @@ "step": 40 }, { - "expr": "tidb_tikvclient_txn_write_size_bytes_sum{tidb_cluster=\"$tidb_cluster\"}", + "expr": "tidb_tikvclient_txn_write_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-sum", @@ -4731,7 +4731,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.80, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.80, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "80", @@ -4739,14 +4739,14 @@ "step": 40 }, { - "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "95", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99", @@ -4842,7 +4842,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_ttl_lifetime_reach_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", + "expr": "sum(rate(tidb_tikvclient_ttl_lifetime_reach_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", @@ -4940,7 +4940,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_load_safepoint_total{tidb_cluster=\"$tidb_cluster\", type=\"ok\"}[1m])) by (instance)", + "expr": "sum(rate(tidb_tikvclient_load_safepoint_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"ok\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -5104,21 +5104,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_commit_txn_counter{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_tikvclient_commit_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "2PC-{{type}}", "refId": "C" }, { - "expr": "sum(rate(tidb_tikvclient_async_commit_txn_counter{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_tikvclient_async_commit_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "async commit-{{type}}", "refId": "A" }, { - "expr": "sum(rate(tidb_tikvclient_one_pc_txn_counter{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_tikvclient_one_pc_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "1PC-{{type}}", @@ -5225,7 +5225,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_count_bucket{tidb_cluster=\"$tidb_cluster\"}[1m]))", + "expr": "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "count - {{instance}}", @@ -5233,7 +5233,7 @@ "step": 40 }, { - "expr": "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m]))", + "expr": "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -5331,7 +5331,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_safets_update_counter{tidb_cluster=\"$tidb_cluster\"}[1m])) by (result, store)", + "expr": "sum(rate(tidb_tikvclient_safets_update_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (result, store)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{result}}-store-{{store}}", @@ -5426,7 +5426,7 @@ "steppedLine": false, "targets": [ { - "expr": "tidb_tikvclient_min_safets_gap_seconds{tidb_cluster=\"$tidb_cluster\"}", + "expr": "tidb_tikvclient_min_safets_gap_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-store-{{store}}", @@ -5540,7 +5540,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, sql_type))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, sql_type))", "format": "time_series", "instant": false, "intervalFactor": 2, @@ -5641,7 +5641,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tidb_session_compile_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, sql_type))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_session_compile_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, sql_type))", "format": "time_series", "instant": false, "intervalFactor": 2, @@ -5742,7 +5742,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, sql_type))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, sql_type))", "format": "time_series", "instant": false, "intervalFactor": 2, @@ -5842,7 +5842,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_executor_expensive_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_executor_expensive_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -5940,7 +5940,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_server_plan_cache_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_server_plan_cache_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -6052,14 +6052,14 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.999, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "999-{{type}}", "refId": "D" }, { - "expr": "histogram_quantile(0.99, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -6069,14 +6069,14 @@ "step": 4 }, { - "expr": "histogram_quantile(0.90, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.90, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "90-{{type}}", "refId": "B" }, { - "expr": "histogram_quantile(0.50, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.50, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -6168,7 +6168,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_distsql_handle_query_duration_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m])) by (copr_type)", + "expr": "sum(rate(tidb_distsql_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (copr_type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{copr_type}}", @@ -6261,7 +6261,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_distsql_scan_keys_partial_num_count{tidb_cluster=\"$tidb_cluster\"}[1m]))", + "expr": "sum(rate(tidb_distsql_scan_keys_partial_num_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -6350,21 +6350,21 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_num_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "100", "refId": "A" }, { - "expr": "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_num_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "90", "refId": "B" }, { - "expr": "histogram_quantile(0.50, sum(rate(tidb_distsql_scan_keys_num_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "50", @@ -6450,21 +6450,21 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "100", "refId": "A" }, { - "expr": "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "90", "refId": "B" }, { - "expr": "histogram_quantile(0.80, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.80, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "50", @@ -6550,21 +6550,21 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_distsql_partial_num_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(1, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "100", "refId": "A" }, { - "expr": "histogram_quantile(0.90, sum(rate(tidb_distsql_partial_num_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "90", "refId": "B" }, { - "expr": "histogram_quantile(0.50, sum(rate(tidb_distsql_partial_num_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "50", @@ -6657,7 +6657,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_distsql_copr_cache_sum{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_distsql_copr_cache_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -6752,7 +6752,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_cop_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_cop_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -6862,7 +6862,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_backoff_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "999", @@ -6870,14 +6870,14 @@ "step": 40 }, { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_backoff_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "B" }, { - "expr": "histogram_quantile(0.80, sum(rate(tidb_tikvclient_backoff_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.80, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "80", @@ -6972,7 +6972,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_region_err_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_tikvclient_region_err_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -6981,7 +6981,7 @@ "step": 40 }, { - "expr": "sum(rate(tidb_tikvclient_region_err_total{tidb_cluster=\"$tidb_cluster\"}{EXTERNAL_LABELtype=\"server_is_busy\"}[1m]))", + "expr": "sum(rate(tidb_tikvclient_region_err_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}{EXTERNAL_LABELtype=\"server_is_busy\"}[1m]))", "format": "time_series", "hide": true, "intervalFactor": 2, @@ -7073,7 +7073,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_backoff_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_tikvclient_backoff_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -7166,7 +7166,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_lock_resolver_actions_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_tikvclient_lock_resolver_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -7262,7 +7262,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_lock_cleanup_task_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_tikvclient_lock_cleanup_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "cleanup_secondary_failure_{{type}}", @@ -7271,7 +7271,7 @@ "step": 40 }, { - "expr": "sum(rate(tidb_tikvclient_load_safepoint_total{tidb_cluster=\"$tidb_cluster\", type=\"fail\"}[1m]))", + "expr": "sum(rate(tidb_tikvclient_load_safepoint_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"fail\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "load_safepoint_failure", @@ -7367,7 +7367,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_replica_selector_failure_counter{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_tikvclient_replica_selector_failure_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{type}}", @@ -7474,7 +7474,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_request_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance, type)", + "expr": "sum(rate(tidb_tikvclient_request_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance, type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}", @@ -7569,7 +7569,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{tidb_cluster=\"$tidb_cluster\", store!=\"0\"}[1m])) by (le, store))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store!=\"0\"}[1m])) by (le, store))", "format": "time_series", "intervalFactor": 2, "legendFormat": "store-{{store}}", @@ -7664,7 +7664,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{tidb_cluster=\"$tidb_cluster\", store!=\"0\"}[1m])) by (le,type))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store!=\"0\"}[1m])) by (le,type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -7757,7 +7757,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_forward_request_counter{tidb_cluster=\"$tidb_cluster\"}[1m])) by (from_store, to_store, result)", + "expr": "sum(rate(tidb_tikvclient_forward_request_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (from_store, to_store, result)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{from_store}}-to-{{to_store}}-{{result}}", @@ -7850,7 +7850,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_forward_request_counter{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type, result)", + "expr": "sum(rate(tidb_tikvclient_forward_request_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type, result)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}-{{result}}", @@ -7958,7 +7958,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", type!=\"tso\"}[1m])) by (type)", + "expr": "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type!=\"tso\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -8051,7 +8051,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type!~\"tso|tso_async_wait\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type!~\"tso|tso_async_wait\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "999-{{type}}", @@ -8059,14 +8059,14 @@ "step": 10 }, { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type!~\"tso|tso_async_wait\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type!~\"tso|tso_async_wait\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99-{{type}}", "refId": "B" }, { - "expr": "histogram_quantile(0.90, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type!~\"tso|tso_async_wait\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.90, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type!~\"tso|tso_async_wait\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "90-{{type}}", @@ -8158,7 +8158,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(pd_client_cmd_handle_failed_cmds_duration_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(pd_client_cmd_handle_failed_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -8249,14 +8249,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m]))", + "expr": "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "cmd", "refId": "C" }, { - "expr": "sum(rate(pd_client_request_handle_requests_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m]))", + "expr": "sum(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "request", @@ -8346,7 +8346,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", + "expr": "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "999", @@ -8354,14 +8354,14 @@ "step": 10 }, { - "expr": "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "B" }, { - "expr": "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "90", @@ -8451,7 +8451,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", + "expr": "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "999", @@ -8459,14 +8459,14 @@ "step": 10 }, { - "expr": "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "B" }, { - "expr": "histogram_quantile(0.90, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "90", @@ -8556,7 +8556,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"tso_async_wait\"}[1m])) by (le))", + "expr": "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso_async_wait\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "999", @@ -8564,14 +8564,14 @@ "step": 10 }, { - "expr": "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"tso_async_wait\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso_async_wait\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "B" }, { - "expr": "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"tso_async_wait\"}[1m])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso_async_wait\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "90", @@ -8768,7 +8768,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -8867,7 +8867,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_domain_load_schema_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance,type)", + "expr": "sum(rate(tidb_domain_load_schema_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance,type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}", @@ -8964,7 +8964,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tidb_session_schema_lease_error_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", + "expr": "sum(increase(tidb_session_schema_lease_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -9063,7 +9063,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_domain_load_privilege_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance,type)", + "expr": "sum(rate(tidb_domain_load_privilege_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance,type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}", @@ -9172,7 +9172,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tidb_ddl_handle_job_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_ddl_handle_job_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -9264,7 +9264,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_ddl_batch_add_idx_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", + "expr": "histogram_quantile(1, sum(rate(tidb_ddl_batch_add_idx_duration_seconds_bucket{k8s_cluster=\"k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -9361,7 +9361,7 @@ "steppedLine": false, "targets": [ { - "expr": "tidb_ddl_waiting_jobs{tidb_cluster=\"$tidb_cluster\"}", + "expr": "tidb_ddl_waiting_jobs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}", @@ -9449,7 +9449,7 @@ "steppedLine": false, "targets": [ { - "expr": "increase(tidb_ddl_worker_operation_total{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "increase(tidb_ddl_worker_operation_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}", @@ -9539,7 +9539,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(increase(tidb_ddl_worker_operation_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type, action, result))", + "expr": "histogram_quantile(0.99, sum(increase(tidb_ddl_worker_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type, action, result))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}-{{action}}-{{result}}", @@ -9627,7 +9627,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_ddl_deploy_syncer_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[2m])) by (le, type, result))", + "expr": "histogram_quantile(1, sum(rate(tidb_ddl_deploy_syncer_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (le, type, result))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}-{{result}}", @@ -9715,7 +9715,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_ddl_owner_handle_syncer_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[2m])) by (le, type, result))", + "expr": "histogram_quantile(1, sum(rate(tidb_ddl_owner_handle_syncer_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (le, type, result))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}-{{result}}", @@ -9803,7 +9803,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_ddl_update_self_ver_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[2m])) by (le, result))", + "expr": "histogram_quantile(1, sum(rate(tidb_ddl_update_self_ver_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (le, result))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{result}}", @@ -9889,14 +9889,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_ddl_handle_job_duration_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_ddl_handle_job_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ type }}", "refId": "A" }, { - "expr": "sum(rate(tidb_ddl_handle_job_duration_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m]))", + "expr": "sum(rate(tidb_ddl_handle_job_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "total", @@ -9985,14 +9985,14 @@ "steppedLine": false, "targets": [ { - "expr": "irate(tidb_ddl_add_index_percentage_progress{tidb_cluster=\"$tidb_cluster\", type=\"add_index\"}[1m])", + "expr": "irate(tidb_ddl_add_index_percentage_progress{k8s_cluster=\"k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"add_index\"}[1m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", "refId": "A" }, { - "expr": "irate(tidb_ddl_add_index_percentage_progress{tidb_cluster=\"$tidb_cluster\", type=\"modify_column\"}[1m])", + "expr": "irate(tidb_ddl_add_index_percentage_progress{k8s_cluster=\"k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"modify_column\"}[1m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", @@ -10093,7 +10093,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tidb_statistics_auto_analyze_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_statistics_auto_analyze_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "auto analyze duration", @@ -10180,7 +10180,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_statistics_auto_analyze_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_statistics_auto_analyze_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -10267,7 +10267,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", @@ -10275,14 +10275,14 @@ "step": 30 }, { - "expr": "histogram_quantile(0.90, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "90", "refId": "B" }, { - "expr": "histogram_quantile(0.50, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "50", @@ -10368,7 +10368,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_statistics_pseudo_estimation_total{tidb_cluster=\"$tidb_cluster\"}[30s])) by (type)", + "expr": "sum(rate(tidb_statistics_pseudo_estimation_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -10455,7 +10455,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_statistics_dump_feedback_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_statistics_dump_feedback_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -10542,7 +10542,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_statistics_store_query_feedback_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type) ", + "expr": "sum(rate(tidb_statistics_store_query_feedback_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -10629,7 +10629,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_statistics_high_error_rate_feedback_total{tidb_cluster=\"$tidb_cluster\"}[1m]))", + "expr": "sum(rate(tidb_statistics_high_error_rate_feedback_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -10716,7 +10716,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_statistics_update_stats_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_statistics_update_stats_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -10810,7 +10810,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_statistics_fast_analyze_status_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", + "expr": "histogram_quantile(1, sum(rate(tidb_statistics_fast_analyze_status_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -10908,7 +10908,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(tidb_statistics_sync_load_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_statistics_sync_load_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -10918,7 +10918,7 @@ }, { "exemplar": true, - "expr": "sum(rate(tidb_statistics_sync_load_timeout_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_statistics_sync_load_timeout_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "hide": false, "interval": "", @@ -11017,7 +11017,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(tidb_statistics_sync_load_latency_millis_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_statistics_sync_load_latency_millis_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -11027,7 +11027,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(tidb_statistics_read_stats_latency_millis_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_statistics_read_stats_latency_millis_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "hide": false, "interval": "", @@ -11136,7 +11136,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tidb_owner_new_session_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, result))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_owner_new_session_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, result))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{result}}", @@ -11224,7 +11224,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_owner_watch_owner_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type, result, instance)", + "expr": "sum(rate(tidb_owner_watch_owner_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type, result, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}-{{result}}-{{instance}}", @@ -11327,7 +11327,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_autoid_operation_duration_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m]))", + "expr": "sum(rate(tidb_autoid_operation_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "AutoID QPS", @@ -11415,14 +11415,14 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_autoid_operation_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_autoid_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99-{{type}}", "refId": "B" }, { - "expr": "histogram_quantile(0.80, sum(rate(tidb_autoid_operation_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.80, sum(rate(tidb_autoid_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "80-{{type}}", @@ -11508,7 +11508,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_region_cache_operations_total{tidb_cluster=\"$tidb_cluster\", result=\"err\"}[1m])) by (type)", + "expr": "sum(rate(tidb_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", result=\"err\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -11597,7 +11597,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_meta_operation_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_meta_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -11700,7 +11700,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tidb_tikvclient_gc_worker_actions_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(increase(tidb_tikvclient_gc_worker_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -11789,7 +11789,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_gc_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", @@ -11877,7 +11877,7 @@ "steppedLine": false, "targets": [ { - "expr": "max(tidb_tikvclient_gc_config{tidb_cluster=\"$tidb_cluster\"}) by (type)", + "expr": "max(tidb_tikvclient_gc_config{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -11963,7 +11963,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tidb_tikvclient_gc_failure{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(increase(tidb_tikvclient_gc_failure{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -12051,7 +12051,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tidb_tikvclient_gc_unsafe_destroy_range_failures{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(increase(tidb_tikvclient_gc_unsafe_destroy_range_failures{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -12137,7 +12137,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tidb_tikvclient_gc_region_too_many_locks{tidb_cluster=\"$tidb_cluster\"}[1m]))", + "expr": "sum(increase(tidb_tikvclient_gc_region_too_many_locks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Locks Error OPM", @@ -12223,7 +12223,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tidb_tikvclient_gc_action_result{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(increase(tidb_tikvclient_gc_action_result{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -12320,7 +12320,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tidb_tikvclient_range_task_stats{tidb_cluster=\"$tidb_cluster\"}) by (type, result)", + "expr": "sum(tidb_tikvclient_range_task_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (type, result)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}-{{result}}", @@ -12413,7 +12413,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_range_task_push_duration_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_range_task_push_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -12516,7 +12516,7 @@ "steppedLine": false, "targets": [ { - "expr": "delta(tidb_tikvclient_batch_client_no_available_connection_total{tidb_cluster=\"$tidb_cluster\"}[30s])", + "expr": "delta(tidb_tikvclient_batch_client_no_available_connection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", @@ -12607,7 +12607,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_client_unavailable_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[30s])) by (le, instance))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_client_unavailable_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (le, instance))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -12701,7 +12701,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_batch_client_wait_connection_establish_bucket{tidb_cluster=\"$tidb_cluster\"}[30s])) by (le, instance))", + "expr": "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_batch_client_wait_connection_establish_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (le, instance))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", @@ -12802,7 +12802,7 @@ "targets": [ { "exemplar": true, - "expr": "rate(tidb_tikvclient_batch_recv_latency_sum{tidb_cluster=\"$tidb_cluster\"}[1m]) / rate(tidb_tikvclient_batch_recv_latency_count{tidb_cluster=\"$tidb_cluster\"}[1m])", + "expr": "rate(tidb_tikvclient_batch_recv_latency_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]) / rate(tidb_tikvclient_batch_recv_latency_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "hide": false, "interval": "", "legendFormat": "{{instance}}-{{result}}", @@ -12810,7 +12810,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_recv_latency_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_recv_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", "format": "time_series", "hide": true, "interval": "", @@ -12883,6 +12883,26 @@ "tags": [], "templating": { "list": [ + { + "allValue": null, + "current": {}, + "datasource": "${DS_TEST-CLUSTER}", + "hide": 2, + "includeAll": false, + "label": "K8s-cluster", + "multi": false, + "name": "k8s_cluster", + "options": [], + "query": "label_values(pd_cluster_status, k8s_cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": null, "current": {}, @@ -12893,7 +12913,7 @@ "multi": false, "name": "tidb_cluster", "options": [], - "query": "label_values(pd_cluster_status, tidb_cluster)", + "query": "label_values(pd_cluster_status{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", "refresh": 2, "regex": "", "sort": 1, diff --git a/metrics/grafana/tidb_runtime.json b/metrics/grafana/tidb_runtime.json index 63136f16d0b02..fefd9d2f5b15e 100644 --- a/metrics/grafana/tidb_runtime.json +++ b/metrics/grafana/tidb_runtime.json @@ -134,7 +134,7 @@ "steppedLine": false, "targets": [ { - "expr": "process_resident_memory_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -142,7 +142,7 @@ "refId": "A" }, { - "expr": "go_memstats_next_gc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} / (1 + tidb_server_gogc{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} / 100)", + "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} / (1 + tidb_server_gogc{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} / 100)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -150,7 +150,7 @@ "refId": "H" }, { - "expr": "go_memstats_heap_alloc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} - go_memstats_next_gc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} / (1 + tidb_server_gogc{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} / 100)", + "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} - go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} / (1 + tidb_server_gogc{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} / 100)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -158,7 +158,7 @@ "refId": "C" }, { - "expr": "go_memstats_heap_idle_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} - go_memstats_heap_released_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_heap_inuse_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} - go_memstats_heap_alloc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} - go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -166,7 +166,7 @@ "refId": "B" }, { - "expr": "go_memstats_stack_sys_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_mspan_sys_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_mcache_sys_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_buck_hash_sys_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_gc_sys_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_other_sys_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "go_memstats_stack_sys_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_mspan_sys_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_mcache_sys_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_buck_hash_sys_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_gc_sys_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} + go_memstats_other_sys_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -174,7 +174,7 @@ "refId": "D" }, { - "expr": "go_memstats_next_gc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -182,7 +182,7 @@ "refId": "E" }, { - "expr": "(clamp_max(idelta(go_memstats_last_gc_time_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]), 1) * go_memstats_next_gc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) > 0", + "expr": "(clamp_max(idelta(go_memstats_last_gc_time_seconds{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]), 1) * go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) > 0", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -287,7 +287,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(process_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])", + "expr": "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])", "format": "time_series", "hide": false, "interval": "", @@ -297,14 +297,14 @@ "step": 40 }, { - "expr": "(idelta((go_memstats_gc_cpu_fraction{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} * (go_memstats_last_gc_time_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} - process_start_time_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) * tidb_server_maxprocs{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})[30s:]) > 0) / 15", + "expr": "(idelta((go_memstats_gc_cpu_fraction{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} * (go_memstats_last_gc_time_seconds{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) * tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})[30s:]) > 0) / 15", "format": "time_series", "intervalFactor": 1, "legendFormat": "gc-cpu", "refId": "C" }, { - "expr": "tidb_server_maxprocs{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -405,7 +405,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_memstats_heap_objects{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "go_memstats_heap_objects{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -502,7 +502,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_gc_duration_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", quantile=\"0\"}", + "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", quantile=\"0\"}", "format": "time_series", "hide": false, "instant": false, @@ -512,7 +512,7 @@ "step": 40 }, { - "expr": "go_gc_duration_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", quantile!~\"0|1\"}", + "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", quantile!~\"0|1\"}", "format": "time_series", "instant": false, "intervalFactor": 1, @@ -520,7 +520,7 @@ "refId": "B" }, { - "expr": "go_gc_duration_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", quantile=\"1\"}", + "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", quantile=\"1\"}", "format": "time_series", "instant": false, "intervalFactor": 1, @@ -624,28 +624,28 @@ "steppedLine": false, "targets": [ { - "expr": "irate(go_memstats_alloc_bytes_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])", + "expr": "irate(go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])", "format": "time_series", "intervalFactor": 1, "legendFormat": "alloc", "refId": "A" }, { - "expr": "irate((go_memstats_alloc_bytes_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} - go_memstats_heap_alloc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})[30s:])", + "expr": "irate((go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})[30s:])", "format": "time_series", "intervalFactor": 1, "legendFormat": "sweep", "refId": "B" }, { - "expr": "irate(go_memstats_mallocs_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])", + "expr": "irate(go_memstats_mallocs_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])", "format": "time_series", "intervalFactor": 1, "legendFormat": "alloc-ops", "refId": "C" }, { - "expr": "irate(go_memstats_frees_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])", + "expr": "irate(go_memstats_frees_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])", "format": "time_series", "intervalFactor": 1, "legendFormat": "swepp-ops", @@ -739,14 +739,14 @@ "steppedLine": false, "targets": [ { - "expr": " go_goroutines{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": " go_goroutines{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "goroutines", "refId": "A" }, { - "expr": "go_threads{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "expr": "go_threads{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "threads", @@ -833,14 +833,14 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{tidb_cluster=\"$tidb_cluster\", store!=\"0\",instance=~\"$instance\"}[30s])) by (le, store))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", store!=\"0\",instance=~\"$instance\"}[30s])) by (le, store))", "format": "time_series", "intervalFactor": 1, "legendFormat": "tidb-to-store{{store}}", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type!=\"kv_gc\"}[30s])) by (le, instance))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", type!=\"kv_gc\"}[30s])) by (le, instance))", "format": "time_series", "intervalFactor": 1, "legendFormat": "tikv-{{instance}}-side", @@ -926,14 +926,14 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[30s])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[30s])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "tidb-side", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(pd_server_handle_tso_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[30s])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[30s])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "pd-side", @@ -1013,7 +1013,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum(increase(tidb_tikvclient_batch_requests_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le)", + "expr": "sum(increase(tidb_tikvclient_batch_requests_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le)", "format": "heatmap", "intervalFactor": 1, "legendFormat": "{{le}}", @@ -1079,7 +1079,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum(increase(tidb_tikvclient_batch_pending_requests_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le)", + "expr": "sum(increase(tidb_tikvclient_batch_pending_requests_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le)", "format": "heatmap", "intervalFactor": 1, "legendFormat": "{{le}}", @@ -1156,7 +1156,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_batch_wait_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le, instance))", + "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_batch_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le, instance))", "format": "time_series", "intervalFactor": 1, "legendFormat": "999", @@ -1164,7 +1164,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_wait_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le, instance))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le, instance))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99", @@ -1172,7 +1172,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_wait_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le, instance))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le, instance))", "format": "time_series", "intervalFactor": 1, "legendFormat": "95", @@ -1265,7 +1265,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_batch_send_latency_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le, instance))", + "expr": "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_batch_send_latency_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le, instance))", "format": "time_series", "intervalFactor": 1, "legendFormat": "9999", @@ -1273,7 +1273,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_batch_send_latency_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le, instance))", + "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_batch_send_latency_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le, instance))", "format": "time_series", "intervalFactor": 1, "legendFormat": "999", @@ -1281,7 +1281,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_send_latency_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le, instance))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_send_latency_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[30s])) by (le, instance))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99", @@ -1345,25 +1345,40 @@ "list": [ { "allValue": null, - "current": { - }, + "current": {}, + "datasource": "${DS_TEST-CLUSTER}", + "hide": 2, + "includeAll": false, + "label": "K8s-cluster", + "multi": false, + "name": "k8s_cluster", + "options": [], + "query": "label_values(pd_cluster_status, k8s_cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": false, "label": "tidb_cluster", "multi": false, "name": "tidb_cluster", - "options": [ - - ], - "query": "label_values(pd_cluster_status, tidb_cluster)", + "options": [], + "query": "label_values(pd_cluster_status{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", - "tags": [ - - ], + "tags": [], "tagsQuery": "", "type": "query", "useTags": false @@ -1372,14 +1387,14 @@ "allValue": null, "current": {}, "datasource": "${DS_TEST-CLUSTER}", - "definition": "label_values(process_start_time_seconds{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}, instance)", + "definition": "label_values(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}, instance)", "hide": 0, "includeAll": true, "label": "instance", "multi": false, "name": "instance", "options": [], - "query": "label_values(process_start_time_seconds{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}, instance)", + "query": "label_values(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}, instance)", "refresh": 1, "regex": "", "skipUrlSync": false, diff --git a/metrics/grafana/tidb_summary.json b/metrics/grafana/tidb_summary.json index 0d12628605f2e..6ce7a10087973 100644 --- a/metrics/grafana/tidb_summary.json +++ b/metrics/grafana/tidb_summary.json @@ -109,7 +109,7 @@ "steppedLine": false, "targets": [ { - "expr": "(time() - process_start_time_seconds{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"})", + "expr": "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"})", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -209,7 +209,7 @@ "steppedLine": false, "targets": [ { - "expr": "tidb_server_connections{tidb_cluster=\"$tidb_cluster\"}", + "expr": "tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -217,7 +217,7 @@ "step": 40 }, { - "expr": "sum(tidb_server_connections{tidb_cluster=\"$tidb_cluster\"})", + "expr": "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "total", @@ -317,7 +317,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}[1m])", + "expr": "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}[1m])", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -415,14 +415,14 @@ "steppedLine": false, "targets": [ { - "expr": "process_resident_memory_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", + "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "process-{{instance}}", "refId": "A" }, { - "expr": "go_memstats_heap_sys_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", + "expr": "go_memstats_heap_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -430,14 +430,14 @@ "refId": "B" }, { - "expr": "go_memstats_heap_inuse_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", + "expr": "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "HeapInuse-{{instance}}", "refId": "C" }, { - "expr": "go_memstats_heap_alloc_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", + "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -445,7 +445,7 @@ "refId": "D" }, { - "expr": "go_memstats_heap_idle_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", + "expr": "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -453,7 +453,7 @@ "refId": "E" }, { - "expr": "go_memstats_heap_released_bytes{tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", + "expr": "go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}", "hide": true, "interval": "", "legendFormat": "HeapReleased-{{instance}}", @@ -558,21 +558,21 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", sql_type!=\"internal\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", sql_type!=\"internal\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "B" }, { - "expr": "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", sql_type!=\"internal\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", sql_type!=\"internal\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95", "refId": "C" }, { - "expr": "sum(rate(tidb_server_handle_query_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", sql_type!=\"internal\"}[30s])) / sum(rate(tidb_server_handle_query_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", sql_type!=\"internal\"}[30s]))", + "expr": "sum(rate(tidb_server_handle_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", sql_type!=\"internal\"}[30s])) / sum(rate(tidb_server_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", sql_type!=\"internal\"}[30s]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -664,7 +664,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tidb_server_execute_error_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type, instance)", + "expr": "sum(increase(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": " {{type}}-{{instance}}", @@ -762,7 +762,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_server_query_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (result)", + "expr": "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (result)", "format": "time_series", "instant": false, "intervalFactor": 2, @@ -771,7 +771,7 @@ "step": 60 }, { - "expr": "sum(rate(tidb_server_query_total{tidb_cluster=\"$tidb_cluster\", result=\"OK\"}[1m] offset 1d))", + "expr": "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", result=\"OK\"}[1m] offset 1d))", "format": "time_series", "hide": true, "instant": false, @@ -781,7 +781,7 @@ "step": 90 }, { - "expr": "sum(tidb_server_connections{tidb_cluster=\"$tidb_cluster\"}) * sum(rate(tidb_server_handle_query_duration_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m])) / sum(rate(tidb_server_handle_query_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\"}[1m]))", + "expr": "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) * sum(rate(tidb_server_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) / sum(rate(tidb_server_handle_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "hide": true, "instant": false, @@ -888,7 +888,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_server_query_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", + "expr": "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} ", @@ -987,7 +987,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_executor_statement_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -995,7 +995,7 @@ "step": 30 }, { - "expr": "sum(rate(tidb_executor_statement_total{tidb_cluster=\"$tidb_cluster\"}[1m]))", + "expr": "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "total", @@ -1089,7 +1089,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_server_query_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": " {{type}}", @@ -1201,7 +1201,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_session_parse_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le))", "format": "time_series", "instant": false, "intervalFactor": 2, @@ -1210,7 +1210,7 @@ "step": 30 }, { - "expr": "histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "95", @@ -1309,7 +1309,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_session_compile_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_session_compile_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le))", "format": "time_series", "instant": false, "intervalFactor": 2, @@ -1318,7 +1318,7 @@ "step": 30 }, { - "expr": "histogram_quantile(0.95, sum(rate(tidb_session_compile_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_session_compile_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "95", @@ -1417,7 +1417,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_session_execute_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_session_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le))", "format": "time_series", "instant": false, "intervalFactor": 2, @@ -1426,7 +1426,7 @@ "step": 30 }, { - "expr": "histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "95", @@ -1524,7 +1524,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_server_plan_cache_total{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_server_plan_cache_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -1632,7 +1632,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_session_transaction_duration_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type, txn_mode)", + "expr": "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type, txn_mode)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}-{{txn_mode}}", @@ -1723,21 +1723,21 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le, txn_mode))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le, txn_mode))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99-{{txn_mode}}", "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le, txn_mode))", + "expr": "histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le, txn_mode))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95-{{txn_mode}}", "refId": "B" }, { - "expr": "histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le, txn_mode))", + "expr": "histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le, txn_mode))", "format": "time_series", "intervalFactor": 2, "legendFormat": "80-{{txn_mode}}", @@ -1825,7 +1825,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_session_transaction_statement_num_bucket{tidb_cluster=\"$tidb_cluster\"}[30s])) by (le))", + "expr": "histogram_quantile(1, sum(rate(tidb_session_transaction_statement_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "max", @@ -1917,7 +1917,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1.0, sum(rate(tidb_session_retry_num_bucket{tidb_cluster=\"$tidb_cluster\"}[30s])) by (le))", + "expr": "histogram_quantile(1.0, sum(rate(tidb_session_retry_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "max", @@ -2027,7 +2027,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type!=\"GC\"}[1m])) by (le, store))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type!=\"GC\"}[1m])) by (le, store))", "format": "time_series", "intervalFactor": 2, "legendFormat": "store-{{store}}", @@ -2122,7 +2122,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type!=\"GC\"}[1m])) by (le,type))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type!=\"GC\"}[1m])) by (le,type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -2215,7 +2215,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_request_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "expr": "sum(rate(tidb_tikvclient_request_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -2309,7 +2309,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", + "expr": "sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2405,7 +2405,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_tikvclient_txn_write_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(1, sum(rate(tidb_tikvclient_txn_write_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2500,7 +2500,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_tikvclient_txn_write_kv_num_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(1, sum(rate(tidb_tikvclient_txn_write_kv_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2594,7 +2594,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(1, sum(rate(tidb_tikvclient_txn_regions_num_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(1, sum(rate(tidb_tikvclient_txn_regions_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2686,7 +2686,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", + "expr": "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "999", @@ -2694,14 +2694,14 @@ "step": 10 }, { - "expr": "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "B" }, { - "expr": "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "90", @@ -2791,7 +2791,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", + "expr": "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "999", @@ -2799,14 +2799,14 @@ "step": 10 }, { - "expr": "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "B" }, { - "expr": "histogram_quantile(0.90, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "90", @@ -2895,7 +2895,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_autoid_operation_duration_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", + "expr": "sum(rate(tidb_autoid_operation_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -2984,14 +2984,14 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tidb_autoid_operation_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tidb_autoid_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "B" }, { - "expr": "sum(rate(tidb_autoid_operation_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\"}[1m])) / sum(rate(tidb_autoid_operation_duration_seconds_count{tidb_cluster=\"$tidb_cluster\"}[1m]))", + "expr": "sum(rate(tidb_autoid_operation_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) / sum(rate(tidb_autoid_operation_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", @@ -3051,6 +3051,31 @@ "tags": [], "templating": { "list": [ + { + "allValue": null, + "current": { + }, + "datasource": "${DS_TEST-CLUSTER}", + "hide": 2, + "includeAll": false, + "label": "K8s-cluster", + "multi": false, + "name": "k8s_cluster", + "options": [ + + ], + "query": "label_values(pd_cluster_status, k8s_cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": null, "current": { @@ -3064,7 +3089,7 @@ "options": [ ], - "query": "label_values(pd_cluster_status, tidb_cluster)", + "query": "label_values(pd_cluster_status{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", "refresh": 2, "regex": "", "sort": 1,