diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json index c405e0347ea..b6f84a1bd1d 100644 --- a/metrics/grafana/pd.json +++ b/metrics/grafana/pd.json @@ -926,915 +926,1319 @@ "y": 13 }, "id": 118, - "panels": [ - { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": "${DS_TEST-CLUSTER}", - "fontSize": "90%", - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 14 - }, - "hideTimeOverride": true, - "id": 116, - "links": [], - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 1, - "desc": true - }, - "styles": [ - { - "alias": "Option", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Metric", - "preserveFormat": false, - "type": "string" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "expr": "pd_config_status", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "timeFrom": "1s", - "title": "PD scheduler config", - "transform": "timeseries_aggregations", - "type": "table" - }, - { - "cacheTimeout": null, - "columns": [], - "datasource": "${DS_TEST-CLUSTER}", - "fontSize": "100%", - "gridPos": { - "h": 3, - "w": 7, - "x": 6, - "y": 14 - }, - "hideTimeOverride": true, - "id": 139, - "links": [], - "pageSize": null, - "scroll": false, - "showHeader": true, - "sort": { - "col": 1, - "desc": false - }, - "styles": [ - { - "alias": "ID", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "mappingType": 1, - "pattern": "Metric", - "preserveFormat": false, - "sanitize": false, - "thresholds": [], - "type": "string", - "unit": "short", - "valueMaps": [ - { - "text": "", - "value": "" - } - ] - } - ], - "targets": [ - { - "expr": "pd_cluster_metadata{instance=\"$instance\"}", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "timeFrom": "1s", - "timeShift": null, - "title": "Cluster ID", - "transform": "timeseries_aggregations", - "type": "table" - }, + "panels": [], + "repeat": null, + "title": "Cluster", + "type": "row" + }, + { + "columns": [ { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": "${DS_TEST-CLUSTER}", - "fontSize": "100%", - "gridPos": { - "h": 7, - "w": 5, - "x": 13, - "y": 14 - }, - "hideTimeOverride": true, - "id": 103, - "links": [], - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": false - }, - "styles": [ - { - "alias": "Type", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "type", - "type": "date" - }, - { - "alias": "Numbers", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "Current", - "thresholds": [], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "expr": "pd_regions_label_level{instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "timeFrom": "1s", - "timeShift": null, - "title": "Region label isolation level", - "transform": "timeseries_aggregations", - "type": "table" - }, + "text": "Current", + "value": "current" + } + ], + "datasource": "${DS_TEST-CLUSTER}", + "fontSize": "90%", + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 14 + }, + "hideTimeOverride": true, + "id": 116, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": "${DS_TEST-CLUSTER}", - "fontSize": "100%", - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 14 - }, - "hideTimeOverride": true, - "id": 117, - "links": [], - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true - }, - "styles": [ - { - "alias": "Label : address", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "link": false, - "linkUrl": "", - "pattern": "Metric", - "thresholds": [ - "un" - ], - "type": "string" - }, - { - "alias": "count number", - "colorMode": null, - "colors": [ - "#bf1b00", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "pattern": "Current", - "thresholds": [ - "0" - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "expr": "pd_cluster_placement_status", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A" - } - ], - "timeFrom": "1s", - "title": "Label distribution", - "transform": "timeseries_aggregations", - "type": "table" + "alias": "Option", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Metric", + "preserveFormat": false, + "type": "string" }, { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, + "alias": "", + "colorMode": null, "colors": [ - "#299c46", + "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_TEST-CLUSTER}", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 2, - "w": 7, - "x": 6, - "y": 17 - }, - "hideTimeOverride": true, - "id": 137, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pd_cluster_tso{instance=\"$instance\"}", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Current TSO", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } + "rgba(50, 172, 45, 0.97)" ], - "valueName": "avg" - }, + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "pd_config_status", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "timeFrom": "1s", + "title": "PD scheduler config", + "transform": "timeseries_aggregations", + "type": "table" + }, + { + "cacheTimeout": null, + "columns": [], + "datasource": "${DS_TEST-CLUSTER}", + "fontSize": "100%", + "gridPos": { + "h": 3, + "w": 7, + "x": 6, + "y": 14 + }, + "hideTimeOverride": true, + "id": 139, + "links": [], + "pageSize": null, + "scroll": false, + "showHeader": true, + "sort": { + "col": 1, + "desc": false + }, + "styles": [ { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, + "alias": "ID", + "colorMode": null, "colors": [ - "#299c46", + "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", - "#d44a3a" + "rgba(50, 172, 45, 0.97)" ], - "datasource": "${DS_TEST-CLUSTER}", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 2, - "w": 7, - "x": 6, - "y": 19 - }, - "hideTimeOverride": true, - "id": 115, - "interval": null, - "links": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pd_cluster_id{instance=\"$instance\"}", - "format": "time_series", - "hide": false, - "instant": true, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": "1s", - "title": "Current ID allocation", - "type": "singlestat", - "valueFontSize": "80%", + "pattern": "Metric", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short", "valueMaps": [ { - "op": "=", - "text": "N/A", - "value": "null" + "text": "", + "value": "" } - ], - "valueName": "avg" + ] } ], - "repeat": null, - "title": "Cluster", - "type": "row" + "targets": [ + { + "expr": "pd_cluster_metadata{instance=\"$instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "timeFrom": "1s", + "timeShift": null, + "title": "Cluster ID", + "transform": "timeseries_aggregations", + "type": "table" }, { - "collapsed": true, + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "${DS_TEST-CLUSTER}", + "fontSize": "100%", "gridPos": { - "h": 1, - "w": 24, - "x": 0, + "h": 7, + "w": 5, + "x": 13, "y": 14 }, - "id": 119, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "hideTimeOverride": true, + "id": 103, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "styles": [ + { + "alias": "Type", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "type", + "type": "date" + }, + { + "alias": "Numbers", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, - "description": "The number of different operators that are newly created", - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 22 - }, - "id": 45, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(delta(pd_schedule_operators_count{instance=\"$instance\", event=\"create\"}[1m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - } + "pattern": "Current", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" ], + "decimals": 2, + "pattern": "/.*/", "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Schedule operator create", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "pd_regions_label_level{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "timeFrom": "1s", + "timeShift": null, + "title": "Region label isolation level", + "transform": "timeseries_aggregations", + "type": "table" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "${DS_TEST-CLUSTER}", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 14 + }, + "hideTimeOverride": true, + "id": 117, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Label : address", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "link": false, + "linkUrl": "", + "pattern": "Metric", + "thresholds": [ + "un" ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "string" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 2, - "description": "The number of different operators that have been checked. It mainly checks if the current step is finished; if yes, it returns the next step to be executed.", - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 22 - }, - "id": 79, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(delta(pd_schedule_operators_count{instance=\"$instance\", event=\"check\"}[1m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - } + "alias": "count number", + "colorMode": null, + "colors": [ + "#bf1b00", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Schedule operator check", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Current", + "thresholds": [ + "0" ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "number", + "unit": "short" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], "decimals": 2, - "description": "The number of different operators that are finished", - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 29 - }, - "id": 77, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(delta(pd_schedule_operators_count{instance=\"$instance\", event=\"finish\"}[1m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - } - ], + "pattern": "/.*/", "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Schedule operator finish", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, + "type": "number", + "unit": "short" + } + ], + "targets": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 2, - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 29 - }, - "id": 78, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(delta(pd_schedule_operators_count{instance=\"$instance\", event=\"timeout\"}[1m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Schedule operator timeout", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "opm", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "expr": "pd_cluster_placement_status", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "timeFrom": "1s", + "title": "Label distribution", + "transform": "timeseries_aggregations", + "type": "table" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_TEST-CLUSTER}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 7, + "x": 6, + "y": 17 + }, + "hideTimeOverride": true, + "id": 137, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 2, - "description": "The number of different operators that are replaced or canceled", - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 36 - }, - "id": 80, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(delta(pd_schedule_operators_count{instance=\"$instance\", event=\"cancel\"}[1m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - }, - { - "expr": "sum(delta(pd_schedule_operators_count{instance=\"$instance\", event=\"replace\"}[1m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pd_cluster_tso{instance=\"$instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current TSO", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_TEST-CLUSTER}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 7, + "x": 6, + "y": 19 + }, + "hideTimeOverride": true, + "id": 115, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pd_cluster_id{instance=\"$instance\"}", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": "1s", + "title": "Current ID allocation", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 119, + "panels": [], + "repeat": null, + "title": "Operator", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, + "description": "The number of different operators that are newly created", + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 45, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(delta(pd_schedule_operators_count{instance=\"$instance\", event=\"create\"}[1m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Schedule operator create", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, + "description": "The number of different operators that have been checked. It mainly checks if the current step is finished; if yes, it returns the next step to be executed.", + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 79, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(delta(pd_schedule_operators_count{instance=\"$instance\", event=\"check\"}[1m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Schedule operator check", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, + "description": "The number of different operators that are finished", + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 77, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(delta(pd_schedule_operators_count{instance=\"$instance\", event=\"finish\"}[1m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Schedule operator finish", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 78, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(delta(pd_schedule_operators_count{instance=\"$instance\", event=\"timeout\"}[1m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Schedule operator timeout", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, + "description": "The number of different operators that are replaced or canceled", + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 80, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(delta(pd_schedule_operators_count{instance=\"$instance\", event=\"cancel\"}[1m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(delta(pd_schedule_operators_count{instance=\"$instance\", event=\"replace\"}[1m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Schedule operator replaced or canceled", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "opm", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The number of operators in different status", + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 47, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(delta(pd_schedule_operators_count{instance=\"$instance\"}[1m])) by (event)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{event}}", + "metric": "pd_scheduler_status", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Schedule operators count by state", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The time consumed when the operator is finished in .99", + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 67, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket[5m])) by (type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}-99%", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket[5m])) by (type, le))", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{type}}-95%", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Operator finish duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The time consumed when the operator step is finished in .99", + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 81, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket[5m])) by (type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}-99%", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket[5m])) by (type, le))", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{type}}-95%", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Operator step finish duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 120, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, + "description": "The capacity size of each TiKV instance", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 83, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"store_capacity\"}", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{address}}-store-{{store}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Schedule operator replaced or canceled", + "title": "Store capacity", "tooltip": { "shared": true, "sort": 0, @@ -1850,7 +2254,101 @@ }, "yaxes": [ { - "format": "opm", + "decimals": null, + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, + "description": "The available capacity size of each TiKV instance", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 91, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "{store=~\"$store\", instance=\"$instance\", type=\"store_available\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{address}}-store-{{store}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Store available", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "decbytes", "label": null, "logBase": 1, "max": null, @@ -1877,22 +2375,23 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of operators in different status", + "decimals": 2, + "description": "The used capacity size of each TiKV instance", "fill": 0, "gridPos": { - "h": 7, + "h": 6, "w": 12, - "x": 12, - "y": 36 + "x": 0, + "y": 29 }, - "id": 47, + "id": 90, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, - "max": true, + "max": false, "min": false, "rightSide": true, "show": true, @@ -1915,20 +2414,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(pd_schedule_operators_count{instance=\"$instance\"}[1m])) by (event)", + "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"store_used\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{event}}", - "metric": "pd_scheduler_status", - "refId": "A", - "step": 4 + "legendFormat": "{{address}}-store-{{store}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Schedule operators count by state", + "title": "Store used", "tooltip": { "shared": true, "sort": 0, @@ -1944,7 +2441,8 @@ }, "yaxes": [ { - "format": "short", + "decimals": 2, + "format": "decbytes", "label": null, "logBase": 1, "max": null, @@ -1971,22 +2469,23 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The time consumed when the operator is finished in .99", + "decimals": 2, + "description": "It is equal to Store available capacity size over Store capacity size for each TiKV instance", "fill": 0, "gridPos": { - "h": 7, + "h": 6, "w": 12, - "x": 0, - "y": 43 + "x": 12, + "y": 29 }, - "id": 67, + "id": 84, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, - "max": true, + "max": false, "min": false, "rightSide": true, "show": true, @@ -1996,7 +2495,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, @@ -2008,27 +2507,112 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket[5m])) by (type, le))", + "expr": "sum(pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"store_available\"}) by (address, store) / sum(pd_scheduler_store_status{instance=\"$instance\", type=\"store_capacity\"}) by (address, store)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}-99%", - "refId": "A", - "step": 4 + "legendFormat": "{{address}}-store-{{store}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Store available ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "expr": "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket[5m])) by (type, le))", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, + "description": "The size amplification, which is equal to Store Region size over Store used capacity size, of each TiKV instance", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 85, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 3, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"region_size\"}) by (address, store) / sum(pd_scheduler_store_status{instance=\"$instance\", type=\"store_used\"}) by (address, store) * 2^20", "format": "time_series", - "hide": true, + "instant": false, "intervalFactor": 2, - "legendFormat": "{{type}}-95%", - "refId": "B" + "legendFormat": "{{address}}-store-{{store}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Operator finish duration", + "title": "Size amplification", "tooltip": { "shared": true, "sort": 0, @@ -2044,7 +2628,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -2071,15 +2655,19 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The time consumed when the operator step is finished in .99", + "decimals": null, + "description": "The Region score of each TiKV instance", + "editable": true, + "error": false, "fill": 0, + "grid": {}, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 12, - "y": 43 + "y": 35 }, - "id": 81, + "id": 41, "legend": { "alignAsTable": true, "avg": false, @@ -2090,7 +2678,8 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -2109,30 +2698,49 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket[5m])) by (type, le))", + "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"region_score\"}", "format": "time_series", + "hide": false, "intervalFactor": 2, - "legendFormat": "{{type}}-99%", - "refId": "A" + "legendFormat": "{{address}}-store-{{store}}", + "refId": "A", + "step": 4 }, { - "expr": "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket[5m])) by (type, le))", + "expr": "pd_scheduler_op_influence{instance=\"$instance\",scheduler=\"balance-region-scheduler\"}", "format": "time_series", "hide": true, - "intervalFactor": 2, - "legendFormat": "{{type}}-95%", - "refId": "B" + "intervalFactor": 1, + "legendFormat": "op-influence-{{store}}-{{type}}", + "refId": "C" + }, + { + "expr": "pd_scheduler_tolerant_resource{instance=\"$instance\",scheduler=\"balance-region-scheduler\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "tolerant-resource-{{source}}-{{target}}", + "refId": "E" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1000000000 } ], - "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Operator step finish duration", + "title": "Store Region score", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -2144,7 +2752,7 @@ }, "yaxes": [ { - "format": "s", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -2164,51 +2772,38 @@ "align": false, "alignLevel": null } - } - ], - "repeat": null, - "title": "Operator", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 120, - "panels": [ + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 2, - "description": "The capacity size of each TiKV instance", + "decimals": null, + "description": "The leader score of each TiKV instance", + "editable": true, + "error": false, "fill": 0, + "grid": {}, "gridPos": { "h": 6, "w": 12, "x": 0, - "y": 23 + "y": 41 }, - "id": 83, + "id": 40, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": null, - "sortDesc": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -2227,24 +2822,32 @@ "steppedLine": false, "targets": [ { - "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"store_capacity\"}", + "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"leader_score\"}", "format": "time_series", - "hide": false, - "instant": false, "intervalFactor": 2, "legendFormat": "{{address}}-store-{{store}}", - "refId": "B" + "refId": "A", + "step": 4 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1000000000 } ], - "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Store capacity", + "title": "Store leader score", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -2256,16 +2859,15 @@ }, "yaxes": [ { - "decimals": null, - "format": "decbytes", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "percent", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -2285,25 +2887,27 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 2, - "description": "The available capacity size of each TiKV instance", + "description": "The total Region size of each TiKV instance", "fill": 0, "gridPos": { "h": 6, "w": 12, "x": 12, - "y": 23 + "y": 41 }, - "id": 91, + "id": 57, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -2322,18 +2926,19 @@ "steppedLine": false, "targets": [ { - "expr": "{store=~\"$store\", instance=\"$instance\", type=\"store_available\"}", + "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"region_size\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{address}}-store-{{store}}", - "refId": "A" + "refId": "A", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Store available", + "title": "Store Region size", "tooltip": { "shared": true, "sort": 0, @@ -2350,7 +2955,7 @@ "yaxes": [ { "decimals": null, - "format": "decbytes", + "format": "decmbytes", "label": null, "logBase": 1, "max": null, @@ -2363,7 +2968,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -2378,25 +2983,26 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 2, - "description": "The used capacity size of each TiKV instance", + "description": " \tThe total leader size of each TiKV instance", "fill": 0, "gridPos": { "h": 6, "w": 12, "x": 0, - "y": 29 + "y": 47 }, - "id": 90, + "id": 56, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, + "sort": "current", "sortDesc": true, "total": false, "values": true @@ -2416,18 +3022,19 @@ "steppedLine": false, "targets": [ { - "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"store_used\"}", + "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"leader_size\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{address}}-store-{{store}}", - "refId": "A" + "refId": "A", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Store used", + "title": "Store leader size", "tooltip": { "shared": true, "sort": 0, @@ -2443,8 +3050,7 @@ }, "yaxes": [ { - "decimals": 2, - "format": "decbytes", + "format": "decmbytes", "label": null, "logBase": 1, "max": null, @@ -2457,7 +3063,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -2471,26 +3077,28 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 2, - "description": "It is equal to Store available capacity size over Store capacity size for each TiKV instance", + "decimals": 3, + "description": "The Region count of each TiKV instance \t", "fill": 0, "gridPos": { "h": 6, "w": 12, "x": 12, - "y": 29 + "y": 47 }, - "id": 84, + "id": 59, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -2509,18 +3117,19 @@ "steppedLine": false, "targets": [ { - "expr": "sum(pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"store_available\"}) by (address, store) / sum(pd_scheduler_store_status{instance=\"$instance\", type=\"store_capacity\"}) by (address, store)", + "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"region_count\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{address}}-store-{{store}}", - "refId": "A" + "refId": "A", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Store available ratio", + "title": "Store Region count", "tooltip": { "shared": true, "sort": 0, @@ -2536,8 +3145,7 @@ }, "yaxes": [ { - "decimals": 2, - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -2550,7 +3158,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -2564,31 +3172,32 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 2, - "description": "The size amplification, which is equal to Store Region size over Store used capacity size, of each TiKV instance", + "decimals": 3, + "description": "The leader count of each TiKV instance", "fill": 0, "gridPos": { "h": 6, "w": 12, "x": 0, - "y": 35 + "y": 53 }, - "id": 85, + "id": 58, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, + "sortDesc": true, "total": false, "values": true }, "lines": true, - "linewidth": 3, + "linewidth": 2, "links": [], "nullPointMode": "null", "paceLength": 10, @@ -2602,19 +3211,19 @@ "steppedLine": false, "targets": [ { - "expr": "sum(pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"region_size\"}) by (address, store) / sum(pd_scheduler_store_status{instance=\"$instance\", type=\"store_used\"}) by (address, store) * 2^20", + "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"leader_count\"}", "format": "time_series", - "instant": false, "intervalFactor": 2, "legendFormat": "{{address}}-store-{{store}}", - "refId": "A" + "refId": "A", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Size amplification", + "title": "Store leader count", "tooltip": { "shared": true, "sort": 0, @@ -2643,7 +3252,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -2657,30 +3266,26 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": null, - "description": "The Region score of each TiKV instance", - "editable": true, - "error": false, + "decimals": 2, + "description": "", "fill": 0, - "grid": {}, "gridPos": { "h": 6, "w": 12, "x": 12, - "y": 35 + "y": 53 }, - "id": 41, + "id": 129, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, - "max": true, + "max": false, "min": false, "rightSide": true, "show": true, - "sort": "current", "sortDesc": true, "total": false, "values": true @@ -2694,55 +3299,54 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/move-store-.*/", + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"region_score\"}", + "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"store_used\"}", "format": "time_series", - "hide": false, "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 + "legendFormat": "store-{{store}}", + "refId": "A" }, { - "expr": "pd_scheduler_op_influence{instance=\"$instance\",scheduler=\"balance-region-scheduler\"}", + "expr": "1048576*pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"region_size\"}", "format": "time_series", - "hide": true, "intervalFactor": 1, - "legendFormat": "op-influence-{{store}}-{{type}}", - "refId": "C" + "legendFormat": "store-{{store}}", + "refId": "B" }, { - "expr": "pd_scheduler_tolerant_resource{instance=\"$instance\",scheduler=\"balance-region-scheduler\"}", + "expr": "-sum(delta(pd_scheduler_balance_region{store=~\"$store\", address=~\".*out\",instance=\"$instance\", type=\"move-peer\"}[1m])) by (address, store)", "format": "time_series", - "hide": true, "intervalFactor": 1, - "legendFormat": "tolerant-resource-{{source}}-{{target}}", - "refId": "E" - } - ], - "thresholds": [ + "legendFormat": "move-store-{{store}}", + "refId": "C" + }, { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 1000000000 + "expr": "sum(delta(pd_scheduler_balance_region{store=~\"$store\", address=~\".*in\",instance=\"$instance\", type=\"move-peer\"}[1m])) by (address, store)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "move-store-{{store}}", + "refId": "D" } ], + "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Store Region score", + "title": "Store used & Region Size & Move Peer", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -2754,7 +3358,7 @@ }, "yaxes": [ { - "format": "none", + "format": "decbytes", "label": null, "logBase": 1, "max": null, @@ -2762,8 +3366,9 @@ "show": true }, { + "decimals": null, "format": "short", - "label": null, + "label": "", "logBase": 1, "max": null, "min": null, @@ -2774,1014 +3379,991 @@ "align": false, "alignLevel": null } + } + ], + "repeat": null, + "title": "Statistics - balance", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 135, + "panels": [], + "title": "Statistics - hot write", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 0, + "description": "The total number of leader Regions under hot write on each TiKV instance", + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 52 + }, + "id": 50, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pd_hotspot_status{store=~\"$store\", instance=\"$instance\", type=\"hot_write_region_as_leader\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{address}}-store-{{store}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Hot Region's leader distribution", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, + "description": "The total bytes of hot write on leader Regions for each TiKV instance", + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 52 + }, + "id": 48, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pd_hotspot_status{store=~\"$store\", instance=\"$instance\", type=\"total_written_bytes_as_leader\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{address}}-store-{{store}}", + "metric": "pd_hotspot_status", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total written bytes on hot leader Regions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 0, + "description": "The total number of Regions which are not leader under hot write on each TiKV instance", + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 51, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pd_hotspot_status{store=~\"$store\", instance=\"$instance\", type=\"hot_write_region_as_peer\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{address}}-store-{{store}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Hot Region's peer distribution", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": null, - "description": "The leader score of each TiKV instance", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 41 - }, - "id": 40, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"leader_score\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 1000000000 - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Store leader score", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, + "description": "The total bytes of hot write on Regions which are not leader for each TiKV instance", + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 49, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pd_hotspot_status{store=~\"$store\", instance=\"$instance\", type=\"total_written_bytes_as_peer\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{address}}-store-{{store}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total written bytes on hot peer Regions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 2, - "description": "The total Region size of each TiKV instance", - "fill": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 41 - }, - "id": 57, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"region_size\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Store Region size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "decmbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 0, + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 61, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pd_scheduler_store_status{store=~\"$store\", type=\"store_write_rate_bytes\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{address}}-store-{{store}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Store write rate bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 2, - "description": " \tThe total leader size of each TiKV instance", - "fill": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 47 - }, - "id": 56, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"leader_size\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Store leader size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decmbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 0, + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 66 + }, + "id": 143, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pd_scheduler_store_status{store=~\"$store\", type=\"store_write_rate_bytes\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{address}}-store-{{store}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Store write rate bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 3, - "description": "The Region count of each TiKV instance \t", - "fill": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 47 - }, - "id": 59, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"region_count\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Store Region count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 0, + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 73 + }, + "id": 146, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pd_hotcache_status{name=\"total_length\", store=~\"store-$store\", type=\"write\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{store}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Hot cache Write entry numbers", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 3, - "description": "The leader count of each TiKV instance", - "fill": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 53 - }, - "id": 58, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"leader_count\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Store leader count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 0, + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 73 + }, + "id": 106, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(pd_scheduler_hot_region{store=~\"$store\", type=~\".*store.*\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}-{{store}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Selector events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 2, - "description": "", - "fill": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 53 - }, - "id": 129, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/move-store-.*/", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"store_used\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "store-{{store}}", - "refId": "A" - }, - { - "expr": "1048576*pd_scheduler_store_status{store=~\"$store\", instance=\"$instance\", type=\"region_size\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "store-{{store}}", - "refId": "B" - }, - { - "expr": "-sum(delta(pd_scheduler_balance_region{store=~\"$store\", address=~\".*out\",instance=\"$instance\", type=\"move-peer\"}[1m])) by (address, store)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "move-store-{{store}}", - "refId": "C" - }, - { - "expr": "sum(delta(pd_scheduler_balance_region{store=~\"$store\", address=~\".*in\",instance=\"$instance\", type=\"move-peer\"}[1m])) by (address, store)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "move-store-{{store}}", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Store used & Region Size & Move Peer", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "decimals": null, - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } ], - "repeat": null, - "title": "Statistics - balance", - "type": "row" + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "collapsed": true, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 0, + "fill": 0, "gridPos": { - "h": 1, - "w": 24, + "h": 7, + "w": 12, "x": 0, - "y": 16 + "y": 80 }, - "id": 121, - "panels": [ + "id": 145, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 0, - "description": "The total number of leader Regions under hot read on each TiKV instance", - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 60, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pd_hotspot_status{store=~\"$store\", instance=\"$instance\", type=\"hot_read_region_as_leader\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Hot Region's leader distribution", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "expr": "- rate(pd_scheduler_hot_region{store=~\"$store-out\", type=\"move-leader\"}[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{type}}-{{store}}", + "refId": "A", + "step": 4 }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 2, - "description": "The total bytes of hot read on leader Regions for each TiKV instance", - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 62, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pd_hotspot_status{store=~\"$store\", instance=\"$instance\", type=\"total_read_bytes_as_leader\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "metric": "pd_hotspot_status", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Total read bytes on hot leader Regions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "expr": "rate(pd_scheduler_hot_region{store=~\"$store-in\", type=\"move-leader\"}[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{type}}-{{store}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Direction of hotspot move leader", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 0, - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 31 - }, - "id": 105, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pd_scheduler_store_status{store=~\"$store\", type=\"store_read_rate_bytes\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Store read rate bytes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 0, + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 80 + }, + "id": 144, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "- rate(pd_scheduler_hot_region{store=~\"$store-out\", type=\"move-peer\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}-{{store}}", + "refId": "A", + "step": 4 }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 0, - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 31 - }, - "id": 107, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pd_scheduler_store_status{store=~\"$store\", type=\"store_read_rate_keys\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Store read rate keys", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "expr": "rate(pd_scheduler_hot_region{store=~\"$store-in\", type=\"move-peer\"}[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{type}}-{{store}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Direction of hotspot move peer", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } ], - "repeat": null, - "title": "Statistics - hot read", - "type": "row" + "yaxis": { + "align": false, + "alignLevel": null + } }, { "collapsed": true, @@ -3789,9 +4371,9 @@ "h": 1, "w": 24, "x": 0, - "y": 17 + "y": 87 }, - "id": 135, + "id": 121, "panels": [ { "aliasColors": {}, @@ -3800,15 +4382,15 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 0, - "description": "The total number of leader Regions under hot write on each TiKV instance", + "description": "The total number of leader Regions under hot read on each TiKV instance", "fill": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 89 + "y": 46 }, - "id": 50, + "id": 60, "legend": { "alignAsTable": true, "avg": false, @@ -3825,7 +4407,7 @@ "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, @@ -3839,7 +4421,7 @@ "steppedLine": false, "targets": [ { - "expr": "pd_hotspot_status{store=~\"$store\", instance=\"$instance\", type=\"hot_write_region_as_leader\"}", + "expr": "pd_hotspot_status{store=~\"$store\", instance=\"$instance\", type=\"hot_read_region_as_leader\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{address}}-store-{{store}}", @@ -3896,15 +4478,15 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 2, - "description": "The total bytes of hot write on leader Regions for each TiKV instance", + "description": "The total bytes of hot read on leader Regions for each TiKV instance", "fill": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 89 + "y": 46 }, - "id": 48, + "id": 62, "legend": { "alignAsTable": true, "avg": false, @@ -3926,7 +4508,7 @@ "nullPointMode": "null", "paceLength": 10, "percentage": false, - "pointradius": 1, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -3935,7 +4517,7 @@ "steppedLine": false, "targets": [ { - "expr": "pd_hotspot_status{store=~\"$store\", instance=\"$instance\", type=\"total_written_bytes_as_leader\"}", + "expr": "pd_hotspot_status{store=~\"$store\", instance=\"$instance\", type=\"total_read_bytes_as_leader\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{address}}-store-{{store}}", @@ -3948,7 +4530,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Total written bytes on hot leader Regions", + "title": "Total read bytes on hot leader Regions", "tooltip": { "shared": true, "sort": 0, @@ -3992,111 +4574,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 0, - "description": "The total number of Regions which are not leader under hot write on each TiKV instance", "fill": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 96 - }, - "id": 51, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pd_hotspot_status{store=~\"$store\", instance=\"$instance\", type=\"hot_write_region_as_peer\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Hot Region's peer distribution", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 2, - "description": "The total bytes of hot write on Regions which are not leader for each TiKV instance", - "fill": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 96 + "y": 53 }, - "id": 49, + "id": 105, "legend": { "alignAsTable": true, "avg": false, @@ -4113,7 +4598,7 @@ "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, @@ -4127,7 +4612,7 @@ "steppedLine": false, "targets": [ { - "expr": "pd_hotspot_status{store=~\"$store\", instance=\"$instance\", type=\"total_written_bytes_as_peer\"}", + "expr": "pd_scheduler_store_status{store=~\"$store\", type=\"store_read_rate_bytes\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{address}}-store-{{store}}", @@ -4139,7 +4624,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Total written bytes on hot peer Regions", + "title": "Store read rate bytes", "tooltip": { "shared": true, "sort": 0, @@ -4155,7 +4640,7 @@ }, "yaxes": [ { - "format": "decbytes", + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -4187,10 +4672,10 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 103 + "x": 12, + "y": 53 }, - "id": 61, + "id": 107, "legend": { "alignAsTable": true, "avg": false, @@ -4221,7 +4706,7 @@ "steppedLine": false, "targets": [ { - "expr": "pd_scheduler_store_status{store=~\"$store\", type=\"store_write_rate_bytes\"}", + "expr": "pd_scheduler_store_status{store=~\"$store\", type=\"store_read_rate_keys\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{address}}-store-{{store}}", @@ -4233,7 +4718,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Store write rate bytes", + "title": "Store read rate keys", "tooltip": { "shared": true, "sort": 0, @@ -4282,17 +4767,17 @@ "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 103 + "x": 0, + "y": 60 }, - "id": 106, + "id": 147, "legend": { "alignAsTable": true, - "avg": false, + "avg": true, "current": true, "hideEmpty": true, "hideZero": true, - "max": true, + "max": false, "min": false, "rightSide": true, "show": true, @@ -4316,10 +4801,10 @@ "steppedLine": false, "targets": [ { - "expr": "pd_scheduler_store_status{store=~\"$store\", type=\"store_write_rate_keys\"}", + "expr": "pd_hotcache_status{name=\"total_length\", store=~\"store-$store\", type=\"read\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", + "legendFormat": "{{store}}", "refId": "A", "step": 4 } @@ -4328,7 +4813,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Store write rate keys", + "title": "Hot cache Read entry numbers", "tooltip": { "shared": true, "sort": 0, @@ -4345,7 +4830,7 @@ "yaxes": [ { "decimals": null, - "format": "Bps", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -4367,7 +4852,8 @@ } } ], - "title": "Statistics - hot write", + "repeat": null, + "title": "Statistics - hot read", "type": "row" }, { @@ -4376,7 +4862,7 @@ "h": 1, "w": 24, "x": 0, - "y": 18 + "y": 88 }, "id": 122, "panels": [ @@ -4392,7 +4878,7 @@ "h": 8, "w": 24, "x": 0, - "y": 26 + "y": 47 }, "id": 46, "legend": { @@ -4486,7 +4972,7 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 55 }, "id": 87, "legend": { @@ -4515,7 +5001,7 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { @@ -4587,7 +5073,7 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 55 }, "hideTimeOverride": false, "id": 86, @@ -4617,7 +5103,7 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { @@ -4688,7 +5174,7 @@ "h": 8, "w": 12, "x": 0, - "y": 42 + "y": 63 }, "id": 89, "legend": { @@ -4779,7 +5265,7 @@ "h": 8, "w": 12, "x": 12, - "y": 42 + "y": 63 }, "id": 88, "legend": { @@ -4811,7 +5297,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(pd_scheduler_balance_region{store=~\"$store\", instance=\"$instance\", type!=\"move-peer\"}[30s])) by (type, address, store)", + "expr": "sum(delta(pd_scheduler_balance_region{store=~\"$store\", instance=\"$instance\"}[30s])) by (type, address, store)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}-store-{{store}}", @@ -4871,7 +5357,7 @@ "h": 8, "w": 12, "x": 0, - "y": 50 + "y": 71 }, "id": 52, "legend": { @@ -4966,9 +5452,103 @@ "h": 8, "w": 12, "x": 12, - "y": 50 + "y": 71 + }, + "id": 53, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(delta(pd_scheduler_event_count{instance=\"$instance\", type=\"balance-region-scheduler\"}[5m])) by (name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "pd_scheduler_event_count", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Balance Region scheduler", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 79 }, - "id": 53, + "id": 108, "legend": { "alignAsTable": true, "avg": false, @@ -4999,10 +5579,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(pd_scheduler_event_count{instance=\"$instance\", type=\"balance-region-scheduler\"}[5m])) by (name)", + "expr": "sum(delta(pd_scheduler_balance_direction{instance=\"$instance\"}[1m])) by (type, source, target)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{name}}", + "legendFormat": "{{source}}-{{target}}-{{type}}", "metric": "pd_scheduler_event_count", "refId": "A", "step": 4 @@ -5012,7 +5592,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Balance Region scheduler", + "title": "Balance Direction", "tooltip": { "shared": true, "sort": 0, @@ -5055,14 +5635,15 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The replica checker's status", "fill": 0, "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 58 + "y": 87 }, - "id": 108, + "id": 70, "legend": { "alignAsTable": true, "avg": false, @@ -5073,8 +5654,6 @@ "min": false, "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, @@ -5093,20 +5672,19 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(pd_scheduler_balance_direction{instance=\"$instance\"}[1m])) by (type, source, target)", + "expr": "sum(delta(pd_checker_event_count{instance=\"$instance\", type=\"replica_checker\"}[1m])) by (name)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{source}}-{{target}}-{{type}}", - "metric": "pd_scheduler_event_count", + "legendFormat": "{{name}}", "refId": "A", - "step": 4 + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Balance Direction", + "title": "Replica checker", "tooltip": { "shared": true, "sort": 0, @@ -5122,7 +5700,7 @@ }, "yaxes": [ { - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -5154,10 +5732,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 66 + "x": 12, + "y": 87 }, - "id": 70, + "id": 141, "legend": { "alignAsTable": true, "avg": false, @@ -5186,7 +5764,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(pd_checker_event_count{instance=\"$instance\", type=\"replica_checker\"}[1m])) by (name)", + "expr": "sum(delta(pd_checker_event_count{instance=\"$instance\", type=\"rule_checker\"}[1m])) by (name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{name}}", @@ -5198,7 +5776,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Replica checker", + "title": "Rule checker", "tooltip": { "shared": true, "sort": 0, @@ -5241,15 +5819,14 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The merge checker's status", "fill": 0, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 66 + "x": 0, + "y": 95 }, - "id": 71, + "id": 110, "legend": { "alignAsTable": true, "avg": false, @@ -5260,6 +5837,8 @@ "min": false, "rightSide": true, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -5278,19 +5857,20 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(pd_checker_event_count{instance=\"$instance\", type=\"merge_checker\"}[1m])) by (name)", + "expr": "sum(delta(pd_schedule_filter{store=~\"$store\", action=\"filter-target\"}[1m])) by (store, type, scope)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{name}}", + "legendFormat": "{{scope}}-store-{{store}}-{{type}}", + "metric": "pd_scheduler_event_count", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Region merge checker", + "title": "Filter target", "tooltip": { "shared": true, "sort": 0, @@ -5306,7 +5886,7 @@ }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -5333,14 +5913,15 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The merge checker's status", "fill": 0, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 74 + "x": 12, + "y": 95 }, - "id": 110, + "id": 71, "legend": { "alignAsTable": true, "avg": false, @@ -5351,8 +5932,6 @@ "min": false, "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, @@ -5371,20 +5950,19 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(pd_schedule_filter{store=~\"$store\", action=\"filter-target\"}[1m])) by (store, type, scope)", + "expr": "sum(delta(pd_checker_event_count{instance=\"$instance\", type=\"merge_checker\"}[1m])) by (name)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{scope}}-store-{{store}}-{{type}}", - "metric": "pd_scheduler_event_count", + "legendFormat": "{{name}}", "refId": "A", - "step": 4 + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Filter target", + "title": "Region merge checker", "tooltip": { "shared": true, "sort": 0, @@ -5400,7 +5978,7 @@ }, "yaxes": [ { - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -5431,8 +6009,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 74 + "x": 0, + "y": 103 }, "id": 109, "legend": { @@ -5526,7 +6104,7 @@ "h": 1, "w": 24, "x": 0, - "y": 19 + "y": 89 }, "id": 123, "panels": [ @@ -5546,7 +6124,7 @@ "h": 8, "w": 12, "x": 0, - "y": 41 + "y": 119 }, "id": 1, "legend": { @@ -5642,7 +6220,7 @@ "h": 8, "w": 12, "x": 12, - "y": 41 + "y": 119 }, "id": 2, "legend": { @@ -5736,7 +6314,7 @@ "h": 1, "w": 24, "x": 0, - "y": 20 + "y": 90 }, "id": 124, "panels": [ @@ -5755,7 +6333,7 @@ "h": 8, "w": 24, "x": 0, - "y": 42 + "y": 120 }, "id": 5, "legend": { @@ -5849,7 +6427,7 @@ "h": 8, "w": 12, "x": 0, - "y": 50 + "y": 128 }, "id": 6, "legend": { @@ -5943,7 +6521,7 @@ "h": 8, "w": 12, "x": 12, - "y": 50 + "y": 128 }, "id": 7, "legend": { @@ -6040,7 +6618,7 @@ "h": 8, "w": 12, "x": 0, - "y": 58 + "y": 136 }, "id": 34, "legend": { @@ -6175,7 +6753,7 @@ "h": 8, "w": 12, "x": 12, - "y": 58 + "y": 136 }, "id": 44, "legend": { @@ -6265,15 +6843,198 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The current term of Raft", + "description": "The current term of Raft", + "fill": 1, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 144 + }, + "id": 92, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pd_server_etcd_state{type=\"term\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} - {{job}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft term", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The last committed index of Raft", + "fill": 1, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 144 + }, + "id": 93, + "legend": { + "alignAsTable": true, + "alignLevel": null, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pd_server_etcd_state{type=\"committedIndex\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} - {{job}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft committed index", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": " \tThe last applied index of Raft", "fill": 1, "gridPos": { "h": 8, "w": 8, - "x": 0, - "y": 66 + "x": 16, + "y": 144 }, - "id": 92, + "id": 94, "legend": { "alignAsTable": true, "avg": false, @@ -6302,7 +7063,7 @@ "steppedLine": false, "targets": [ { - "expr": "pd_server_etcd_state{type=\"term\"}", + "expr": "pd_server_etcd_state{type=\"appliedIndex\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - {{job}}", @@ -6313,7 +7074,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Raft term", + "title": "Raft applied index", "tooltip": { "shared": true, "sort": 0, @@ -6349,40 +7110,60 @@ "align": false, "alignLevel": null } - }, + } + ], + "repeat": null, + "title": "etcd", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 91 + }, + "id": 125, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The last committed index of Raft", + "description": "The time consumed of handling TiDB requests", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 66 + "h": 9, + "w": 12, + "x": 0, + "y": 114 }, - "id": 93, + "id": 29, "legend": { "alignAsTable": true, - "alignLevel": null, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": false, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "paceLength": 10, "percentage": false, "pointradius": 5, @@ -6394,19 +7175,29 @@ "steppedLine": false, "targets": [ { - "expr": "pd_server_etcd_state{type=\"committedIndex\"}", + "expr": "histogram_quantile(0.99, sum(rate(pd_server_handle_tso_duration_seconds_bucket[30s])) by (type, le))", "format": "time_series", + "hide": false, "intervalFactor": 2, - "legendFormat": "{{instance}} - {{job}}", - "refId": "A" + "legendFormat": "99% tso", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99999, sum(rate(pd_server_handle_tso_duration_seconds_bucket[30s])) by (type, le))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "99.999% tso", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Raft committed index", + "title": "PD server TSO handle time + Client revc time", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -6421,11 +7212,11 @@ }, "yaxes": [ { - "format": "none", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -6448,32 +7239,38 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": " \tThe last applied index of Raft", + "description": "The time consumed of handling TiDB requests", + "editable": true, + "error": false, "fill": 1, + "grid": {}, "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 66 + "h": 9, + "w": 12, + "x": 12, + "y": 114 }, - "id": 94, + "id": 142, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": false, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "paceLength": 10, "percentage": false, "pointradius": 5, @@ -6485,19 +7282,28 @@ "steppedLine": false, "targets": [ { - "expr": "pd_server_etcd_state{type=\"appliedIndex\"}", - "format": "time_series", + "expr": "histogram_quantile(0.98, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket[30s])) by (type, le))", + "hide": false, "intervalFactor": 2, - "legendFormat": "{{instance}} - {{job}}", - "refId": "A" + "legendFormat": "{{type}} 98th percentile", + "refId": "A", + "step": 2 + }, + { + "expr": "avg(rate(pd_client_request_handle_requests_duration_seconds_sum[30s])) by (type) / avg(rate(pd_client_request_handle_requests_duration_seconds_count[30s])) by (type)", + "intervalFactor": 2, + "legendFormat": "{{type}} average", + "refId": "B", + "step": 2 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Raft applied index", + "title": "Handle requests duration", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -6512,11 +7318,11 @@ }, "yaxes": [ { - "format": "none", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -6532,22 +7338,7 @@ "align": false, "alignLevel": null } - } - ], - "repeat": null, - "title": "etcd", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 125, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -6560,10 +7351,10 @@ "fill": 1, "grid": {}, "gridPos": { - "h": 8, - "w": 24, + "h": 9, + "w": 12, "x": 0, - "y": 83 + "y": 123 }, "id": 28, "legend": { @@ -6594,6 +7385,7 @@ "targets": [ { "expr": "sum(rate(pd_client_request_handle_requests_duration_seconds_count[1m])) by (type)", + "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "A", @@ -6641,38 +7433,50 @@ "align": false, "alignLevel": null } - }, + } + ], + "repeat": null, + "title": "TiDB", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 92 + }, + "id": 126, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The time consumed of handling TiDB requests", + "description": "The heartbeat latency of each TiKV instance in .99", "editable": true, "error": false, - "fill": 1, + "fill": 0, "grid": {}, "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 91 + "y": 23 }, - "id": 29, + "id": 74, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, - "hideZero": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, - "sideWidth": 300, - "sort": "current", - "sortDesc": false, "total": false, "values": true }, @@ -6691,26 +7495,20 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.98, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket[30s])) by (type, le))", + "expr": "sum(rate(pd_cluster_region_event{store=~\"$store\", instance=\"$instance\"}[1m])) by (address, store, event)", + "format": "time_series", "hide": false, "intervalFactor": 2, - "legendFormat": "{{type}} 98th percentile", + "legendFormat": "{{event}}", "refId": "A", - "step": 2 - }, - { - "expr": "avg(rate(pd_client_request_handle_requests_duration_seconds_sum[30s])) by (type) / avg(rate(pd_client_request_handle_requests_duration_seconds_count[30s])) by (type)", - "intervalFactor": 2, - "legendFormat": "{{type}} average", - "refId": "B", - "step": 2 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Handle requests duration", + "title": "Heartbeat region event QPS", "tooltip": { "msResolution": false, "shared": true, @@ -6727,7 +7525,7 @@ }, "yaxes": [ { - "format": "s", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -6735,7 +7533,7 @@ "show": true }, { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -6747,22 +7545,7 @@ "align": false, "alignLevel": null } - } - ], - "repeat": null, - "title": "TiDB", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 126, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -6777,10 +7560,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 150 + "x": 12, + "y": 23 }, - "id": 74, + "id": 140, "legend": { "alignAsTable": true, "avg": false, @@ -6866,7 +7649,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The count of the corresponding schedule commands which PD sends to each TiKV instance", + "description": "The count of the heartbeats which each TiKV instance reports to PD", "editable": true, "error": false, "fill": 0, @@ -6874,10 +7657,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 150 + "x": 0, + "y": 31 }, - "id": 64, + "id": 54, "legend": { "alignAsTable": true, "avg": true, @@ -6906,11 +7689,11 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(pd_scheduler_region_heartbeat{store=~\"$store\", type=\"push\",instance=\"$instance\"}[5m])*60) by (address, store, status)", + "expr": "sum(delta(pd_scheduler_region_heartbeat{store=~\"$store\", instance=\"$instance\", type=\"report\", status=\"ok\"}[1m])) by (address, store)", "format": "time_series", "hide": false, "intervalFactor": 2, - "legendFormat": "{{address}}-{{status}}-store-{{store}}", + "legendFormat": "{{address}}-store-{{store}}", "refId": "A", "step": 4 } @@ -6919,7 +7702,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Region schedule push", + "title": "Region heartbeat report", "tooltip": { "msResolution": false, "shared": true, @@ -6963,7 +7746,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The count of the heartbeats which each TiKV instance reports to PD", + "description": "The count of the corresponding schedule commands which PD sends to each TiKV instance", "editable": true, "error": false, "fill": 0, @@ -6971,10 +7754,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 158 + "x": 12, + "y": 31 }, - "id": 54, + "id": 64, "legend": { "alignAsTable": true, "avg": true, @@ -7003,11 +7786,11 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(pd_scheduler_region_heartbeat{store=~\"$store\", instance=\"$instance\", type=\"report\", status=\"ok\"}[1m])) by (address, store)", + "expr": "sum(rate(pd_scheduler_region_heartbeat{store=~\"$store\", type=\"push\",instance=\"$instance\"}[5m])*60) by (address, store, status)", "format": "time_series", "hide": false, "intervalFactor": 2, - "legendFormat": "{{address}}-store-{{store}}", + "legendFormat": "{{address}}-{{status}}-store-{{store}}", "refId": "A", "step": 4 } @@ -7016,7 +7799,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Region heartbeat report", + "title": "Region schedule push", "tooltip": { "msResolution": false, "shared": true, @@ -7060,19 +7843,19 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The count of the heartbeats with the ok status", - "fill": 0, + "description": "The count of the heartbeats with the error status", + "fill": 1, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 158 + "x": 0, + "y": 39 }, - "id": 131, + "id": 133, "legend": { "alignAsTable": true, "avg": true, - "current": true, + "current": false, "hideEmpty": true, "hideZero": true, "max": true, @@ -7085,7 +7868,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 2, @@ -7096,7 +7879,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(pd_scheduler_region_heartbeat{store=~\"$store\", instance=\"$instance\", type=\"report\", status=\"bind\"}[1m])) by (address, store)", + "expr": "sum(delta(pd_scheduler_region_heartbeat{store=~\"$store\", instance=\"$instance\", type=\"report\", status=\"err\"}[1m])) by (address, store)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{address}}-store-{{store}}", @@ -7107,7 +7890,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Region heartbeat report active", + "title": "Region heartbeat report error", "tooltip": { "shared": true, "sort": 0, @@ -7127,7 +7910,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -7150,19 +7933,19 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The count of the heartbeats with the error status", - "fill": 1, + "description": "The count of the heartbeats with the ok status", + "fill": 0, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 166 + "x": 12, + "y": 39 }, - "id": 133, + "id": 131, "legend": { "alignAsTable": true, "avg": true, - "current": false, + "current": true, "hideEmpty": true, "hideZero": true, "max": true, @@ -7175,7 +7958,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "paceLength": 10, "percentage": false, "pointradius": 2, @@ -7186,7 +7969,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(pd_scheduler_region_heartbeat{store=~\"$store\", instance=\"$instance\", type=\"report\", status=\"err\"}[1m])) by (address, store)", + "expr": "sum(delta(pd_scheduler_region_heartbeat{store=~\"$store\", instance=\"$instance\", type=\"report\", status=\"bind\"}[1m])) by (address, store)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{address}}-store-{{store}}", @@ -7197,7 +7980,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Region heartbeat report error", + "title": "Region heartbeat report active", "tooltip": { "shared": true, "sort": 0, @@ -7217,7 +8000,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -7245,7 +8028,7 @@ "h": 1, "w": 24, "x": 0, - "y": 23 + "y": 93 }, "id": 127, "panels": [