From 59a27ad841df7d6172188bd5334f5de1c653b78b Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Fri, 16 Dec 2022 17:48:24 +0800 Subject: [PATCH 1/6] metrics: add EMA cpu usage Signed-off-by: Weizhen Wang --- metrics/BUILD.bazel | 1 + metrics/resourcemanager.go | 27 +++++++++++++++++++++++++++ util/cpu/BUILD.bazel | 1 + util/cpu/cpu.go | 2 ++ 4 files changed, 31 insertions(+) create mode 100644 metrics/resourcemanager.go diff --git a/metrics/BUILD.bazel b/metrics/BUILD.bazel index 4b6b6cf9f2ee1..c414e04d9c907 100644 --- a/metrics/BUILD.bazel +++ b/metrics/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "meta.go", "metrics.go", "owner.go", + "resourcemanager.go", "server.go", "session.go", "sli.go", diff --git a/metrics/resourcemanager.go b/metrics/resourcemanager.go new file mode 100644 index 0000000000000..d45b8833e1225 --- /dev/null +++ b/metrics/resourcemanager.go @@ -0,0 +1,27 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +var ( + // EMACPUUsageGauge means exponential moving average of CPU usage + EMACPUUsageGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "tidb", + Subsystem: "rm", + Name: "ema_cpu_usage", + Help: "exponential moving average of CPU usage", + }) +) diff --git a/util/cpu/BUILD.bazel b/util/cpu/BUILD.bazel index 8a4852337d65e..58bc047a332c4 100644 --- a/util/cpu/BUILD.bazel +++ b/util/cpu/BUILD.bazel @@ -6,6 +6,7 @@ go_library( importpath = "github.com/pingcap/tidb/util/cpu", visibility = ["//visibility:public"], deps = [ + "//metrics", "//util/cgroup", "//util/mathutil", "@com_github_cloudfoundry_gosigar//:gosigar", diff --git a/util/cpu/cpu.go b/util/cpu/cpu.go index 5edb8b64ba487..416b3c3eaeb99 100644 --- a/util/cpu/cpu.go +++ b/util/cpu/cpu.go @@ -21,6 +21,7 @@ import ( "github.com/cloudfoundry/gosigar" "github.com/pingcap/log" + "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/util/cgroup" "github.com/pingcap/tidb/util/mathutil" "go.uber.org/atomic" @@ -66,6 +67,7 @@ func (c *Observer) Start() { curr := c.observe() c.cpu.Add(curr) cpuUsage.Store(c.cpu.Get()) + metrics.EMACPUUsageGauge.Set(c.cpu.Get()) case <-c.exit: return } From ab7e5a10d242c4105c0b683701812852d3e5f151 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Fri, 16 Dec 2022 19:15:12 +0800 Subject: [PATCH 2/6] improve code Signed-off-by: Weizhen Wang --- metrics/grafana/tidb.json | 108 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/metrics/grafana/tidb.json b/metrics/grafana/tidb.json index 7677a718e156f..579ee9ca77b32 100644 --- a/metrics/grafana/tidb.json +++ b/metrics/grafana/tidb.json @@ -17823,6 +17823,114 @@ "align": false, "alignLevel": null } + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 289, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Test-Cluster", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 52 + }, + "hiddenSeries": false, + "id": 291, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.10", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "tidb_server_gogc{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "{{instance}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Panel Title", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Resource Manager", + "type": "row" } ], "refresh": "30s", From fcc922270b1cdd77a90e0ec0b8e816290704b56e Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Fri, 16 Dec 2022 19:16:04 +0800 Subject: [PATCH 3/6] improve code Signed-off-by: Weizhen Wang --- metrics/grafana/tidb.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/grafana/tidb.json b/metrics/grafana/tidb.json index 579ee9ca77b32..a08b7fdaac929 100644 --- a/metrics/grafana/tidb.json +++ b/metrics/grafana/tidb.json @@ -17891,7 +17891,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "Panel Title", + "title": "GOGC", "tooltip": { "shared": true, "sort": 0, From c4898645125f5c6cb0d85bb088d3e2da146d7b4d Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 19 Dec 2022 12:11:10 +0800 Subject: [PATCH 4/6] update Signed-off-by: Weizhen Wang --- metrics/grafana/tidb.json | 108 -------------------------------------- 1 file changed, 108 deletions(-) diff --git a/metrics/grafana/tidb.json b/metrics/grafana/tidb.json index a08b7fdaac929..7677a718e156f 100644 --- a/metrics/grafana/tidb.json +++ b/metrics/grafana/tidb.json @@ -17823,114 +17823,6 @@ "align": false, "alignLevel": null } - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 289, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Test-Cluster", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 52 - }, - "hiddenSeries": false, - "id": 291, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.10", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "tidb_server_gogc{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{instance}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "GOGC", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "title": "Resource Manager", - "type": "row" } ], "refresh": "30s", From dba6e036dc08321a326052507802b84d6c7441f3 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 19 Dec 2022 12:24:18 +0800 Subject: [PATCH 5/6] update Signed-off-by: Weizhen Wang --- metrics/metrics.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metrics/metrics.go b/metrics/metrics.go index 2984b66ddb27c..0cca442e5eb83 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -215,6 +215,8 @@ func RegisterMetrics() { prometheus.MustRegister(TTLJobStatus) prometheus.MustRegister(TTLPhaseTime) + prometheus.MustRegister(EMACPUUsageGauge) + tikvmetrics.InitMetrics(TiDB, TiKVClient) tikvmetrics.RegisterMetrics() tikvmetrics.TiKVPanicCounter = PanicCounter // reset tidb metrics for tikv metrics From f33ee5dd6e109f50f9df288d750c1528149ed3ed Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 19 Dec 2022 12:26:00 +0800 Subject: [PATCH 6/6] update Signed-off-by: Weizhen Wang --- resourcemanager/rm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resourcemanager/rm.go b/resourcemanager/rm.go index 54ae27f82a4f6..1195176fbde95 100644 --- a/resourcemanager/rm.go +++ b/resourcemanager/rm.go @@ -43,5 +43,5 @@ func (r *ResourceManager) Start() { // Stop is to stop resource manager func (r *ResourceManager) Stop() { r.cpuObserver.Stop() - r.wg.Done() + r.wg.Wait() }