From 15336f7f044060a369efcf0984434856e504c16c Mon Sep 17 00:00:00 2001 From: David Smith <68763658+dsmith111@users.noreply.github.com> Date: Mon, 26 Sep 2022 11:00:40 -0700 Subject: [PATCH] Add controller performance metrics (#391) * Added GameServer control performance metrics; Updated Grafana dashboard * Update monitoring documentation; Refactor non-active handling * Update yaml * Fix capitalization * Revert extra changes triggering installfile alret * Handle dereferencing; Add util funciton; Remove repeated lines * Add pointers * Decreasing time diff * PR Updates; Rename deleteSum; Fix GS state update; * Add patching exception * Change metric emission to nodeagent * Update dashboard * Revert test * Cleanup deletes * Minor tweaks * Conditional * PR Suggested changes * Remove spacing added to gameserverbuild * Remove empty line in nodeagent * Renaming * Update dashboard * Remove metric Co-authored-by: Dimitris-Ilias Gkanatsios --- .gitignore | 3 + cmd/nodeagent/nodeagentmanager.go | 4 + cmd/nodeagent/types.go | 18 + docs/howtos/monitoring.md | 2 + .../controllers/gameserverbuild_controller.go | 15 +- samples/grafana/dashboard.json | 1659 ++++++++++------- 6 files changed, 1006 insertions(+), 695 deletions(-) diff --git a/.gitignore b/.gitignore index 4805176d..39753958 100644 --- a/.gitignore +++ b/.gitignore @@ -23,5 +23,8 @@ installfilesdev .uptodate +# vscode settings +.vscode + # allocator compiled plugin kubectl-gameserver diff --git a/cmd/nodeagent/nodeagentmanager.go b/cmd/nodeagent/nodeagentmanager.go index a09c5bea..d5922566 100644 --- a/cmd/nodeagent/nodeagentmanager.go +++ b/cmd/nodeagent/nodeagentmanager.go @@ -452,8 +452,12 @@ func (n *NodeAgentManager) updateHealthAndStateIfNeeded(ctx context.Context, hb now := metav1.Time{Time: n.nowFunc()} if hb.CurrentGameState == GameStateInitializing { status.ReachedInitializingOn = &now + timeDif := time.Now().UnixMilli() - gsd.CreationTime + GameServerReachedInitializingDuration.WithLabelValues(gsd.BuildName).Set(float64(timeDif)) } else if hb.CurrentGameState == GameStateStandingBy { status.ReachedStandingByOn = &now + timeDif := time.Now().UnixMilli() - gsd.CreationTime + GameServerReachedStandingByDuration.WithLabelValues(gsd.BuildName).Set(float64(timeDif)) } } diff --git a/cmd/nodeagent/types.go b/cmd/nodeagent/types.go index 74935bb6..d09592c4 100644 --- a/cmd/nodeagent/types.go +++ b/cmd/nodeagent/types.go @@ -58,6 +58,24 @@ var ( Name: "connected_players", Help: "Number of connected players per GameServer", }, []string{"namespace", "ServerName", "BuildName"}) + + GameServerReachedStandingByDuration = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "thundernetes", + Name: "gameserver_standing_by_duration", + Help: "Time taken for a GameServer to reach StandingBy", + }, + []string{"BuildName"}, + ) + + GameServerReachedInitializingDuration = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "thundernetes", + Name: "gameserver_initialization_duration", + Help: "Time taken for a GameServer to reach initialization", + }, + []string{"BuildName"}, + ) ) // HeartbeatRequest contains data for the heartbeat request coming from the GSDK running alongside GameServer diff --git a/docs/howtos/monitoring.md b/docs/howtos/monitoring.md index fe3877d4..e7a62d04 100644 --- a/docs/howtos/monitoring.md +++ b/docs/howtos/monitoring.md @@ -84,6 +84,8 @@ There is a custom Grafana dashboard example that visualizes some of this data in | --- | --- | --- | | gameserver_states | Gauge | nodeagent | | connected_players | Gauge | nodeagent | +| gameserver_initialization_duration | Gauge | nodeagent | +| gameserver_standing_by_duration | Gauge | nodeagent | | gameservers_current_state_per_build | Gauge | controller-manager | | gameservers_created_total | Counter | controller-manager | | gameservers_sessionended_total | Counter | controller-manager | diff --git a/pkg/operator/controllers/gameserverbuild_controller.go b/pkg/operator/controllers/gameserverbuild_controller.go index 80ab5dd9..f976d330 100644 --- a/pkg/operator/controllers/gameserverbuild_controller.go +++ b/pkg/operator/controllers/gameserverbuild_controller.go @@ -183,19 +183,18 @@ func (r *GameServerBuildReconciler) Reconcile(ctx context.Context, req ctrl.Requ // calculate the total amount of servers not in the active state nonActiveGameServersCount := standingByCount + initializingCount + pendingCount + // Evaluate desired number of servers against actual + var totalNumberOfGameServersToDelete int = 0 // user has decreased standingBy numbers if nonActiveGameServersCount > gsb.Spec.StandingBy { - totalNumberOfGameServersToDelete := int(math.Min(float64(nonActiveGameServersCount-gsb.Spec.StandingBy), maxNumberOfGameServersToDelete)) - err := r.deleteNonActiveGameServers(ctx, &gsb, &gameServers, totalNumberOfGameServersToDelete) - if err != nil { - return ctrl.Result{}, err - } + totalNumberOfGameServersToDelete += int(math.Min(float64(nonActiveGameServersCount-gsb.Spec.StandingBy), maxNumberOfGameServersToDelete)) } - - // we need to check if we are above the max + // we also need to check if we are above the max // this can happen if the user modifies the spec.Max during the GameServerBuild's lifetime if nonActiveGameServersCount+activeCount > gsb.Spec.Max { - totalNumberOfGameServersToDelete := int(math.Min(float64(nonActiveGameServersCount+activeCount-gsb.Spec.Max), maxNumberOfGameServersToDelete)) + totalNumberOfGameServersToDelete += int(math.Min(float64(totalNumberOfGameServersToDelete+(nonActiveGameServersCount+activeCount-gsb.Spec.Max)), maxNumberOfGameServersToDelete)) + } + if totalNumberOfGameServersToDelete > 0 { err := r.deleteNonActiveGameServers(ctx, &gsb, &gameServers, totalNumberOfGameServersToDelete) if err != nil { return ctrl.Result{}, err diff --git a/samples/grafana/dashboard.json b/samples/grafana/dashboard.json index 255b46c4..7dec2d81 100644 --- a/samples/grafana/dashboard.json +++ b/samples/grafana/dashboard.json @@ -3,7 +3,10 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -21,7 +24,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 26, + "id": 27, "links": [], "liveNow": false, "panels": [ @@ -75,9 +78,13 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.4.5", + "pluginVersion": "9.1.0", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, "exemplar": true, "expr": "sum(thundernetes_gameservers_current_state_per_build{state=\"active\"})", "interval": "", @@ -138,9 +145,13 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.4.5", + "pluginVersion": "9.1.0", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, "exemplar": true, "expr": "sum(thundernetes_gameservers_current_state_per_build{state=\"standingby\"})", "interval": "", @@ -201,7 +212,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.4.5", + "pluginVersion": "9.1.0", "targets": [ { "datasource": { @@ -268,7 +279,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.4.5", + "pluginVersion": "9.1.0", "targets": [ { "datasource": { @@ -286,7 +297,11 @@ "type": "stat" }, { - "collapsed": false, + "collapsed": true, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "gridPos": { "h": 1, "w": 24, @@ -294,667 +309,925 @@ "y": 8 }, "id": 20, - "panels": [], - "title": "Status", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ + "overrides": [ { - "color": "green", - "value": null + "matcher": { + "id": "byName", + "options": "gameserverbuild-sample-openarena-StandBy" + }, + "properties": [ + { + "id": "custom.fillBelowTo", + "value": "gameserverbuild-sample-openarena-Active" + }, + { + "id": "custom.fillOpacity", + "value": 6 + } + ] } ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "gameserverbuild-sample-openarena-StandBy" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "properties": [ - { - "id": "custom.fillBelowTo", - "value": "gameserverbuild-sample-openarena-Active" + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" }, - { - "id": "custom.fillOpacity", - "value": 6 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "exemplar": true, + "expr": "sum(thundernetes_gameservers_current_state_per_build) by (state)", + "hide": false, + "interval": "", + "legendFormat": "{{state}}", + "refId": "B" + } + ], + "title": "Game Server History", + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ { - "exemplar": true, - "expr": "sum(thundernetes_gameservers_current_state_per_build) by (state)", - "hide": false, - "interval": "", - "legendFormat": "{{state}}", - "refId": "B" - } - ], - "title": "Game Server History", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "exemplar": true, + "expr": "sum(thundernetes_allocations_total{}) by (BuildName)", + "instant": false, + "interval": "", + "legendFormat": "{{ BuildName }}", + "refId": "A" + } + ], + "title": "Total allocations time series", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, - "exemplar": true, - "expr": "sum(thundernetes_allocations_total{}) by (BuildName)", - "instant": false, - "interval": "", - "legendFormat": "{{ BuildName }}", - "refId": "A" - } - ], - "title": "Total allocations time series", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 17 - }, - "id": 23, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "exemplar": true, + "expr": "sum(thundernetes_gameservers_current_state_per_build{state=\"active\"}) by (BuildName)", + "instant": false, + "interval": "", + "legendFormat": "{{ BuildName }}", + "refId": "A" + } + ], + "title": "Active servers time series", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, - "exemplar": true, - "expr": "sum(thundernetes_gameservers_current_state_per_build{state=\"active\"}) by (BuildName)", - "instant": false, - "interval": "", - "legendFormat": "{{ BuildName }}", - "refId": "A" - } - ], - "title": "Active servers time series", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 17 - }, - "id": 25, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "exemplar": true, + "expr": "sum(thundernetes_gameservers_current_state_per_build{state=\"standingby\"}) by (BuildName)", + "instant": false, + "interval": "", + "legendFormat": "{{ BuildName }}", + "refId": "A" + } + ], + "title": "Standby servers time series", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, - "exemplar": true, - "expr": "sum(thundernetes_gameservers_current_state_per_build{state=\"standingby\"}) by (BuildName)", - "instant": false, - "interval": "", - "legendFormat": "{{ BuildName }}", - "refId": "A" - } - ], - "title": "Standby servers time series", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 6, + "y": 25 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 6, - "y": 25 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" + "exemplar": true, + "expr": "sum(thundernetes_connected_players) by (BuildName)", + "instant": false, + "interval": "", + "legendFormat": "{{ BuildName}}", + "refId": "A" + } + ], + "title": "Connected players time series", + "type": "timeseries" } - }, + ], "targets": [ { "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "type": "datasource", + "uid": "grafana" }, - "exemplar": true, - "expr": "sum(thundernetes_connected_players) by (BuildName)", - "instant": false, - "interval": "", - "legendFormat": "{{ BuildName}}", "refId": "A" } ], - "title": "Connected players time series", - "type": "timeseries" + "title": "Status", + "type": "row" }, { - "collapsed": false, + "collapsed": true, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 9 }, "id": 9, - "panels": [], - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" }, - { - "color": "red", - "value": 80 - } - ] + "exemplar": true, + "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"gameserverbuild.+\"}[5m])) by (pod) /\nsum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{pod=~\"gameserverbuild.+\"}) by (pod)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Game Server % of CPU Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "unit": "percentunit" + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "exemplar": true, + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{pod=~\"gameserverbuild.+\"}) by (pod) /\nsum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{namespace=\"default\"}) by (pod)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Game Server % of Memory Utilization", + "type": "timeseries" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 34 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 14, + "interval": "20", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "exemplar": true, + "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"gameserverbuild.+\"}[5m])) by (pod) * 100", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Game Server CPU Total Utilization", + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, - "exemplar": true, - "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"gameserverbuild.+\"}[5m])) by (pod) /\nsum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{pod=~\"gameserverbuild.+\"}) by (pod)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "title": "Game Server % of CPU Utilization", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 16, + "interval": "20", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" }, - { - "color": "red", - "value": 80 - } - ] + "exemplar": true, + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{pod=~\"gameserverbuild.+\"}) by (pod)", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Game Server Total Memory Utilization", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" }, - "unit": "percentunit" - }, - "overrides": [] + "refId": "A" + } + ], + "title": "Resources", + "type": "row" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 34 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "h": 1, + "w": 24, + "x": 0, + "y": 10 }, + "id": 29, + "panels": [], "targets": [ { - "exemplar": true, - "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{pod=~\"gameserverbuild.+\"}) by (pod) /\nsum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{namespace=\"default\"}) by (pod)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "refId": "A" } ], - "title": "Game Server % of Memory Utilization", - "type": "timeseries" + "title": "Thundernetes Controller", + "type": "row" }, { "datasource": { @@ -964,22 +1237,24 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "thresholds" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", + "fillOpacity": 20, + "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "linear", - "lineWidth": 1, + "lineInterpolation": "smooth", + "lineWidth": 3, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -1016,21 +1291,24 @@ "h": 8, "w": 12, "x": 0, - "y": 42 + "y": 11 }, - "id": 14, - "interval": "20", + "id": 27, + "interval": "1m", + "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, + "pluginVersion": "8.4.3", "targets": [ { "datasource": { @@ -1038,13 +1316,16 @@ "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, - "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"gameserverbuild.+\"}[5m])) by (pod) * 100", + "expr": "rate(process_cpu_seconds_total{job=\"thundernetes-controller-manager-metrics-service\", namespace=\"thundernetes-system\", pod=~\"thundernetes-controller-manager.+\"}[5m]) * 100", + "format": "time_series", "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "Pod: {{pod}} | Container: {{container}}", + "refId": "A", + "step": 10 } ], - "title": "Game Server CPU Total Utilization", + "title": "Controller CPU Usage", "type": "timeseries" }, { @@ -1055,22 +1336,25 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "fixedColor": "green", + "mode": "thresholds" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", + "fillOpacity": 20, + "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "linear", - "lineWidth": 1, + "lineInterpolation": "smooth", + "lineWidth": 3, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -1092,10 +1376,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, @@ -1107,57 +1387,57 @@ "h": 8, "w": 12, "x": 12, - "y": 42 + "y": 11 }, - "id": 16, - "interval": "20", + "id": 31, + "interval": "1m", + "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, + "pluginVersion": "8.4.3", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, "exemplar": true, - "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{pod=~\"gameserverbuild.+\"}) by (pod)", + "expr": "process_resident_memory_bytes{job=\"thundernetes-controller-manager-metrics-service\", namespace=\"thundernetes-system\", pod=~\"thundernetes-controller-manager.+\"}", + "format": "time_series", "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "Pod: {{pod}} | Container: {{container}}", + "refId": "A", + "step": 10 } ], - "title": "Game Server Total Memory Utilization", + "title": "Controller Memory Usage", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 50 - }, - "id": 29, - "panels": [], - "title": "Thundernetes Controller", - "type": "row" - }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, + "description": "Total number of reconciliation per controller", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -1192,14 +1472,10 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "percent" + "unit": "cpm" }, "overrides": [] }, @@ -1207,40 +1483,37 @@ "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 19 }, - "id": 27, - "interval": "1m", - "links": [], + "id": 35, "options": { "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, - "pluginVersion": "8.4.3", "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, + "editorMode": "code", "exemplar": true, - "expr": "rate(process_cpu_seconds_total{job=\"thundernetes-controller-manager-metrics-service\", namespace=\"thundernetes-system\", pod=~\"thundernetes-controller-manager.+\"}[5m]) * 100", - "format": "time_series", + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"thundernetes-controller-manager-metrics-service\", namespace=\"thundernetes-system\"}[5m])) by (instance, pod)", "interval": "", - "intervalFactor": 2, - "legendFormat": "Pod: {{pod}} | Container: {{container}}", - "refId": "A", - "step": 10 + "legendFormat": "{{instance}} {{pod}}", + "range": true, + "refId": "A" } ], - "title": "Controller CPU Usage", + "title": "Reconciliation Total Count Per Controller", "type": "timeseries" }, { @@ -1248,19 +1521,21 @@ "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, + "description": "Total number of reconciliation errors per controller", "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", "mode": "thresholds" }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, - "gradientMode": "opacity", + "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, @@ -1292,7 +1567,7 @@ } ] }, - "unit": "bytes" + "unit": "cpm" }, "overrides": [] }, @@ -1300,40 +1575,37 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 19 }, - "id": 31, - "interval": "1m", - "links": [], + "id": 33, "options": { "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, - "pluginVersion": "8.4.3", "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, + "editorMode": "code", "exemplar": true, - "expr": "process_resident_memory_bytes{job=\"thundernetes-controller-manager-metrics-service\", namespace=\"thundernetes-system\", pod=~\"thundernetes-controller-manager.+\"}", - "format": "time_series", + "expr": "sum(rate(controller_runtime_reconcile_errors_total{job=\"thundernetes-controller-manager-metrics-service\", namespace=\"thundernetes-system\"}[5m])) by (instance, pod)", "interval": "", - "intervalFactor": 2, - "legendFormat": "Pod: {{pod}} | Container: {{container}}", - "refId": "A", - "step": 10 + "legendFormat": "{{instance}} {{pod}}", + "range": true, + "refId": "A" } ], - "title": "Controller Memory Usage", + "title": "Reconciliation Error Count Per Controller", "type": "timeseries" }, { @@ -1341,14 +1613,16 @@ "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, - "description": "Total number of reconciliation per controller", + "description": "This panel consists of the 5-minute average time among all of the new GameServers to reach StandBy.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "custom": { - "axisLabel": "", + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Time (milliseconds)", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -1382,10 +1656,13 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - }, - "unit": "cpm" + } }, "overrides": [] }, @@ -1393,14 +1670,17 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 27 }, - "id": 35, + "id": 37, + "interval": "10", + "maxDataPoints": 596, "options": { "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", @@ -1414,15 +1694,14 @@ "uid": "P1809F7CD0C75ACF3" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(controller_runtime_reconcile_total{job=\"thundernetes-controller-manager-metrics-service\", namespace=\"thundernetes-system\"}[5m])) by (instance, pod)", - "interval": "", - "legendFormat": "{{instance}} {{pod}}", + "expr": "", + "hide": false, + "legendFormat": "{{BuildName}}-{{instance}}", "range": true, "refId": "A" } ], - "title": "Reconciliation Total Count Per Controller", + "title": "Average Time Until GameServer Reaches StandBy", "type": "timeseries" }, { @@ -1430,19 +1709,21 @@ "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, - "description": "Total number of reconciliation errors per controller", + "description": "This panel consists of the 5-minute average time among all of the new GameServers to reach Initialization.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "custom": { - "axisLabel": "", + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Time (milliseconds)", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, - "gradientMode": "scheme", + "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, @@ -1471,10 +1752,13 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - }, - "unit": "cpm" + } }, "overrides": [] }, @@ -1482,14 +1766,16 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 27 }, - "id": 33, + "id": 39, + "interval": "0.01", "options": { "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", @@ -1502,34 +1788,33 @@ "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(controller_runtime_reconcile_errors_total{job=\"thundernetes-controller-manager-metrics-service\", namespace=\"thundernetes-system\"}[5m])) by (instance, pod)", - "interval": "", - "legendFormat": "{{instance}} {{pod}}", + "editorMode": "builder", + "expr": "avg_over_time(thundernetes_gameserver_initialization_duration[5m])", + "hide": false, + "legendFormat": "{{BuildName}}-{{instance}}", "range": true, "refId": "A" } ], - "title": "Reconciliation Error Count Per Controller", + "title": "Average Time Until GameServer Reaches Initialization", "type": "timeseries" } ], - "refresh": "10s", - "schemaVersion": 35, + "refresh": false, + "schemaVersion": 37, "style": "dark", "tags": [], "templating": { "list": [] }, "time": { - "from": "now-1h", - "to": "now" + "from": "2022-09-25T15:57:24.300Z", + "to": "2022-09-25T17:57:24.300Z" }, "timepicker": {}, "timezone": "", "title": "Thundernetes GameServer Demo", "uid": "T9KjuZOnz", - "version": 3, + "version": 2, "weekStart": "" -} +} \ No newline at end of file