Add controller performance metrics (#391)

* Added GameServer control performance metrics;
Updated Grafana dashboard

* Update monitoring documentation;
Refactor non-active handling

* Update yaml

* Fix capitalization

* Revert extra changes triggering installfile alret

* Handle dereferencing;
Add util funciton;
Remove repeated lines

* Add pointers

* Decreasing time diff

* PR Updates;
Rename deleteSum;
Fix GS state update;

* Add patching exception

* Change metric emission to nodeagent

* Update dashboard

* Revert test

* Cleanup deletes

* Minor tweaks

* Conditional

* PR Suggested changes

* Remove spacing added to gameserverbuild

* Remove empty line in nodeagent

* Renaming

* Update dashboard

* Remove metric

Co-authored-by: Dimitris-Ilias Gkanatsios <dgkanatsios@outlook.com>

.gitignore

@@ -23,5 +23,8 @@ installfilesdev
 
 .uptodate
 
+# vscode settings
+.vscode
+
 # allocator compiled plugin
 kubectl-gameserver

cmd/nodeagent/nodeagentmanager.go

@@ -452,8 +452,12 @@ func (n *NodeAgentManager) updateHealthAndStateIfNeeded(ctx context.Context, hb
  now := metav1.Time{Time: n.nowFunc()}
  if hb.CurrentGameState == GameStateInitializing {
  status.ReachedInitializingOn = &now
+ timeDif := time.Now().UnixMilli() - gsd.CreationTime
+ GameServerReachedInitializingDuration.WithLabelValues(gsd.BuildName).Set(float64(timeDif))
  } else if hb.CurrentGameState == GameStateStandingBy {
  status.ReachedStandingByOn = &now
+ timeDif := time.Now().UnixMilli() - gsd.CreationTime
+ GameServerReachedStandingByDuration.WithLabelValues(gsd.BuildName).Set(float64(timeDif))
  }
  }
 

cmd/nodeagent/types.go

@@ -58,6 +58,24 @@ var (
  Name: "connected_players",
  Help: "Number of connected players per GameServer",
  }, []string{"namespace", "ServerName", "BuildName"})
+
+ GameServerReachedStandingByDuration = promauto.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Namespace: "thundernetes",
+ Name: "gameserver_standing_by_duration",
+ Help: "Time taken for a GameServer to reach StandingBy",
+ },
+ []string{"BuildName"},
+ )
+
+ GameServerReachedInitializingDuration = promauto.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Namespace: "thundernetes",
+ Name: "gameserver_initialization_duration",
+ Help: "Time taken for a GameServer to reach initialization",
+ },
+ []string{"BuildName"},
+ )
 )
 
 // HeartbeatRequest contains data for the heartbeat request coming from the GSDK running alongside GameServer

docs/howtos/monitoring.md

@@ -84,6 +84,8 @@ There is a custom Grafana dashboard example that visualizes some of this data in
 | --- | --- | --- |
 | gameserver_states | Gauge | nodeagent |
 | connected_players | Gauge | nodeagent |
+| gameserver_initialization_duration | Gauge | nodeagent |
+| gameserver_standing_by_duration | Gauge | nodeagent |
 | gameservers_current_state_per_build | Gauge | controller-manager |
 | gameservers_created_total | Counter | controller-manager |
 | gameservers_sessionended_total | Counter | controller-manager |

pkg/operator/controllers/gameserverbuild_controller.go

@@ -183,19 +183,18 @@ func (r *GameServerBuildReconciler) Reconcile(ctx context.Context, req ctrl.Requ
  // calculate the total amount of servers not in the active state
  nonActiveGameServersCount := standingByCount + initializingCount + pendingCount
 
+ // Evaluate desired number of servers against actual
+ var totalNumberOfGameServersToDelete int = 0
  // user has decreased standingBy numbers
  if nonActiveGameServersCount > gsb.Spec.StandingBy {
- totalNumberOfGameServersToDelete := int(math.Min(float64(nonActiveGameServersCount-gsb.Spec.StandingBy), maxNumberOfGameServersToDelete))
- err := r.deleteNonActiveGameServers(ctx, &gsb, &gameServers, totalNumberOfGameServersToDelete)
- if err != nil {
- return ctrl.Result{}, err
- }
+ totalNumberOfGameServersToDelete += int(math.Min(float64(nonActiveGameServersCount-gsb.Spec.StandingBy), maxNumberOfGameServersToDelete))
  }
-
- // we need to check if we are above the max
+ // we also need to check if we are above the max
  // this can happen if the user modifies the spec.Max during the GameServerBuild's lifetime
  if nonActiveGameServersCount+activeCount > gsb.Spec.Max {
- totalNumberOfGameServersToDelete := int(math.Min(float64(nonActiveGameServersCount+activeCount-gsb.Spec.Max), maxNumberOfGameServersToDelete))
+ totalNumberOfGameServersToDelete += int(math.Min(float64(totalNumberOfGameServersToDelete+(nonActiveGameServersCount+activeCount-gsb.Spec.Max)), maxNumberOfGameServersToDelete))
+ }
+ if totalNumberOfGameServersToDelete > 0 {
  err := r.deleteNonActiveGameServers(ctx, &gsb, &gameServers, totalNumberOfGameServersToDelete)
  if err != nil {
  return ctrl.Result{}, err