From dabe645b6db3fbd8fee6289e4978d519da32e399 Mon Sep 17 00:00:00 2001
From: MasterPtato <maxpopkov.contact@gmail.com>
Date: Mon, 22 Apr 2024 23:03:10 +0000
Subject: [PATCH] feat: add provisioning dashboard

---
 .../grafana_dashboards/provisioning.json      | 1601 +++++++++++++++++
 lib/metrics/src/buckets.rs                    |    8 +
 lib/metrics/src/lib.rs                        |    1 +
 svc/Cargo.lock                                |   21 +
 svc/Cargo.toml                                |    1 +
 .../migrations/20231201000927_init.up.sql     |    1 +
 .../standalone/metrics-publish/Cargo.toml     |   28 +
 .../standalone/metrics-publish/Service.toml   |    8 +
 .../standalone/metrics-publish/src/lib.rs     |  209 +++
 .../standalone/metrics-publish/src/main.rs    |   31 +
 .../metrics-publish/tests/integration.rs      |    1 +
 svc/pkg/cluster/util/Cargo.toml               |    2 +
 svc/pkg/cluster/util/src/lib.rs               |    1 +
 svc/pkg/cluster/util/src/metrics.rs           |   62 +
 .../worker/src/workers/datacenter_scale.rs    |   44 +-
 .../src/workers/nomad_node_registered.rs      |   41 +-
 .../worker/src/workers/server_install/mod.rs  |   56 +-
 .../worker/src/workers/server_provision.rs    |   45 +-
 .../worker/tests/datacenter_tls_issue.rs      |    1 +
 19 files changed, 2131 insertions(+), 31 deletions(-)
 create mode 100644 infra/tf/k8s_infra/grafana_dashboards/provisioning.json
 create mode 100644 svc/pkg/cluster/standalone/metrics-publish/Cargo.toml
 create mode 100644 svc/pkg/cluster/standalone/metrics-publish/Service.toml
 create mode 100644 svc/pkg/cluster/standalone/metrics-publish/src/lib.rs
 create mode 100644 svc/pkg/cluster/standalone/metrics-publish/src/main.rs
 create mode 100644 svc/pkg/cluster/standalone/metrics-publish/tests/integration.rs
 create mode 100644 svc/pkg/cluster/util/src/metrics.rs

diff --git a/infra/tf/k8s_infra/grafana_dashboards/provisioning.json b/infra/tf/k8s_infra/grafana_dashboards/provisioning.json
new file mode 100644
index 0000000000..0a0865c3e5
--- /dev/null
+++ b/infra/tf/k8s_infra/grafana_dashboards/provisioning.json
@@ -0,0 +1,1601 @@
+{
+	"annotations": {
+		"list": [
+			{
+				"builtIn": 1,
+				"datasource": {
+					"type": "grafana",
+					"uid": "-- Grafana --"
+				},
+				"enable": true,
+				"hide": true,
+				"iconColor": "rgba(0, 211, 255, 1)",
+				"name": "Annotations & Alerts",
+				"type": "dashboard"
+			}
+		]
+	},
+	"editable": true,
+	"fiscalYearStartMonth": 0,
+	"graphTooltip": 1,
+	"id": 48,
+	"links": [],
+	"liveNow": false,
+	"panels": [
+		{
+			"datasource": {
+				"type": "prometheus",
+				"uid": "prometheus"
+			},
+			"fieldConfig": {
+				"defaults": {
+					"color": {
+						"mode": "palette-classic"
+					},
+					"custom": {
+						"axisCenteredZero": false,
+						"axisColorMode": "text",
+						"axisLabel": "",
+						"axisPlacement": "auto",
+						"barAlignment": 0,
+						"drawStyle": "line",
+						"fillOpacity": 0,
+						"gradientMode": "none",
+						"hideFrom": {
+							"legend": false,
+							"tooltip": false,
+							"viz": false
+						},
+						"insertNulls": false,
+						"lineInterpolation": "linear",
+						"lineWidth": 1,
+						"pointSize": 5,
+						"scaleDistribution": {
+							"type": "linear"
+						},
+						"showPoints": "auto",
+						"spanNulls": false,
+						"stacking": {
+							"group": "A",
+							"mode": "none"
+						},
+						"thresholdsStyle": {
+							"mode": "off"
+						}
+					},
+					"mappings": [],
+					"thresholds": {
+						"mode": "absolute",
+						"steps": [
+							{
+								"color": "green",
+								"value": null
+							},
+							{
+								"color": "red",
+								"value": 80
+							}
+						]
+					}
+				},
+				"overrides": []
+			},
+			"gridPos": {
+				"h": 8,
+				"w": 12,
+				"x": 0,
+				"y": 0
+			},
+			"id": 1,
+			"options": {
+				"legend": {
+					"calcs": [],
+					"displayMode": "list",
+					"placement": "bottom",
+					"showLegend": true
+				},
+				"tooltip": {
+					"mode": "multi",
+					"sort": "none"
+				}
+			},
+			"targets": [
+				{
+					"datasource": {
+						"type": "prometheus",
+						"uid": "prometheus"
+					},
+					"editorMode": "code",
+					"expr": "sum by (pool_type) (rivet_provision_active_servers{cluster_id=~\"[[cluster_id]]\", datacenter_id=~\"[[datacenter_id]]\", provider_datacenter_id=~\"[[provider_datacenter_id]]\"})",
+					"instant": false,
+					"legendFormat": "__auto",
+					"range": true,
+					"refId": "A"
+				}
+			],
+			"title": "Active Servers",
+			"type": "timeseries"
+		},
+		{
+			"datasource": {
+				"type": "prometheus",
+				"uid": "prometheus"
+			},
+			"fieldConfig": {
+				"defaults": {
+					"color": {
+						"mode": "palette-classic"
+					},
+					"custom": {
+						"axisCenteredZero": false,
+						"axisColorMode": "text",
+						"axisLabel": "",
+						"axisPlacement": "auto",
+						"barAlignment": 0,
+						"drawStyle": "line",
+						"fillOpacity": 0,
+						"gradientMode": "none",
+						"hideFrom": {
+							"legend": false,
+							"tooltip": false,
+							"viz": false
+						},
+						"insertNulls": false,
+						"lineInterpolation": "linear",
+						"lineWidth": 1,
+						"pointSize": 5,
+						"scaleDistribution": {
+							"type": "linear"
+						},
+						"showPoints": "auto",
+						"spanNulls": false,
+						"stacking": {
+							"group": "A",
+							"mode": "none"
+						},
+						"thresholdsStyle": {
+							"mode": "off"
+						}
+					},
+					"mappings": [],
+					"thresholds": {
+						"mode": "absolute",
+						"steps": [
+							{
+								"color": "green",
+								"value": null
+							},
+							{
+								"color": "red",
+								"value": 80
+							}
+						]
+					}
+				},
+				"overrides": []
+			},
+			"gridPos": {
+				"h": 8,
+				"w": 12,
+				"x": 12,
+				"y": 0
+			},
+			"id": 2,
+			"options": {
+				"legend": {
+					"calcs": [],
+					"displayMode": "list",
+					"placement": "bottom",
+					"showLegend": true
+				},
+				"tooltip": {
+					"mode": "multi",
+					"sort": "none"
+				}
+			},
+			"targets": [
+				{
+					"datasource": {
+						"type": "prometheus",
+						"uid": "prometheus"
+					},
+					"editorMode": "code",
+					"expr": "sum by (pool_type) (rivet_provision_draining_servers{cluster_id=~\"[[cluster_id]]\", datacenter_id=~\"[[datacenter_id]]\", provider_datacenter_id=~\"[[provider_datacenter_id]]\"})",
+					"instant": false,
+					"legendFormat": "__auto",
+					"range": true,
+					"refId": "A"
+				}
+			],
+			"title": "Draining Servers",
+			"type": "timeseries"
+		},
+		{
+			"datasource": {
+				"type": "prometheus",
+				"uid": "prometheus"
+			},
+			"fieldConfig": {
+				"defaults": {
+					"color": {
+						"mode": "palette-classic"
+					},
+					"custom": {
+						"axisCenteredZero": false,
+						"axisColorMode": "text",
+						"axisLabel": "",
+						"axisPlacement": "auto",
+						"barAlignment": 0,
+						"drawStyle": "line",
+						"fillOpacity": 0,
+						"gradientMode": "none",
+						"hideFrom": {
+							"legend": false,
+							"tooltip": false,
+							"viz": false
+						},
+						"insertNulls": false,
+						"lineInterpolation": "linear",
+						"lineWidth": 1,
+						"pointSize": 5,
+						"scaleDistribution": {
+							"type": "linear"
+						},
+						"showPoints": "auto",
+						"spanNulls": false,
+						"stacking": {
+							"group": "A",
+							"mode": "none"
+						},
+						"thresholdsStyle": {
+							"mode": "off"
+						}
+					},
+					"mappings": [],
+					"thresholds": {
+						"mode": "absolute",
+						"steps": [
+							{
+								"color": "green",
+								"value": null
+							},
+							{
+								"color": "red",
+								"value": 80
+							}
+						]
+					}
+				},
+				"overrides": []
+			},
+			"gridPos": {
+				"h": 8,
+				"w": 12,
+				"x": 0,
+				"y": 8
+			},
+			"id": 6,
+			"options": {
+				"legend": {
+					"calcs": [],
+					"displayMode": "list",
+					"placement": "bottom",
+					"showLegend": true
+				},
+				"tooltip": {
+					"mode": "multi",
+					"sort": "none"
+				}
+			},
+			"targets": [
+				{
+					"datasource": {
+						"type": "prometheus",
+						"uid": "prometheus"
+					},
+					"editorMode": "code",
+					"expr": "sum by (pool_type) (rivet_provision_provisioning_servers{cluster_id=~\"[[cluster_id]]\", datacenter_id=~\"[[datacenter_id]]\", provider_datacenter_id=~\"[[provider_datacenter_id]]\"})",
+					"instant": false,
+					"legendFormat": "__auto",
+					"range": true,
+					"refId": "A"
+				}
+			],
+			"title": "Provisioning Servers",
+			"type": "timeseries"
+		},
+		{
+			"datasource": {
+				"type": "prometheus",
+				"uid": "prometheus"
+			},
+			"fieldConfig": {
+				"defaults": {
+					"custom": {
+						"hideFrom": {
+							"legend": false,
+							"tooltip": false,
+							"viz": false
+						},
+						"scaleDistribution": {
+							"type": "linear"
+						}
+					}
+				},
+				"overrides": []
+			},
+			"gridPos": {
+				"h": 8,
+				"w": 12,
+				"x": 12,
+				"y": 8
+			},
+			"id": 9,
+			"options": {
+				"calculate": false,
+				"cellGap": 0,
+				"color": {
+					"exponent": 0.5,
+					"fill": "dark-orange",
+					"mode": "scheme",
+					"reverse": false,
+					"scale": "exponential",
+					"scheme": "RdBu",
+					"steps": 64
+				},
+				"exemplars": {
+					"color": "rgba(255,0,255,0.7)"
+				},
+				"filterValues": {
+					"le": 1e-9
+				},
+				"legend": {
+					"show": true
+				},
+				"rowsFrame": {
+					"layout": "auto"
+				},
+				"tooltip": {
+					"show": true,
+					"yHistogram": true
+				},
+				"yAxis": {
+					"axisPlacement": "left",
+					"reverse": false,
+					"unit": "s"
+				}
+			},
+			"pluginVersion": "10.1.4",
+			"targets": [
+				{
+					"datasource": {
+						"type": "prometheus",
+						"uid": "prometheus"
+					},
+					"editorMode": "code",
+					"expr": "sum(increase(rivet_provision_provision_duration_bucket{cluster_id=~\"[[cluster_id]]\", datacenter_id=~\"[[datacenter_id]]\", provider_datacenter_id=~\"[[provider_datacenter_id]]\"} [$__rate_interval])) by (le)",
+					"format": "heatmap",
+					"instant": false,
+					"legendFormat": "__auto",
+					"range": true,
+					"refId": "A"
+				}
+			],
+			"title": "Server Provision Duration",
+			"type": "heatmap"
+		},
+		{
+			"datasource": {
+				"type": "prometheus",
+				"uid": "prometheus"
+			},
+			"fieldConfig": {
+				"defaults": {
+					"color": {
+						"mode": "palette-classic"
+					},
+					"custom": {
+						"axisCenteredZero": false,
+						"axisColorMode": "text",
+						"axisLabel": "",
+						"axisPlacement": "auto",
+						"barAlignment": 0,
+						"drawStyle": "line",
+						"fillOpacity": 0,
+						"gradientMode": "none",
+						"hideFrom": {
+							"legend": false,
+							"tooltip": false,
+							"viz": false
+						},
+						"insertNulls": false,
+						"lineInterpolation": "linear",
+						"lineWidth": 1,
+						"pointSize": 5,
+						"scaleDistribution": {
+							"type": "linear"
+						},
+						"showPoints": "auto",
+						"spanNulls": false,
+						"stacking": {
+							"group": "A",
+							"mode": "none"
+						},
+						"thresholdsStyle": {
+							"mode": "off"
+						}
+					},
+					"mappings": [],
+					"thresholds": {
+						"mode": "absolute",
+						"steps": [
+							{
+								"color": "green",
+								"value": null
+							},
+							{
+								"color": "red",
+								"value": 80
+							}
+						]
+					}
+				},
+				"overrides": []
+			},
+			"gridPos": {
+				"h": 8,
+				"w": 12,
+				"x": 0,
+				"y": 16
+			},
+			"id": 5,
+			"options": {
+				"legend": {
+					"calcs": [],
+					"displayMode": "list",
+					"placement": "bottom",
+					"showLegend": true
+				},
+				"tooltip": {
+					"mode": "multi",
+					"sort": "none"
+				}
+			},
+			"targets": [
+				{
+					"datasource": {
+						"type": "prometheus",
+						"uid": "prometheus"
+					},
+					"editorMode": "code",
+					"expr": "sum by (pool_type) (rivet_provision_installing_servers{cluster_id=~\"[[cluster_id]]\", datacenter_id=~\"[[datacenter_id]]\", provider_datacenter_id=~\"[[provider_datacenter_id]]\"})",
+					"instant": false,
+					"legendFormat": "__auto",
+					"range": true,
+					"refId": "A"
+				}
+			],
+			"title": "Installing Servers",
+			"type": "timeseries"
+		},
+		{
+			"datasource": {
+				"type": "prometheus",
+				"uid": "prometheus"
+			},
+			"fieldConfig": {
+				"defaults": {
+					"custom": {
+						"hideFrom": {
+							"legend": false,
+							"tooltip": false,
+							"viz": false
+						},
+						"scaleDistribution": {
+							"type": "linear"
+						}
+					}
+				},
+				"overrides": []
+			},
+			"gridPos": {
+				"h": 8,
+				"w": 12,
+				"x": 12,
+				"y": 16
+			},
+			"id": 10,
+			"options": {
+				"calculate": false,
+				"cellGap": 0,
+				"color": {
+					"exponent": 0.5,
+					"fill": "dark-orange",
+					"mode": "scheme",
+					"reverse": false,
+					"scale": "exponential",
+					"scheme": "RdBu",
+					"steps": 64
+				},
+				"exemplars": {
+					"color": "rgba(255,0,255,0.7)"
+				},
+				"filterValues": {
+					"le": 1e-9
+				},
+				"legend": {
+					"show": true
+				},
+				"rowsFrame": {
+					"layout": "auto"
+				},
+				"tooltip": {
+					"show": true,
+					"yHistogram": true
+				},
+				"yAxis": {
+					"axisPlacement": "left",
+					"reverse": false,
+					"unit": "s"
+				}
+			},
+			"pluginVersion": "10.1.4",
+			"targets": [
+				{
+					"datasource": {
+						"type": "prometheus",
+						"uid": "prometheus"
+					},
+					"editorMode": "code",
+					"expr": "sum(increase(rivet_provision_install_duration_bucket{cluster_id=~\"[[cluster_id]]\", datacenter_id=~\"[[datacenter_id]]\", provider_datacenter_id=~\"[[provider_datacenter_id]]\"} [$__rate_interval])) by (le)",
+					"format": "heatmap",
+					"instant": false,
+					"legendFormat": "__auto",
+					"range": true,
+					"refId": "A"
+				}
+			],
+			"title": "Server Install Duration",
+			"type": "heatmap"
+		},
+		{
+			"datasource": {
+				"type": "prometheus",
+				"uid": "prometheus"
+			},
+			"fieldConfig": {
+				"defaults": {
+					"color": {
+						"mode": "palette-classic"
+					},
+					"custom": {
+						"axisCenteredZero": false,
+						"axisColorMode": "text",
+						"axisLabel": "",
+						"axisPlacement": "auto",
+						"barAlignment": 0,
+						"drawStyle": "line",
+						"fillOpacity": 0,
+						"gradientMode": "none",
+						"hideFrom": {
+							"legend": false,
+							"tooltip": false,
+							"viz": false
+						},
+						"insertNulls": false,
+						"lineInterpolation": "linear",
+						"lineWidth": 1,
+						"pointSize": 5,
+						"scaleDistribution": {
+							"type": "linear"
+						},
+						"showPoints": "auto",
+						"spanNulls": false,
+						"stacking": {
+							"group": "A",
+							"mode": "none"
+						},
+						"thresholdsStyle": {
+							"mode": "off"
+						}
+					},
+					"mappings": [],
+					"thresholds": {
+						"mode": "absolute",
+						"steps": [
+							{
+								"color": "green",
+								"value": null
+							},
+							{
+								"color": "red",
+								"value": 80
+							}
+						]
+					}
+				},
+				"overrides": []
+			},
+			"gridPos": {
+				"h": 8,
+				"w": 12,
+				"x": 0,
+				"y": 24
+			},
+			"id": 7,
+			"options": {
+				"legend": {
+					"calcs": [],
+					"displayMode": "list",
+					"placement": "bottom",
+					"showLegend": true
+				},
+				"tooltip": {
+					"mode": "multi",
+					"sort": "none"
+				}
+			},
+			"targets": [
+				{
+					"datasource": {
+						"type": "prometheus",
+						"uid": "prometheus"
+					},
+					"editorMode": "code",
+					"expr": "sum (rivet_provision_nomad_servers{cluster_id=~\"[[cluster_id]]\", datacenter_id=~\"[[datacenter_id]]\", provider_datacenter_id=~\"[[provider_datacenter_id]]\"})",
+					"instant": false,
+					"legendFormat": "Count",
+					"range": true,
+					"refId": "A"
+				}
+			],
+			"title": "Nomad Servers",
+			"type": "timeseries"
+		},
+		{
+			"datasource": {
+				"type": "prometheus",
+				"uid": "prometheus"
+			},
+			"fieldConfig": {
+				"defaults": {
+					"custom": {
+						"hideFrom": {
+							"legend": false,
+							"tooltip": false,
+							"viz": false
+						},
+						"scaleDistribution": {
+							"type": "linear"
+						}
+					}
+				},
+				"overrides": []
+			},
+			"gridPos": {
+				"h": 8,
+				"w": 12,
+				"x": 12,
+				"y": 24
+			},
+			"id": 8,
+			"options": {
+				"calculate": false,
+				"cellGap": 0,
+				"color": {
+					"exponent": 0.5,
+					"fill": "dark-orange",
+					"mode": "scheme",
+					"reverse": false,
+					"scale": "exponential",
+					"scheme": "RdBu",
+					"steps": 64
+				},
+				"exemplars": {
+					"color": "rgba(255,0,255,0.7)"
+				},
+				"filterValues": {
+					"le": 1e-9
+				},
+				"legend": {
+					"show": true
+				},
+				"rowsFrame": {
+					"layout": "auto"
+				},
+				"tooltip": {
+					"show": true,
+					"yHistogram": true
+				},
+				"yAxis": {
+					"axisPlacement": "left",
+					"reverse": false,
+					"unit": "s"
+				}
+			},
+			"pluginVersion": "10.1.4",
+			"targets": [
+				{
+					"datasource": {
+						"type": "prometheus",
+						"uid": "prometheus"
+					},
+					"editorMode": "code",
+					"expr": "sum(increase(rivet_provision_nomad_join_duration_bucket{cluster_id=~\"[[cluster_id]]\", datacenter_id=~\"[[datacenter_id]]\", provider_datacenter_id=~\"[[provider_datacenter_id]]\"} [$__rate_interval])) by (le)",
+					"format": "heatmap",
+					"instant": false,
+					"legendFormat": "__auto",
+					"range": true,
+					"refId": "A"
+				}
+			],
+			"title": "Nomad Join Duration",
+			"type": "heatmap"
+		},
+		{
+			"datasource": {
+				"type": "prometheus",
+				"uid": "prometheus"
+			},
+			"fieldConfig": {
+				"defaults": {
+					"color": {
+						"mode": "palette-classic"
+					},
+					"custom": {
+						"axisCenteredZero": false,
+						"axisColorMode": "text",
+						"axisLabel": "",
+						"axisPlacement": "auto",
+						"barAlignment": 0,
+						"drawStyle": "line",
+						"fillOpacity": 0,
+						"gradientMode": "none",
+						"hideFrom": {
+							"legend": false,
+							"tooltip": false,
+							"viz": false
+						},
+						"insertNulls": false,
+						"lineInterpolation": "linear",
+						"lineWidth": 1,
+						"pointSize": 5,
+						"scaleDistribution": {
+							"type": "linear"
+						},
+						"showPoints": "auto",
+						"spanNulls": false,
+						"stacking": {
+							"group": "A",
+							"mode": "none"
+						},
+						"thresholdsStyle": {
+							"mode": "off"
+						}
+					},
+					"mappings": [],
+					"thresholds": {
+						"mode": "absolute",
+						"steps": [
+							{
+								"color": "green",
+								"value": null
+							},
+							{
+								"color": "red",
+								"value": 80
+							}
+						]
+					}
+				},
+				"overrides": []
+			},
+			"gridPos": {
+				"h": 8,
+				"w": 12,
+				"x": 0,
+				"y": 32
+			},
+			"id": 3,
+			"options": {
+				"legend": {
+					"calcs": [],
+					"displayMode": "list",
+					"placement": "bottom",
+					"showLegend": true
+				},
+				"tooltip": {
+					"mode": "multi",
+					"sort": "none"
+				}
+			},
+			"targets": [
+				{
+					"datasource": {
+						"type": "prometheus",
+						"uid": "prometheus"
+					},
+					"editorMode": "code",
+					"expr": "sum by (pool_type) (rivet_provision_tainted_servers{cluster_id=~\"[[cluster_id]]\", datacenter_id=~\"[[datacenter_id]]\", provider_datacenter_id=~\"[[provider_datacenter_id]]\"})",
+					"instant": false,
+					"legendFormat": "__auto",
+					"range": true,
+					"refId": "A"
+				}
+			],
+			"title": "Tainted Servers",
+			"type": "timeseries"
+		},
+		{
+			"datasource": {
+				"type": "prometheus",
+				"uid": "prometheus"
+			},
+			"fieldConfig": {
+				"defaults": {
+					"color": {
+						"mode": "palette-classic"
+					},
+					"custom": {
+						"axisCenteredZero": false,
+						"axisColorMode": "text",
+						"axisLabel": "",
+						"axisPlacement": "auto",
+						"barAlignment": 0,
+						"drawStyle": "line",
+						"fillOpacity": 0,
+						"gradientMode": "none",
+						"hideFrom": {
+							"legend": false,
+							"tooltip": false,
+							"viz": false
+						},
+						"insertNulls": false,
+						"lineInterpolation": "linear",
+						"lineWidth": 1,
+						"pointSize": 5,
+						"scaleDistribution": {
+							"type": "linear"
+						},
+						"showPoints": "auto",
+						"spanNulls": false,
+						"stacking": {
+							"group": "A",
+							"mode": "none"
+						},
+						"thresholdsStyle": {
+							"mode": "off"
+						}
+					},
+					"mappings": [],
+					"thresholds": {
+						"mode": "absolute",
+						"steps": [
+							{
+								"color": "green",
+								"value": null
+							},
+							{
+								"color": "red",
+								"value": 80
+							}
+						]
+					}
+				},
+				"overrides": []
+			},
+			"gridPos": {
+				"h": 8,
+				"w": 12,
+				"x": 12,
+				"y": 32
+			},
+			"id": 4,
+			"options": {
+				"legend": {
+					"calcs": [],
+					"displayMode": "list",
+					"placement": "bottom",
+					"showLegend": true
+				},
+				"tooltip": {
+					"mode": "multi",
+					"sort": "none"
+				}
+			},
+			"targets": [
+				{
+					"datasource": {
+						"type": "prometheus",
+						"uid": "prometheus"
+					},
+					"editorMode": "code",
+					"expr": "sum by (pool_type) (\n    rivet_provision_tainted_servers{cluster_id=~\"[[cluster_id]]\", datacenter_id=~\"[[datacenter_id]]\", provider_datacenter_id=~\"[[provider_datacenter_id]]\"} and\n    rivet_provision_draining_servers{cluster_id=~\"[[cluster_id]]\", datacenter_id=~\"[[datacenter_id]]\", provider_datacenter_id=~\"[[provider_datacenter_id]]\"}\n)",
+					"instant": false,
+					"legendFormat": "__auto",
+					"range": true,
+					"refId": "A"
+				}
+			],
+			"title": "Draining Tainted Servers",
+			"type": "timeseries"
+		},
+		{
+			"datasource": {
+				"type": "prometheus",
+				"uid": "prometheus"
+			},
+			"fieldConfig": {
+				"defaults": {
+					"color": {
+						"mode": "palette-classic"
+					},
+					"custom": {
+						"axisCenteredZero": false,
+						"axisColorMode": "text",
+						"axisLabel": "",
+						"axisPlacement": "auto",
+						"barAlignment": 0,
+						"drawStyle": "line",
+						"fillOpacity": 25,
+						"gradientMode": "none",
+						"hideFrom": {
+							"legend": false,
+							"tooltip": false,
+							"viz": false
+						},
+						"insertNulls": false,
+						"lineInterpolation": "linear",
+						"lineWidth": 1,
+						"pointSize": 5,
+						"scaleDistribution": {
+							"type": "linear"
+						},
+						"showPoints": "auto",
+						"spanNulls": false,
+						"stacking": {
+							"group": "A",
+							"mode": "normal"
+						},
+						"thresholdsStyle": {
+							"mode": "off"
+						}
+					},
+					"mappings": [],
+					"thresholds": {
+						"mode": "absolute",
+						"steps": [
+							{
+								"color": "green",
+								"value": null
+							},
+							{
+								"color": "red",
+								"value": 80
+							}
+						]
+					},
+					"unit": "percentunit"
+				},
+				"overrides": []
+			},
+			"gridPos": {
+				"h": 8,
+				"w": 12,
+				"x": 0,
+				"y": 40
+			},
+			"id": 13,
+			"options": {
+				"legend": {
+					"calcs": [],
+					"displayMode": "list",
+					"placement": "bottom",
+					"showLegend": true
+				},
+				"tooltip": {
+					"mode": "multi",
+					"sort": "none"
+				}
+			},
+			"targets": [
+				{
+					"datasource": {
+						"type": "prometheus",
+						"uid": "prometheus"
+					},
+					"editorMode": "code",
+					"expr": "sum by (server_id) (nomad_client_allocated_cpu{datacenter_id=~\"[[datacenter_id]]\"}) /\nsum by (server_id) (nomad_client_allocated_cpu{datacenter_id=~\"[[datacenter_id]]\"} + nomad_client_unallocated_cpu{datacenter_id=~\"[[datacenter_id]]\"})",
+					"instant": false,
+					"legendFormat": "__auto",
+					"range": true,
+					"refId": "A"
+				}
+			],
+			"title": "Job Allocated CPU",
+			"type": "timeseries"
+		},
+		{
+			"datasource": {
+				"type": "prometheus",
+				"uid": "prometheus"
+			},
+			"fieldConfig": {
+				"defaults": {
+					"color": {
+						"mode": "palette-classic"
+					},
+					"custom": {
+						"axisCenteredZero": false,
+						"axisColorMode": "text",
+						"axisLabel": "",
+						"axisPlacement": "auto",
+						"barAlignment": 0,
+						"drawStyle": "line",
+						"fillOpacity": 25,
+						"gradientMode": "none",
+						"hideFrom": {
+							"legend": false,
+							"tooltip": false,
+							"viz": false
+						},
+						"insertNulls": false,
+						"lineInterpolation": "linear",
+						"lineWidth": 1,
+						"pointSize": 5,
+						"scaleDistribution": {
+							"type": "linear"
+						},
+						"showPoints": "auto",
+						"spanNulls": false,
+						"stacking": {
+							"group": "A",
+							"mode": "normal"
+						},
+						"thresholdsStyle": {
+							"mode": "off"
+						}
+					},
+					"mappings": [],
+					"thresholds": {
+						"mode": "absolute",
+						"steps": [
+							{
+								"color": "green",
+								"value": null
+							},
+							{
+								"color": "red",
+								"value": 80
+							}
+						]
+					},
+					"unit": "percentunit"
+				},
+				"overrides": []
+			},
+			"gridPos": {
+				"h": 8,
+				"w": 12,
+				"x": 12,
+				"y": 40
+			},
+			"id": 14,
+			"options": {
+				"legend": {
+					"calcs": [],
+					"displayMode": "list",
+					"placement": "bottom",
+					"showLegend": true
+				},
+				"tooltip": {
+					"mode": "multi",
+					"sort": "none"
+				}
+			},
+			"targets": [
+				{
+					"datasource": {
+						"type": "prometheus",
+						"uid": "prometheus"
+					},
+					"editorMode": "code",
+					"expr": "sum by (server_id) (nomad_client_allocated_memory{datacenter_id=~\"[[datacenter_id]]\"}) /\nsum by (server_id) (nomad_client_allocated_memory{datacenter_id=~\"[[datacenter_id]]\"} + nomad_client_unallocated_memory{datacenter_id=~\"[[datacenter_id]]\"})",
+					"instant": false,
+					"legendFormat": "__auto",
+					"range": true,
+					"refId": "A"
+				}
+			],
+			"title": "Job Allocated Memory",
+			"type": "timeseries"
+		},
+		{
+			"datasource": {
+				"type": "prometheus",
+				"uid": "prometheus"
+			},
+			"fieldConfig": {
+				"defaults": {
+					"color": {
+						"mode": "palette-classic"
+					},
+					"custom": {
+						"axisCenteredZero": false,
+						"axisColorMode": "text",
+						"axisLabel": "",
+						"axisPlacement": "auto",
+						"barAlignment": 0,
+						"drawStyle": "line",
+						"fillOpacity": 9,
+						"gradientMode": "none",
+						"hideFrom": {
+							"legend": false,
+							"tooltip": false,
+							"viz": false
+						},
+						"insertNulls": false,
+						"lineInterpolation": "linear",
+						"lineWidth": 1,
+						"pointSize": 5,
+						"scaleDistribution": {
+							"type": "linear"
+						},
+						"showPoints": "auto",
+						"spanNulls": false,
+						"stacking": {
+							"group": "A",
+							"mode": "normal"
+						},
+						"thresholdsStyle": {
+							"mode": "off"
+						}
+					},
+					"mappings": [],
+					"thresholds": {
+						"mode": "absolute",
+						"steps": [
+							{
+								"color": "green",
+								"value": null
+							},
+							{
+								"color": "red",
+								"value": 80
+							}
+						]
+					},
+					"unit": "percent"
+				},
+				"overrides": []
+			},
+			"gridPos": {
+				"h": 8,
+				"w": 12,
+				"x": 0,
+				"y": 48
+			},
+			"id": 11,
+			"options": {
+				"legend": {
+					"calcs": [],
+					"displayMode": "list",
+					"placement": "bottom",
+					"showLegend": true
+				},
+				"tooltip": {
+					"mode": "multi",
+					"sort": "none"
+				}
+			},
+			"targets": [
+				{
+					"datasource": {
+						"type": "prometheus",
+						"uid": "prometheus"
+					},
+					"editorMode": "code",
+					"exemplar": false,
+					"expr": "last_over_time((\n\tsum by (server_id) (\n\t\tirate(\n\t\t\tnode_cpu_seconds_total{\n\t\t\t\tdatacenter_id=~\"[[datacenter_id]]\",\n\t\t\t\tpool_type=\"gg\",\n\n\t\t\t\tmode!=\"idle\",\n\t\t\t\tmode!=\"iowait\",\n\t\t\t\tmode!=\"steal\"\n\t\t\t}\n\t\t\t[5m]\n\t\t)\n\t) * 100\n) [15m:15s])",
+					"instant": false,
+					"interval": "",
+					"legendFormat": "__auto",
+					"range": true,
+					"refId": "A"
+				}
+			],
+			"title": "GG CPU",
+			"type": "timeseries"
+		},
+		{
+			"datasource": {
+				"type": "prometheus",
+				"uid": "prometheus"
+			},
+			"fieldConfig": {
+				"defaults": {
+					"color": {
+						"mode": "palette-classic"
+					},
+					"custom": {
+						"axisCenteredZero": false,
+						"axisColorMode": "text",
+						"axisLabel": "",
+						"axisPlacement": "auto",
+						"barAlignment": 0,
+						"drawStyle": "line",
+						"fillOpacity": 9,
+						"gradientMode": "none",
+						"hideFrom": {
+							"legend": false,
+							"tooltip": false,
+							"viz": false
+						},
+						"insertNulls": false,
+						"lineInterpolation": "linear",
+						"lineWidth": 1,
+						"pointSize": 5,
+						"scaleDistribution": {
+							"type": "linear"
+						},
+						"showPoints": "auto",
+						"spanNulls": false,
+						"stacking": {
+							"group": "A",
+							"mode": "normal"
+						},
+						"thresholdsStyle": {
+							"mode": "off"
+						}
+					},
+					"mappings": [],
+					"thresholds": {
+						"mode": "absolute",
+						"steps": [
+							{
+								"color": "green",
+								"value": null
+							},
+							{
+								"color": "red",
+								"value": 80
+							}
+						]
+					},
+					"unit": "percent"
+				},
+				"overrides": []
+			},
+			"gridPos": {
+				"h": 8,
+				"w": 12,
+				"x": 12,
+				"y": 48
+			},
+			"id": 15,
+			"options": {
+				"legend": {
+					"calcs": [],
+					"displayMode": "list",
+					"placement": "bottom",
+					"showLegend": true
+				},
+				"tooltip": {
+					"mode": "multi",
+					"sort": "none"
+				}
+			},
+			"targets": [
+				{
+					"datasource": {
+						"type": "prometheus",
+						"uid": "prometheus"
+					},
+					"editorMode": "code",
+					"exemplar": false,
+					"expr": "sum by (server_id) (\n    node_memory_Active_bytes{\n        datacenter_id=~\"[[datacenter_id]]\",\n        pool_type=\"gg\"\n    } /\n    node_memory_MemTotal_bytes{\n        datacenter_id=~\"[[datacenter_id]]\",\n        pool_type=\"gg\"\n    } *\n    100\n)",
+					"instant": false,
+					"interval": "",
+					"legendFormat": "__auto",
+					"range": true,
+					"refId": "A"
+				}
+			],
+			"title": "GG Memory",
+			"type": "timeseries"
+		},
+		{
+			"datasource": {
+				"type": "prometheus",
+				"uid": "prometheus"
+			},
+			"fieldConfig": {
+				"defaults": {
+					"color": {
+						"mode": "palette-classic"
+					},
+					"custom": {
+						"axisCenteredZero": false,
+						"axisColorMode": "text",
+						"axisLabel": "",
+						"axisPlacement": "auto",
+						"barAlignment": 0,
+						"drawStyle": "line",
+						"fillOpacity": 9,
+						"gradientMode": "none",
+						"hideFrom": {
+							"legend": false,
+							"tooltip": false,
+							"viz": false
+						},
+						"insertNulls": false,
+						"lineInterpolation": "linear",
+						"lineWidth": 1,
+						"pointSize": 5,
+						"scaleDistribution": {
+							"type": "linear"
+						},
+						"showPoints": "auto",
+						"spanNulls": false,
+						"stacking": {
+							"group": "A",
+							"mode": "normal"
+						},
+						"thresholdsStyle": {
+							"mode": "off"
+						}
+					},
+					"mappings": [],
+					"thresholds": {
+						"mode": "absolute",
+						"steps": [
+							{
+								"color": "green",
+								"value": null
+							},
+							{
+								"color": "red",
+								"value": 80
+							}
+						]
+					},
+					"unit": "percent"
+				},
+				"overrides": []
+			},
+			"gridPos": {
+				"h": 8,
+				"w": 12,
+				"x": 0,
+				"y": 56
+			},
+			"id": 16,
+			"options": {
+				"legend": {
+					"calcs": [],
+					"displayMode": "list",
+					"placement": "bottom",
+					"showLegend": true
+				},
+				"tooltip": {
+					"mode": "multi",
+					"sort": "none"
+				}
+			},
+			"targets": [
+				{
+					"datasource": {
+						"type": "prometheus",
+						"uid": "prometheus"
+					},
+					"editorMode": "code",
+					"exemplar": false,
+					"expr": "last_over_time((\n\tsum by (server_id) (\n\t\tirate(\n\t\t\tnode_cpu_seconds_total{\n\t\t\t\tdatacenter_id=~\"[[datacenter_id]]\",\n\t\t\t\tpool_type=\"ats\",\n\n\t\t\t\tmode!=\"idle\",\n\t\t\t\tmode!=\"iowait\",\n\t\t\t\tmode!=\"steal\"\n\t\t\t}\n\t\t\t[5m]\n\t\t)\n\t) * 100\n) [15m:15s])",
+					"instant": false,
+					"interval": "",
+					"legendFormat": "__auto",
+					"range": true,
+					"refId": "A"
+				}
+			],
+			"title": "ATS CPU",
+			"type": "timeseries"
+		},
+		{
+			"datasource": {
+				"type": "prometheus",
+				"uid": "prometheus"
+			},
+			"fieldConfig": {
+				"defaults": {
+					"color": {
+						"mode": "palette-classic"
+					},
+					"custom": {
+						"axisCenteredZero": false,
+						"axisColorMode": "text",
+						"axisLabel": "",
+						"axisPlacement": "auto",
+						"barAlignment": 0,
+						"drawStyle": "line",
+						"fillOpacity": 9,
+						"gradientMode": "none",
+						"hideFrom": {
+							"legend": false,
+							"tooltip": false,
+							"viz": false
+						},
+						"insertNulls": false,
+						"lineInterpolation": "linear",
+						"lineWidth": 1,
+						"pointSize": 5,
+						"scaleDistribution": {
+							"type": "linear"
+						},
+						"showPoints": "auto",
+						"spanNulls": false,
+						"stacking": {
+							"group": "A",
+							"mode": "normal"
+						},
+						"thresholdsStyle": {
+							"mode": "off"
+						}
+					},
+					"mappings": [],
+					"thresholds": {
+						"mode": "absolute",
+						"steps": [
+							{
+								"color": "green",
+								"value": null
+							},
+							{
+								"color": "red",
+								"value": 80
+							}
+						]
+					},
+					"unit": "percent"
+				},
+				"overrides": []
+			},
+			"gridPos": {
+				"h": 8,
+				"w": 12,
+				"x": 12,
+				"y": 56
+			},
+			"id": 17,
+			"options": {
+				"legend": {
+					"calcs": [],
+					"displayMode": "list",
+					"placement": "bottom",
+					"showLegend": true
+				},
+				"tooltip": {
+					"mode": "multi",
+					"sort": "none"
+				}
+			},
+			"targets": [
+				{
+					"datasource": {
+						"type": "prometheus",
+						"uid": "prometheus"
+					},
+					"editorMode": "code",
+					"exemplar": false,
+					"expr": "sum by (server_id) (\n    node_memory_Active_bytes{\n        datacenter_id=~\"[[datacenter_id]]\",\n        pool_type=\"ats\"\n    } /\n    node_memory_MemTotal_bytes{\n        datacenter_id=~\"[[datacenter_id]]\",\n        pool_type=\"ats\"\n    } *\n    100\n)",
+					"instant": false,
+					"interval": "",
+					"legendFormat": "__auto",
+					"range": true,
+					"refId": "A"
+				}
+			],
+			"title": "ATS Memory",
+			"type": "timeseries"
+		}
+	],
+	"refresh": "auto",
+	"schemaVersion": 38,
+	"style": "dark",
+	"tags": [],
+	"templating": {
+		"list": [
+			{
+				"current": {
+					"selected": true,
+					"text": ["All"],
+					"value": ["$__all"]
+				},
+				"datasource": {
+					"type": "prometheus",
+					"uid": "prometheus"
+				},
+				"definition": "label_values(rivet_provision_active_servers,cluster_id)",
+				"hide": 0,
+				"includeAll": true,
+				"label": "Cluster ID",
+				"multi": true,
+				"name": "cluster_id",
+				"options": [],
+				"query": {
+					"query": "label_values(rivet_provision_active_servers,cluster_id)",
+					"refId": "PrometheusVariableQueryEditor-VariableQuery"
+				},
+				"refresh": 1,
+				"regex": "",
+				"skipUrlSync": false,
+				"sort": 0,
+				"type": "query"
+			},
+			{
+				"current": {
+					"selected": true,
+					"text": ["All"],
+					"value": ["$__all"]
+				},
+				"datasource": {
+					"type": "prometheus",
+					"uid": "prometheus"
+				},
+				"definition": "label_values(rivet_provision_active_servers,datacenter_id)",
+				"hide": 0,
+				"includeAll": true,
+				"label": "Datacenter ID",
+				"multi": true,
+				"name": "datacenter_id",
+				"options": [],
+				"query": {
+					"query": "label_values(rivet_provision_active_servers,datacenter_id)",
+					"refId": "PrometheusVariableQueryEditor-VariableQuery"
+				},
+				"refresh": 1,
+				"regex": "",
+				"skipUrlSync": false,
+				"sort": 0,
+				"type": "query"
+			},
+			{
+				"current": {
+					"selected": true,
+					"text": ["us-southeast"],
+					"value": ["us-southeast"]
+				},
+				"datasource": {
+					"type": "prometheus",
+					"uid": "prometheus"
+				},
+				"definition": "label_values(rivet_provision_active_servers,provider_datacenter_id)",
+				"hide": 0,
+				"includeAll": true,
+				"label": "Provider Region",
+				"multi": true,
+				"name": "provider_datacenter_id",
+				"options": [],
+				"query": {
+					"query": "label_values(rivet_provision_active_servers,provider_datacenter_id)",
+					"refId": "PrometheusVariableQueryEditor-VariableQuery"
+				},
+				"refresh": 1,
+				"regex": "",
+				"skipUrlSync": false,
+				"sort": 0,
+				"type": "query"
+			}
+		]
+	},
+	"time": {
+		"from": "now-24h",
+		"to": "now"
+	},
+	"timepicker": {},
+	"timezone": "",
+	"title": "Provisioning",
+	"uid": "aa49514c-cb94-47f0-ae0f-12a4ec20069a",
+	"version": 21,
+	"weekStart": ""
+}
diff --git a/lib/metrics/src/buckets.rs b/lib/metrics/src/buckets.rs
index facc7b9b02..31b09e3128 100644
--- a/lib/metrics/src/buckets.rs
+++ b/lib/metrics/src/buckets.rs
@@ -5,3 +5,11 @@ pub const BUCKETS: &[f64] = &[
 	0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, // Added
 	25.0, 50.0, 100.0, 250.0, 500.0,
 ];
+
+pub const PROVISION_BUCKETS: &[f64] = &[
+	0.5,
+	1.0, 2.5, 5.0,
+	10.0, 25.0, 35.0, 50.0, 75.0,
+	100.0, 125.0, 250.0, 500.0,
+	1000.0,
+];
diff --git a/lib/metrics/src/lib.rs b/lib/metrics/src/lib.rs
index 207aca3f02..196ed2d48b 100644
--- a/lib/metrics/src/lib.rs
+++ b/lib/metrics/src/lib.rs
@@ -3,6 +3,7 @@ mod registry;
 mod server;
 
 pub use buckets::BUCKETS;
+pub use buckets::PROVISION_BUCKETS;
 pub use prometheus;
 pub use registry::REGISTRY;
 pub use server::run_standalone;
diff --git a/svc/Cargo.lock b/svc/Cargo.lock
index 5f816a344d..300f8604c1 100644
--- a/svc/Cargo.lock
+++ b/svc/Cargo.lock
@@ -2542,6 +2542,25 @@ dependencies = [
  "sqlx",
 ]
 
+[[package]]
+name = "cluster-metrics-publish"
+version = "0.0.1"
+dependencies = [
+ "chirp-client",
+ "chirp-worker",
+ "cluster-datacenter-get",
+ "rivet-connection",
+ "rivet-health-checks",
+ "rivet-metrics",
+ "rivet-operation",
+ "rivet-runtime",
+ "rivet-util-cluster",
+ "sqlx",
+ "tokio",
+ "tracing",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "cluster-server-get"
 version = "0.0.1"
@@ -7527,6 +7546,8 @@ name = "rivet-util-cluster"
 version = "0.1.0"
 dependencies = [
  "hex",
+ "lazy_static",
+ "rivet-metrics",
  "rivet-util",
  "sha2",
  "tokio",
diff --git a/svc/Cargo.toml b/svc/Cargo.toml
index 4c1c8457a9..332e341d95 100644
--- a/svc/Cargo.toml
+++ b/svc/Cargo.toml
@@ -75,6 +75,7 @@ members = [
 	"pkg/cluster/standalone/datacenter-tls-renew",
 	"pkg/cluster/standalone/default-update",
 	"pkg/cluster/standalone/gc",
+	"pkg/cluster/standalone/metrics-publish",
 	"pkg/cluster/worker",
 	"pkg/custom-user-avatar/ops/list-for-game",
 	"pkg/custom-user-avatar/ops/upload-complete",
diff --git a/svc/pkg/cluster/db/cluster/migrations/20231201000927_init.up.sql b/svc/pkg/cluster/db/cluster/migrations/20231201000927_init.up.sql
index 0fc2106d7b..a4ada64c5d 100644
--- a/svc/pkg/cluster/db/cluster/migrations/20231201000927_init.up.sql
+++ b/svc/pkg/cluster/db/cluster/migrations/20231201000927_init.up.sql
@@ -50,6 +50,7 @@ CREATE TABLE servers (
 	nomad_node_id TEXT,
 
 	create_ts INT NOT NULL,
+	provision_complete_ts INT,
 	install_complete_ts INT,
 	nomad_join_ts INT,
 	-- Null if not draining
diff --git a/svc/pkg/cluster/standalone/metrics-publish/Cargo.toml b/svc/pkg/cluster/standalone/metrics-publish/Cargo.toml
new file mode 100644
index 0000000000..206f7ec62e
--- /dev/null
+++ b/svc/pkg/cluster/standalone/metrics-publish/Cargo.toml
@@ -0,0 +1,28 @@
+[package]
+name = "cluster-metrics-publish"
+version = "0.0.1"
+edition = "2018"
+authors = ["Rivet Gaming, LLC <developer@rivet.gg>"]
+license = "Apache-2.0"
+
+[dependencies]
+chirp-client = { path = "../../../../../lib/chirp/client" }
+rivet-connection = { path = "../../../../../lib/connection" }
+rivet-health-checks = { path = "../../../../../lib/health-checks" }
+rivet-metrics = { path = "../../../../../lib/metrics" }
+rivet-operation = { path = "../../../../../lib/operation/core" }
+rivet-runtime = { path = "../../../../../lib/runtime" }
+tokio = { version = "1.29", features = ["full"] }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] }
+util-cluster = { package = "rivet-util-cluster", path = "../../util" }
+
+cluster-datacenter-get = { path = "../../ops/datacenter-get" }
+
+[dependencies.sqlx]
+version = "0.7"
+default-features = false
+
+[dev-dependencies]
+chirp-worker = { path = "../../../../../lib/chirp/worker" }
+util-cluster = { package = "rivet-util-cluster", path = "../../util" }
diff --git a/svc/pkg/cluster/standalone/metrics-publish/Service.toml b/svc/pkg/cluster/standalone/metrics-publish/Service.toml
new file mode 100644
index 0000000000..a6eceac5ca
--- /dev/null
+++ b/svc/pkg/cluster/standalone/metrics-publish/Service.toml
@@ -0,0 +1,8 @@
+[service]
+name = "cluster-metrics-publish"
+
+[runtime]
+kind = "rust"
+
+[headless]
+singleton = true
diff --git a/svc/pkg/cluster/standalone/metrics-publish/src/lib.rs b/svc/pkg/cluster/standalone/metrics-publish/src/lib.rs
new file mode 100644
index 0000000000..86cfbfcf2e
--- /dev/null
+++ b/svc/pkg/cluster/standalone/metrics-publish/src/lib.rs
@@ -0,0 +1,209 @@
+use std::convert::{TryFrom, TryInto};
+
+use backend::cluster::PoolType::*;
+use proto::backend;
+use rivet_operation::prelude::*;
+use util_cluster::metrics;
+
+#[derive(sqlx::FromRow)]
+struct ServerRow {
+	datacenter_id: Uuid,
+	pool_type: i64,
+	is_provisioned: bool,
+	is_installed: bool,
+	has_nomad_node: bool,
+	is_draining: bool,
+	is_drained: bool,
+	is_tainted: bool,
+}
+
+struct Server {
+	datacenter_id: Uuid,
+	pool_type: backend::cluster::PoolType,
+	is_provisioned: bool,
+	is_installed: bool,
+	has_nomad_node: bool,
+	is_draining: bool,
+	is_tainted: bool,
+}
+
+impl TryFrom<ServerRow> for Server {
+	type Error = GlobalError;
+
+	fn try_from(value: ServerRow) -> GlobalResult<Self> {
+		Ok(Server {
+			datacenter_id: value.datacenter_id,
+			pool_type: unwrap!(backend::cluster::PoolType::from_i32(value.pool_type as i32)),
+			is_provisioned: value.is_provisioned,
+			is_installed: value.is_installed,
+			has_nomad_node: value.has_nomad_node,
+			is_tainted: value.is_tainted,
+			is_draining: value.is_draining && !value.is_drained,
+		})
+	}
+}
+
+#[tracing::instrument(skip_all)]
+pub async fn run_from_env(_ts: i64, pools: rivet_pools::Pools) -> GlobalResult<()> {
+	let client =
+		chirp_client::SharedClient::from_env(pools.clone())?.wrap_new("cluster-metrics-publish");
+	let cache = rivet_cache::CacheInner::from_env(pools.clone())?;
+	let ctx = OperationContext::new(
+		"cluster-metrics-publish".into(),
+		std::time::Duration::from_secs(60),
+		rivet_connection::Connection::new(client, pools, cache),
+		Uuid::new_v4(),
+		Uuid::new_v4(),
+		util::timestamp::now(),
+		util::timestamp::now(),
+		(),
+		Vec::new(),
+	);
+
+	let servers = select_servers(&ctx).await?;
+
+	let datacenters_res = op!([ctx] cluster_datacenter_get {
+		datacenter_ids: servers
+			.iter()
+			.map(|s| s.datacenter_id.into())
+			.collect::<Vec<_>>(),
+	})
+	.await?;
+
+	for dc in &datacenters_res.datacenters {
+		insert_metrics(dc, &servers)?;
+	}
+
+	Ok(())
+}
+
+async fn select_servers(ctx: &OperationContext<()>) -> GlobalResult<Vec<Server>> {
+	let servers = sql_fetch_all!(
+		[ctx, ServerRow]
+		"
+		SELECT
+			datacenter_id, pool_type,
+			(provider_server_id IS NOT NULL) AS is_provisioned,
+			(install_complete_ts IS NOT NULL) AS is_installed,
+			(nomad_node_id IS NOT NULL) AS has_nomad_node,
+			(drain_ts IS NOT NULL) AS is_draining,
+			(drain_complete_ts IS NOT NULL) AS is_drained,
+			(taint_ts IS NOT NULL) AS is_tainted
+		FROM db_cluster.servers AS OF SYSTEM TIME '-5s'
+		WHERE
+			-- Filters out servers that are being destroyed/already destroyed
+			cloud_destroy_ts IS NULL
+		",
+	)
+	.await?;
+
+	servers
+		.into_iter()
+		.map(TryInto::try_into)
+		.collect::<GlobalResult<Vec<_>>>()
+}
+
+fn insert_metrics(dc: &backend::cluster::Datacenter, servers: &[Server]) -> GlobalResult<()> {
+	let datacenter_id = unwrap_ref!(dc.datacenter_id).as_uuid();
+	let servers_in_dc = servers.iter().filter(|s| s.datacenter_id == datacenter_id);
+
+	let datacenter_id = datacenter_id.to_string();
+	let cluster_id = unwrap_ref!(dc.cluster_id).as_uuid().to_string();
+
+	let servers_per_pool = [
+		(
+			Job,
+			servers_in_dc
+				.clone()
+				.filter(|s| matches!(s.pool_type, Job))
+				.collect::<Vec<_>>(),
+		),
+		(
+			Gg,
+			servers_in_dc
+				.clone()
+				.filter(|s| matches!(s.pool_type, Gg))
+				.collect::<Vec<_>>(),
+		),
+		(
+			Ats,
+			servers_in_dc
+				.clone()
+				.filter(|s| matches!(s.pool_type, Ats))
+				.collect::<Vec<_>>(),
+		),
+	];
+
+	// Aggregate all states per pool type
+	for (pool_type, servers) in servers_per_pool {
+		let mut provisioning = 0;
+		let mut installing = 0;
+		let mut active = 0;
+		let mut nomad = 0;
+		let mut draining = 0;
+		let mut tainted = 0;
+
+		for server in servers {
+			if server.is_draining {
+				draining += 1;
+			} else if server.is_provisioned {
+				if server.is_installed {
+					active += 1;
+
+					if server.has_nomad_node {
+						nomad += 1;
+					}
+				} else {
+					installing += 1;
+				}
+			} else {
+				provisioning += 1;
+			}
+
+			if server.is_tainted {
+				tainted += 1;
+			}
+		}
+
+		let labels = [
+			cluster_id.as_str(),
+			datacenter_id.as_str(),
+			&dc.provider_datacenter_id,
+			&dc.name_id,
+			match pool_type {
+				Job => "job",
+				Gg => "gg",
+				Ats => "ats",
+			},
+		];
+
+		metrics::PROVISIONING_SERVERS
+			.with_label_values(&labels)
+			.set(provisioning);
+		metrics::INSTALLING_SERVERS
+			.with_label_values(&labels)
+			.set(installing);
+		metrics::ACTIVE_SERVERS
+			.with_label_values(&labels)
+			.set(active);
+		metrics::DRAINING_SERVERS
+			.with_label_values(&labels)
+			.set(draining);
+		metrics::TAINTED_SERVERS
+			.with_label_values(&labels)
+			.set(tainted);
+
+		if let Job = pool_type {
+			metrics::NOMAD_SERVERS
+				.with_label_values(&[
+					&cluster_id,
+					&datacenter_id,
+					&dc.provider_datacenter_id,
+					&dc.name_id,
+				])
+				.set(nomad);
+		}
+	}
+
+	Ok(())
+}
diff --git a/svc/pkg/cluster/standalone/metrics-publish/src/main.rs b/svc/pkg/cluster/standalone/metrics-publish/src/main.rs
new file mode 100644
index 0000000000..aafcac0bca
--- /dev/null
+++ b/svc/pkg/cluster/standalone/metrics-publish/src/main.rs
@@ -0,0 +1,31 @@
+use std::time::Duration;
+
+use rivet_operation::prelude::*;
+
+fn main() -> GlobalResult<()> {
+	rivet_runtime::run(start()).unwrap()
+}
+
+async fn start() -> GlobalResult<()> {
+	let pools = rivet_pools::from_env("cluster-metrics-publish").await?;
+
+	tokio::task::Builder::new()
+		.name("cluster_metrics_publish::health_checks")
+		.spawn(rivet_health_checks::run_standalone(
+			rivet_health_checks::Config {
+				pools: Some(pools.clone()),
+			},
+		))?;
+
+	tokio::task::Builder::new()
+		.name("cluster_metrics_publish::metrics")
+		.spawn(rivet_metrics::run_standalone())?;
+
+	let mut interval = tokio::time::interval(Duration::from_secs(15));
+	loop {
+		interval.tick().await;
+
+		let ts = util::timestamp::now();
+		cluster_metrics_publish::run_from_env(ts, pools.clone()).await?;
+	}
+}
diff --git a/svc/pkg/cluster/standalone/metrics-publish/tests/integration.rs b/svc/pkg/cluster/standalone/metrics-publish/tests/integration.rs
new file mode 100644
index 0000000000..6c8ea4d0f2
--- /dev/null
+++ b/svc/pkg/cluster/standalone/metrics-publish/tests/integration.rs
@@ -0,0 +1 @@
+// TODO:
diff --git a/svc/pkg/cluster/util/Cargo.toml b/svc/pkg/cluster/util/Cargo.toml
index 962aedb730..f3bdbec3c2 100644
--- a/svc/pkg/cluster/util/Cargo.toml
+++ b/svc/pkg/cluster/util/Cargo.toml
@@ -6,6 +6,8 @@ authors = ["Rivet Gaming, LLC <developer@rivet.gg>"]
 license = "Apache-2.0"
 
 [dependencies]
+lazy_static = "1.4"
+rivet-metrics = { path = "../../../../lib/metrics" }
 rivet-util = { path = "../../../../lib/util/core" }
 types = { path = "../../../../lib/types/core" }
 uuid = { version = "1", features = ["v4", "serde"] }
diff --git a/svc/pkg/cluster/util/src/lib.rs b/svc/pkg/cluster/util/src/lib.rs
index 02298a3df0..dcb9320232 100644
--- a/svc/pkg/cluster/util/src/lib.rs
+++ b/svc/pkg/cluster/util/src/lib.rs
@@ -2,6 +2,7 @@ use types::rivet::backend::{self, pkg::*};
 use uuid::Uuid;
 
 pub mod test;
+pub mod metrics;
 
 // Use the hash of the server install script in the image variant so that if the install scripts are updated
 // we won't be using the old image anymore
diff --git a/svc/pkg/cluster/util/src/metrics.rs b/svc/pkg/cluster/util/src/metrics.rs
new file mode 100644
index 0000000000..5fe1dfe614
--- /dev/null
+++ b/svc/pkg/cluster/util/src/metrics.rs
@@ -0,0 +1,62 @@
+use rivet_metrics::{prometheus::*, PROVISION_BUCKETS, REGISTRY};
+
+lazy_static::lazy_static! {
+	pub static ref PROVISIONING_SERVERS: IntGaugeVec = register_int_gauge_vec_with_registry!(
+		"provision_provisioning_servers",
+		"Servers being provisioned.",
+		&["cluster_id", "datacenter_id", "provider_datacenter_id", "datacenter_name_id", "pool_type"],
+		*REGISTRY,
+	).unwrap();
+	pub static ref INSTALLING_SERVERS: IntGaugeVec = register_int_gauge_vec_with_registry!(
+		"provision_installing_servers",
+		"Servers currently installing.",
+		&["cluster_id", "datacenter_id", "provider_datacenter_id", "datacenter_name_id", "pool_type"],
+		*REGISTRY,
+	).unwrap();
+	pub static ref ACTIVE_SERVERS: IntGaugeVec = register_int_gauge_vec_with_registry!(
+		"provision_active_servers",
+		"Servers that are completely installed.",
+		&["cluster_id", "datacenter_id", "provider_datacenter_id", "datacenter_name_id", "pool_type"],
+		*REGISTRY,
+	).unwrap();
+	pub static ref NOMAD_SERVERS: IntGaugeVec = register_int_gauge_vec_with_registry!(
+		"provision_nomad_servers",
+		"Job servers with nomad connected.",
+		&["cluster_id", "datacenter_id", "provider_datacenter_id", "datacenter_name_id"],
+		*REGISTRY,
+	).unwrap();
+	pub static ref DRAINING_SERVERS: IntGaugeVec = register_int_gauge_vec_with_registry!(
+		"provision_draining_servers",
+		"Servers being drained.",
+		&["cluster_id", "datacenter_id", "provider_datacenter_id", "datacenter_name_id", "pool_type"],
+		*REGISTRY,
+	).unwrap();
+	pub static ref TAINTED_SERVERS: IntGaugeVec = register_int_gauge_vec_with_registry!(
+		"provision_tainted_servers",
+		"Tainted servers.",
+		&["cluster_id", "datacenter_id", "provider_datacenter_id", "datacenter_name_id", "pool_type"],
+		*REGISTRY,
+	).unwrap();
+
+	pub static ref PROVISION_DURATION: HistogramVec = register_histogram_vec_with_registry!(
+		"provision_provision_duration",
+		"Time from created to provisioned.",
+		&["cluster_id", "datacenter_id", "provider_datacenter_id", "datacenter_name_id", "pool_type"],
+		PROVISION_BUCKETS.to_vec(),
+		*REGISTRY,
+	).unwrap();
+	pub static ref INSTALL_DURATION: HistogramVec = register_histogram_vec_with_registry!(
+		"provision_install_duration",
+		"Time from provisioned to installed.",
+		&["cluster_id", "datacenter_id", "provider_datacenter_id", "datacenter_name_id", "pool_type"],
+		PROVISION_BUCKETS.to_vec(),
+		*REGISTRY,
+	).unwrap();
+	pub static ref NOMAD_JOIN_DURATION: HistogramVec = register_histogram_vec_with_registry!(
+		"provision_nomad_join_duration",
+		"Time from installed to nomad joined.",
+		&["cluster_id", "datacenter_id", "provider_datacenter_id", "datacenter_name_id"],
+		PROVISION_BUCKETS.to_vec(),
+		*REGISTRY,
+	).unwrap();
+}
diff --git a/svc/pkg/cluster/worker/src/workers/datacenter_scale.rs b/svc/pkg/cluster/worker/src/workers/datacenter_scale.rs
index 2a01340f51..d3c12df24d 100644
--- a/svc/pkg/cluster/worker/src/workers/datacenter_scale.rs
+++ b/svc/pkg/cluster/worker/src/workers/datacenter_scale.rs
@@ -1,3 +1,5 @@
+use std::convert::{TryFrom, TryInto};
+
 // TERMINOLOGY:
 //
 // server: a non-destroyed non-tainted server
@@ -38,8 +40,29 @@ struct Server {
 	pool_type: backend::cluster::PoolType,
 	is_installed: bool,
 	has_nomad_node: bool,
-	is_tainted: bool,
 	drain_state: DrainState,
+	is_tainted: bool,
+}
+
+impl TryFrom<ServerRow> for Server {
+	type Error = GlobalError;
+
+	fn try_from(value: ServerRow) -> GlobalResult<Self> {
+		Ok(Server {
+			server_id: value.server_id,
+			pool_type: unwrap!(backend::cluster::PoolType::from_i32(value.pool_type as i32)),
+			is_installed: value.is_installed,
+			has_nomad_node: value.has_nomad_node,
+			is_tainted: value.is_tainted,
+			drain_state: if value.is_drained {
+				DrainState::Complete
+			} else if value.is_draining {
+				DrainState::Draining
+			} else {
+				DrainState::None
+			},
+		})
+	}
 }
 
 enum DrainState {
@@ -140,23 +163,8 @@ async fn inner(
 
 	let mut servers = servers
 		.into_iter()
-		.map(|row| {
-			Ok(Server {
-				server_id: row.server_id,
-				pool_type: unwrap!(backend::cluster::PoolType::from_i32(row.pool_type as i32)),
-				is_installed: row.is_installed,
-				has_nomad_node: row.has_nomad_node,
-				is_tainted: row.is_tainted,
-				drain_state: if row.is_drained {
-					DrainState::Complete
-				} else if row.is_draining {
-					DrainState::Draining
-				} else {
-					DrainState::None
-				},
-			})
-		})
-		.collect::<GlobalResult<Vec<_>>>()?;
+		.map(TryInto::try_into)
+		.collect::<GlobalResult<Vec<Server>>>()?;
 
 	// Sort job servers by memory usage
 	servers.sort_by_key(|server| memory_by_server.get(&server.server_id));
diff --git a/svc/pkg/cluster/worker/src/workers/nomad_node_registered.rs b/svc/pkg/cluster/worker/src/workers/nomad_node_registered.rs
index 0f3c1e70bb..e29e18989f 100644
--- a/svc/pkg/cluster/worker/src/workers/nomad_node_registered.rs
+++ b/svc/pkg/cluster/worker/src/workers/nomad_node_registered.rs
@@ -1,14 +1,16 @@
 use chirp_worker::prelude::*;
 use proto::backend::pkg::*;
+use util_cluster::metrics;
 
 #[worker(name = "cluster-nomad-node-registered")]
 async fn worker(
 	ctx: &OperationContext<nomad::msg::monitor_node_registered::Message>,
 ) -> GlobalResult<()> {
 	let server_id = unwrap_ref!(ctx.server_id).as_uuid();
+	let nomad_join_ts = util::timestamp::now();
 
-	let (datacenter_id, old_nomad_node_id) = sql_fetch_one!(
-		[ctx, (Uuid, Option<String>)]
+	let (datacenter_id, old_nomad_node_id, install_complete_ts) = sql_fetch_one!(
+		[ctx, (Uuid, Option<String>, i64)]
 		"
 		UPDATE db_cluster.servers
 		SET
@@ -16,11 +18,11 @@ async fn worker(
 			nomad_join_ts = $3
 		WHERE
 			server_id = $1
-		RETURNING datacenter_id, nomad_node_id
+		RETURNING datacenter_id, nomad_node_id, install_complete_ts
 		",
 		&server_id,
 		&ctx.node_id,
-		util::timestamp::now(),
+		nomad_join_ts,
 	)
 	.await?;
 
@@ -34,5 +36,36 @@ async fn worker(
 	})
 	.await?;
 
+	// Insert metrics
+	insert_metrics(&ctx, datacenter_id, nomad_join_ts, install_complete_ts).await?;
+
+	Ok(())
+}
+
+async fn insert_metrics(
+	ctx: &OperationContext<nomad::msg::monitor_node_registered::Message>,
+	datacenter_id: Uuid,
+	nomad_join_ts: i64,
+	install_complete_ts: i64,
+) -> GlobalResult<()> {
+	let datacenters_res = op!([ctx] cluster_datacenter_get {
+		datacenter_ids: vec![datacenter_id.into()],
+	})
+	.await?;
+	let dc = unwrap!(datacenters_res.datacenters.first());
+
+	let datacenter_id = datacenter_id.to_string();
+	let cluster_id = unwrap_ref!(dc.cluster_id).as_uuid().to_string();
+	let dt = (nomad_join_ts - install_complete_ts) as f64 / 1000.0;
+
+	metrics::NOMAD_JOIN_DURATION
+		.with_label_values(&[
+			cluster_id.as_str(),
+			datacenter_id.as_str(),
+			&dc.provider_datacenter_id,
+			&dc.name_id,
+		])
+		.observe(dt);
+
 	Ok(())
 }
diff --git a/svc/pkg/cluster/worker/src/workers/server_install/mod.rs b/svc/pkg/cluster/worker/src/workers/server_install/mod.rs
index 9fe5d14a9e..6d488d2741 100644
--- a/svc/pkg/cluster/worker/src/workers/server_install/mod.rs
+++ b/svc/pkg/cluster/worker/src/workers/server_install/mod.rs
@@ -7,6 +7,7 @@ use std::{
 use chirp_worker::prelude::*;
 use proto::backend::{self, pkg::*};
 use ssh2::Session;
+use util_cluster::metrics;
 
 mod install_scripts;
 
@@ -134,6 +135,7 @@ async fn worker(ctx: &OperationContext<cluster::msg::server_install::Message>) -
 	})
 	.await??;
 
+	// Check if destroyed again after installing
 	if let Some(server_id) = ctx.server_id {
 		let (is_destroying,) = sql_fetch_one!(
 			[ctx, (bool,)]
@@ -150,7 +152,7 @@ async fn worker(ctx: &OperationContext<cluster::msg::server_install::Message>) -
 		.await?;
 
 		if is_destroying {
-			tracing::info!("server marked for deletion, not installing");
+			tracing::info!("server marked for deletion, not marking as installed");
 			return Ok(());
 		}
 	}
@@ -167,15 +169,18 @@ async fn worker(ctx: &OperationContext<cluster::msg::server_install::Message>) -
 
 	// If the server id is set this is not a prebake server
 	if let Some(server_id) = ctx.server_id {
-		sql_execute!(
-			[ctx]
+		let install_complete_ts = util::timestamp::now();
+
+		let (provision_complete_ts,) = sql_fetch_one!(
+			[ctx, (i64,)]
 			"
 			UPDATE db_cluster.servers
 			SET install_complete_ts = $2
 			WHERE server_id = $1
+			RETURNING provision_complete_ts
 			",
 			server_id.as_uuid(),
-			util::timestamp::now(),
+			install_complete_ts,
 		)
 		.await?;
 
@@ -185,6 +190,15 @@ async fn worker(ctx: &OperationContext<cluster::msg::server_install::Message>) -
 			datacenter_id: ctx.datacenter_id,
 		})
 		.await?;
+
+		insert_metrics(
+			ctx,
+			&pool_type,
+			datacenter_id,
+			install_complete_ts,
+			provision_complete_ts,
+		)
+		.await?;
 	}
 
 	Ok(())
@@ -222,3 +236,37 @@ fn write_script(sess: &Session, script_name: &str, content: &str) -> GlobalResul
 
 	Ok(())
 }
+
+async fn insert_metrics(
+	ctx: &OperationContext<cluster::msg::server_install::Message>,
+	pool_type: &backend::cluster::PoolType,
+	datacenter_id: Uuid,
+	install_complete_ts: i64,
+	provision_complete_ts: i64,
+) -> GlobalResult<()> {
+	let datacenters_res = op!([ctx] cluster_datacenter_get {
+		datacenter_ids: vec![datacenter_id.into()],
+	})
+	.await?;
+	let dc = unwrap!(datacenters_res.datacenters.first());
+
+	let datacenter_id = unwrap_ref!(dc.datacenter_id).as_uuid().to_string();
+	let cluster_id = unwrap_ref!(dc.cluster_id).as_uuid().to_string();
+	let dt = (install_complete_ts - provision_complete_ts) as f64 / 1000.0;
+
+	metrics::INSTALL_DURATION
+		.with_label_values(&[
+			cluster_id.as_str(),
+			datacenter_id.as_str(),
+			&dc.provider_datacenter_id,
+			&dc.name_id,
+			match pool_type {
+				backend::cluster::PoolType::Job => "job",
+				backend::cluster::PoolType::Gg => "gg",
+				backend::cluster::PoolType::Ats => "ats",
+			},
+		])
+		.observe(dt);
+
+	Ok(())
+}
diff --git a/svc/pkg/cluster/worker/src/workers/server_provision.rs b/svc/pkg/cluster/worker/src/workers/server_provision.rs
index de664890c5..9cdb65a1d2 100644
--- a/svc/pkg/cluster/worker/src/workers/server_provision.rs
+++ b/svc/pkg/cluster/worker/src/workers/server_provision.rs
@@ -4,6 +4,7 @@ use chirp_worker::prelude::*;
 use futures_util::FutureExt;
 use proto::backend::{self, cluster::PoolType, pkg::*};
 use rand::Rng;
+use util_cluster::metrics;
 
 struct ProvisionResponse {
 	provider_server_id: String,
@@ -138,26 +139,31 @@ async fn inner(
 	};
 
 	if let Some(provision_res) = provision_res {
-		sql_execute!(
-			[ctx]
+		let provision_complete_ts = util::timestamp::now();
+
+		let (create_ts,) = sql_fetch_one!(
+			[ctx, (i64,)]
 			"
 			UPDATE db_cluster.servers
 			SET
 				provider_server_id = $2,
 				provider_hardware = $3,
 				public_ip = $4,
-				install_complete_ts = $5
+				provision_complete_ts = $5,
+				install_complete_ts = $6
 			WHERE server_id = $1
+			RETURNING create_ts
 			",
 			server_id,
 			&provision_res.provider_server_id,
 			&provision_res.provider_hardware,
 			&provision_res.public_ip,
+			provision_complete_ts,
 			if provision_res.already_installed {
-				Some(util::timestamp::now())
+				Some(provision_complete_ts)
 			} else {
 				None
-			}
+			},
 		)
 		.await?;
 
@@ -195,6 +201,8 @@ async fn inner(
 			})
 			.await?;
 		}
+
+		insert_metrics(datacenter, &pool_type, provision_complete_ts, create_ts).await?;
 	} else {
 		tracing::error!(?server_id, hardware_options=?pool.hardware.len(), "failed all attempts to provision server");
 		bail!("failed all attempts to provision server");
@@ -272,3 +280,30 @@ async fn cleanup(
 
 	Ok(())
 }
+
+async fn insert_metrics(
+	dc: &backend::cluster::Datacenter,
+	pool_type: &backend::cluster::PoolType,
+	provision_complete_ts: i64,
+	create_ts: i64,
+) -> GlobalResult<()> {
+	let datacenter_id = unwrap_ref!(dc.datacenter_id).as_uuid().to_string();
+	let cluster_id = unwrap_ref!(dc.cluster_id).as_uuid().to_string();
+	let dt = (provision_complete_ts - create_ts) as f64 / 1000.0;
+
+	metrics::PROVISION_DURATION
+		.with_label_values(&[
+			cluster_id.as_str(),
+			datacenter_id.as_str(),
+			&dc.provider_datacenter_id,
+			&dc.name_id,
+			match pool_type {
+				backend::cluster::PoolType::Job => "job",
+				backend::cluster::PoolType::Gg => "gg",
+				backend::cluster::PoolType::Ats => "ats",
+			},
+		])
+		.observe(dt);
+
+	Ok(())
+}
diff --git a/svc/pkg/cluster/worker/tests/datacenter_tls_issue.rs b/svc/pkg/cluster/worker/tests/datacenter_tls_issue.rs
index 2d04571a01..ad1d6a6e9f 100644
--- a/svc/pkg/cluster/worker/tests/datacenter_tls_issue.rs
+++ b/svc/pkg/cluster/worker/tests/datacenter_tls_issue.rs
@@ -62,4 +62,5 @@ async fn datacenter_tls_issue(ctx: TestCtx) {
 	}
 }
 
+
 // TODO: test renewal