metrics: refactor prometheus metric registration

Register prometheus metrics right after creation to reduce likelihood of missing registration. Follow up on #3129 Signed-off-by: Alexander Yastrebov <alexander.yastrebov@zalando.de>
zalando · Jun 27, 2024 · 1d7f2a4 · 1d7f2a4
1 parent 02515fe
commit 1d7f2a4
Showing 1 changed file with 60 additions and 105 deletions.
diff --git a/metrics/prometheus.go b/metrics/prometheus.go
@@ -59,105 +59,114 @@ type Prometheus struct {
 func NewPrometheus(opts Options) *Prometheus {
 	opts = applyCompatibilityDefaults(opts)
 
+	p := &Prometheus{
+		registry: opts.PrometheusRegistry,
+		opts:     opts,
+	}
+
+	if p.registry == nil {
+		p.registry = prometheus.NewRegistry()
+	}
+
 	namespace := promNamespace
 	if opts.Prefix != "" {
 		namespace = strings.TrimSuffix(opts.Prefix, ".")
 	}
 
-	routeLookup := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+	p.routeLookupM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: promRouteSubsystem,
 		Name:      "lookup_duration_seconds",
 		Help:      "Duration in seconds of a route lookup.",
 		Buckets:   opts.HistogramBuckets,
-	}, []string{})
+	}, []string{}))
 
-	routeErrors := prometheus.NewCounterVec(prometheus.CounterOpts{
+	p.routeErrorsM = register(p, prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: namespace,
 		Subsystem: promRouteSubsystem,
 		Name:      "error_total",
 		Help:      "The total of route lookup errors.",
-	}, []string{})
+	}, []string{}))
 
-	response := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+	p.responseM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: promResponseSubsystem,
 		Name:      "duration_seconds",
 		Help:      "Duration in seconds of a response.",
 		Buckets:   opts.HistogramBuckets,
-	}, []string{"code", "method", "route"})
+	}, []string{"code", "method", "route"}))
 
-	filterCreate := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+	p.filterCreateM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: promFilterSubsystem,
 		Name:      "create_duration_seconds",
 		Help:      "Duration in seconds of filter creation.",
 		Buckets:   opts.HistogramBuckets,
-	}, []string{"filter"})
+	}, []string{"filter"}))
 
-	filterRequest := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+	p.filterRequestM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: promFilterSubsystem,
 		Name:      "request_duration_seconds",
 		Help:      "Duration in seconds of a filter request.",
 		Buckets:   opts.HistogramBuckets,
-	}, []string{"filter"})
+	}, []string{"filter"}))
 
-	filterAllRequest := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+	p.filterAllRequestM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: promFilterSubsystem,
 		Name:      "all_request_duration_seconds",
 		Help:      "Duration in seconds of a filter request by all filters.",
 		Buckets:   opts.HistogramBuckets,
-	}, []string{"route"})
+	}, []string{"route"}))
 
-	filterAllCombinedRequest := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+	p.filterAllCombinedRequestM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: promFilterSubsystem,
 		Name:      "all_combined_request_duration_seconds",
 		Help:      "Duration in seconds of a filter request combined by all filters.",
 		Buckets:   opts.HistogramBuckets,
-	}, []string{})
+	}, []string{}))
 
-	proxyBackend := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+	p.proxyBackendM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: promProxySubsystem,
 		Name:      "duration_seconds",
 		Help:      "Duration in seconds of a proxy backend.",
 		Buckets:   opts.HistogramBuckets,
-	}, []string{"route", "host"})
+	}, []string{"route", "host"}))
 
-	proxyBackendCombined := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+	p.proxyBackendCombinedM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: promProxySubsystem,
 		Name:      "combined_duration_seconds",
 		Help:      "Duration in seconds of a proxy backend combined.",
 		Buckets:   opts.HistogramBuckets,
-	}, []string{})
+	}, []string{}))
 
-	filterResponse := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+	p.filterResponseM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: promFilterSubsystem,
 		Name:      "response_duration_seconds",
 		Help:      "Duration in seconds of a filter request.",
 		Buckets:   opts.HistogramBuckets,
-	}, []string{"filter"})
+	}, []string{"filter"}))
 
-	filterAllResponse := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+	p.filterAllResponseM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: promFilterSubsystem,
 		Name:      "all_response_duration_seconds",
 		Help:      "Duration in seconds of a filter response by all filters.",
 		Buckets:   opts.HistogramBuckets,
-	}, []string{"route"})
+	}, []string{"route"}))
 
-	filterAllCombinedResponse := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+	p.filterAllCombinedResponseM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: promFilterSubsystem,
 		Name:      "all_combined_response_duration_seconds",
 		Help:      "Duration in seconds of a filter response combined by all filters.",
 		Buckets:   opts.HistogramBuckets,
-	}, []string{})
+	}, []string{}))
 
 	metrics := []string{}
 	if opts.EnableServeStatusCodeMetric {
@@ -166,108 +175,80 @@ func NewPrometheus(opts Options) *Prometheus {
 	if opts.EnableServeMethodMetric {
 		metrics = append(metrics, "method")
 	}
-	serveRoute := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+	p.serveRouteM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: promServeSubsystem,
 		Name:      "route_duration_seconds",
 		Help:      "Duration in seconds of serving a route.",
 		Buckets:   opts.HistogramBuckets,
-	}, append(metrics, "route"))
-	serveRouteCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
+	}, append(metrics, "route")))
+	p.serveRouteCounterM = register(p, prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: namespace,
 		Subsystem: promServeSubsystem,
 		Name:      "route_count",
 		Help:      "Total number of requests of serving a route.",
-	}, []string{"code", "method", "route"})
+	}, []string{"code", "method", "route"}))
 
-	serveHost := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+	p.serveHostM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: promServeSubsystem,
 		Name:      "host_duration_seconds",
 		Help:      "Duration in seconds of serving a host.",
 		Buckets:   opts.HistogramBuckets,
-	}, append(metrics, "host"))
-	serveHostCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
+	}, append(metrics, "host")))
+	p.serveHostCounterM = register(p, prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: namespace,
 		Subsystem: promServeSubsystem,
 		Name:      "host_count",
 		Help:      "Total number of requests of serving a host.",
-	}, []string{"code", "method", "host"})
+	}, []string{"code", "method", "host"}))
 
-	proxyBackend5xx := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+	p.proxyBackend5xxM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: promProxySubsystem,
 		Name:      "5xx_duration_seconds",
 		Help:      "Duration in seconds of backend 5xx.",
 		Buckets:   opts.HistogramBuckets,
-	}, []string{})
-	proxyBackendErrors := prometheus.NewCounterVec(prometheus.CounterOpts{
+	}, []string{}))
+	p.proxyBackendErrorsM = register(p, prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: namespace,
 		Subsystem: promProxySubsystem,
 		Name:      "error_total",
 		Help:      "Total number of backend route errors.",
-	}, []string{"route"})
-	proxyStreamingErrors := prometheus.NewCounterVec(prometheus.CounterOpts{
+	}, []string{"route"}))
+	p.proxyStreamingErrorsM = register(p, prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: namespace,
 		Subsystem: promStreamingSubsystem,
 		Name:      "error_total",
 		Help:      "Total number of streaming route errors.",
-	}, []string{"route"})
+	}, []string{"route"}))
 
-	customCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
+	p.customCounterM = register(p, prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: namespace,
 		Subsystem: promCustomSubsystem,
 		Name:      "total",
 		Help:      "Total number of custom metrics.",
-	}, []string{"key"})
-	customGauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+	}, []string{"key"}))
+	p.customGaugeM = register(p, prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: namespace,
 		Subsystem: promCustomSubsystem,
 		Name:      "gauges",
 		Help:      "Gauges number of custom metrics.",
-	}, []string{"key"})
-	customHistogram := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+	}, []string{"key"}))
+	p.customHistogramM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: promCustomSubsystem,
 		Name:      "duration_seconds",
 		Help:      "Duration in seconds of custom metrics.",
 		Buckets:   opts.HistogramBuckets,
-	}, []string{"key"})
-
-	p := &Prometheus{
-		routeLookupM:               routeLookup,
-		routeErrorsM:               routeErrors,
-		responseM:                  response,
-		filterCreateM:              filterCreate,
-		filterRequestM:             filterRequest,
-		filterAllRequestM:          filterAllRequest,
-		filterAllCombinedRequestM:  filterAllCombinedRequest,
-		proxyBackendM:              proxyBackend,
-		proxyBackendCombinedM:      proxyBackendCombined,
-		filterResponseM:            filterResponse,
-		filterAllResponseM:         filterAllResponse,
-		filterAllCombinedResponseM: filterAllCombinedResponse,
-		serveRouteM:                serveRoute,
-		serveRouteCounterM:         serveRouteCounter,
-		serveHostM:                 serveHost,
-		serveHostCounterM:          serveHostCounter,
-		proxyBackend5xxM:           proxyBackend5xx,
-		proxyBackendErrorsM:        proxyBackendErrors,
-		proxyStreamingErrorsM:      proxyStreamingErrors,
-		customCounterM:             customCounter,
-		customGaugeM:               customGauge,
-		customHistogramM:           customHistogram,
+	}, []string{"key"}))
 
-		registry: opts.PrometheusRegistry,
-		opts:     opts,
-	}
-
-	if p.registry == nil {
-		p.registry = prometheus.NewRegistry()
+	// Register prometheus runtime collectors if required.
+	if opts.EnableRuntimeMetrics {
+		register(p, collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
+		register(p, collectors.NewGoCollector())
 	}
 
-	// Register all metrics.
-	p.registerMetrics()
 	return p
 }
 
@@ -276,35 +257,9 @@ func (p *Prometheus) sinceS(start time.Time) float64 {
 	return time.Since(start).Seconds()
 }
 
-func (p *Prometheus) registerMetrics() {
-	p.registry.MustRegister(p.routeLookupM)
-	p.registry.MustRegister(p.responseM)
-	p.registry.MustRegister(p.routeErrorsM)
-	p.registry.MustRegister(p.filterCreateM)
-	p.registry.MustRegister(p.filterRequestM)
-	p.registry.MustRegister(p.filterAllRequestM)
-	p.registry.MustRegister(p.filterAllCombinedRequestM)
-	p.registry.MustRegister(p.proxyBackendM)
-	p.registry.MustRegister(p.proxyBackendCombinedM)
-	p.registry.MustRegister(p.filterResponseM)
-	p.registry.MustRegister(p.filterAllResponseM)
-	p.registry.MustRegister(p.filterAllCombinedResponseM)
-	p.registry.MustRegister(p.serveRouteM)
-	p.registry.MustRegister(p.serveRouteCounterM)
-	p.registry.MustRegister(p.serveHostM)
-	p.registry.MustRegister(p.serveHostCounterM)
-	p.registry.MustRegister(p.proxyBackend5xxM)
-	p.registry.MustRegister(p.proxyBackendErrorsM)
-	p.registry.MustRegister(p.proxyStreamingErrorsM)
-	p.registry.MustRegister(p.customCounterM)
-	p.registry.MustRegister(p.customHistogramM)
-	p.registry.MustRegister(p.customGaugeM)
-
-	// Register prometheus runtime collectors if required.
-	if p.opts.EnableRuntimeMetrics {
-		p.registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
-		p.registry.MustRegister(collectors.NewGoCollector())
-	}
+func register[T prometheus.Collector](p *Prometheus, cs T) T {
+	p.registry.MustRegister(cs)
+	return cs
 }
 
 func (p *Prometheus) CreateHandler() http.Handler {