Add scaler metric latency support

Add scaler metric latency support Signed-off-by: kevin <tengkang@msn.com>
kedacore · Dec 25, 2022 · 3b741a3 · 3b741a3
1 parent b7607fc
commit 3b741a3
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 0 deletions.
diff --git a/pkg/prommetrics/adapter/adapter_prommetrics.go b/pkg/prommetrics/adapter/adapter_prommetrics.go
@@ -45,6 +45,15 @@ var (
 		},
 		metricLabels,
 	)
+	scalerMetricsLatency = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "keda_metrics_adapter",
+			Subsystem: "scaler",
+			Name:      "metrics_latency",
+			Help:      "Scaler Metrics Latency",
+		},
+		metricLabels,
+	)
 	scalerErrors = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: "keda_metrics_adapter",
@@ -74,6 +83,7 @@ func init() {
 	registry = prometheus.NewRegistry()
 	registry.MustRegister(scalerErrorsTotal)
 	registry.MustRegister(scalerMetricsValue)
+	registry.MustRegister(scalerMetricsLatency)
 	registry.MustRegister(scalerErrors)
 	registry.MustRegister(scaledObjectErrors)
 }
@@ -104,6 +114,11 @@ func (metricsServer PrometheusMetricServer) RecordHPAScalerMetric(namespace stri
 	scalerMetricsValue.With(getLabels(namespace, scaledObject, scaler, scalerIndex, metric)).Set(value)
 }
 
+// RecordHPAScalerLatency create a measurement of the latency to external metric
+func (metricsServer PrometheusMetricServer) RecordHPAScalerLatency(namespace string, scaledObject string, scaler string, scalerIndex int, metric string, value float64) {
+	scalerMetricsLatency.With(getLabels(namespace, scaledObject, scaler, scalerIndex, metric)).Set(value)
+}
+
 // RecordHPAScalerError counts the number of errors occurred in trying get an external metric used by the HPA
 func (metricsServer PrometheusMetricServer) RecordHPAScalerError(namespace string, scaledObject string, scaler string, scalerIndex int, metric string, err error) {
 	if err != nil {

diff --git a/pkg/prommetrics/prommetrics.go b/pkg/prommetrics/prommetrics.go
@@ -55,6 +55,15 @@ var (
 		},
 		metricLabels,
 	)
+	scalerMetricsLatency = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: DefaultPromMetricsNamespace,
+			Subsystem: "scaler",
+			Name:      "metrics_latency",
+			Help:      "Scaler Metrics Latency",
+		},
+		metricLabels,
+	)
 	scalerErrors = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: DefaultPromMetricsNamespace,
@@ -96,6 +105,7 @@ var (
 func init() {
 	metrics.Registry.MustRegister(scalerErrorsTotal)
 	metrics.Registry.MustRegister(scalerMetricsValue)
+	metrics.Registry.MustRegister(scalerMetricsLatency)
 	metrics.Registry.MustRegister(scalerErrors)
 	metrics.Registry.MustRegister(scaledObjectErrors)
 
@@ -108,6 +118,11 @@ func RecordScalerMetric(namespace string, scaledObject string, scaler string, sc
 	scalerMetricsValue.With(getLabels(namespace, scaledObject, scaler, scalerIndex, metric)).Set(value)
 }
 
+// RecordScalerLatency create a measurement of the latency to external metric
+func RecordScalerLatency(namespace string, scaledObject string, scaler string, scalerIndex int, metric string, value float64) {
+	scalerMetricsLatency.With(getLabels(namespace, scaledObject, scaler, scalerIndex, metric)).Set(value)
+}
+
 // RecordScalerError counts the number of errors occurred in trying get an external metric used by the HPA
 func RecordScalerError(namespace string, scaledObject string, scaler string, scalerIndex int, metric string, err error) {
 	if err != nil {

diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go
@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/go-logr/logr"
 	apiErrors "k8s.io/apimachinery/pkg/api/errors"
@@ -166,7 +167,10 @@ func (p *KedaProvider) GetExternalMetric(ctx context.Context, namespace string,
 			}
 			// Filter only the desired metric
 			if strings.EqualFold(metricSpec.External.Metric.Name, info.Metric) {
+				startTime := time.Now()
 				metrics, err := cache.GetMetricsForScaler(ctx, scalerIndex, info.Metric)
+				scalerLatency := time.Since(startTime).Milliseconds()
+				promMetricsServer.RecordHPAScalerLatency(namespace, scaledObject.Name, scalerName, scalerIndex, info.Metric, float64(scalerLatency))
 				metrics, err = fallback.GetMetricsWithFallback(ctx, p.client, logger, metrics, err, info.Metric, scaledObject, metricSpec)
 				if err != nil {
 					scalerError = true

diff --git a/pkg/scaling/scale_handler.go b/pkg/scaling/scale_handler.go
@@ -440,7 +440,10 @@ func (h *scaleHandler) GetScaledObjectMetrics(ctx context.Context, scaledObjectN
 				}
 
 				if !metricsFoundInCache {
+					startTime := time.Now()
 					metrics, err = cache.GetMetricsForScaler(ctx, scalerIndex, metricName)
+					scalerLatency := time.Since(startTime).Milliseconds()
+					prommetrics.RecordScalerLatency(scaledObjectNamespace, scaledObject.Name, scalerName, scalerIndex, metricName, float64(scalerLatency))
 					h.logger.V(1).Info("Getting metrics from scaler", "scaledObject.Namespace", scaledObjectNamespace, "scaledObject.Name", scaledObjectName, "scaler", scalerName, "metricName", metricSpec.External.Metric.Name, "metrics", metrics, "scalerError", err)
 				}
 				metrics, err = fallback.GetMetricsWithFallback(ctx, h.client, h.logger, metrics, err, metricName, scaledObject, metricSpec)