Add metrics for total number of admissions and wait time

pkg/metrics/metrics.go

@@ -21,6 +21,8 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
+
+	kueue "sigs.k8s.io/kueue/apis/kueue/v1alpha1"
 )
 
 type AdmissionResult string
@@ -33,22 +35,24 @@ const (
 )
 
 var (
-	admissionAttempts = prometheus.NewCounterVec(
+	admissionAttemptsTotal = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Subsystem: subsystemName,
 			Name:      "admission_attempts_total",
-			Help:      "Number of attempts to admit one or more workloads, broken down by result. `success` means that at least one workload was admitted, `inadmissible` means that no workload was admitted.",
+			Help:      "Total number of attempts to admit one or more workloads, broken down by result. `success` means that at least one workload was admitted, `inadmissible` means that no workload was admitted.",
 		}, []string{"result"},
 	)
 
-	admissionAttemptLatency = prometheus.NewHistogramVec(
+	admissionAttemptDuration = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Subsystem: subsystemName,
 			Name:      "admission_attempt_duration_seconds",
 			Help:      "Latency of an admission attempt, broken down by result.",
 		}, []string{"result"},
 	)
 
+	// Metrics tied to the queue system.
+
 	PendingWorkloads = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Subsystem: subsystemName,
@@ -57,6 +61,24 @@ var (
 		}, []string{"cluster_queue"},
 	)
 
+	AdmittedWorkloadsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: subsystemName,
+			Name:      "admitted_workloads_total",
+			Help:      "Total number of admitted workloads per cluster_queue",
+		}, []string{"cluster_queue"},
+	)
+
+	admissionWaitTime = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Subsystem: subsystemName,
+			Name:      "admission_wait_time_seconds",
+			Help:      "The wait time since a workload was created until it was admitted, per cluster_queue",
+		}, []string{"cluster_queue"},
+	)
+
+	// Metrics tied to the cache.
+
 	AdmittedActiveWorkloads = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Subsystem: subsystemName,
@@ -67,15 +89,28 @@ var (
 )
 
 func AdmissionAttempt(result AdmissionResult, duration time.Duration) {
-	admissionAttempts.WithLabelValues(string(result)).Inc()
-	admissionAttemptLatency.WithLabelValues(string(result)).Observe(duration.Seconds())
+	admissionAttemptsTotal.WithLabelValues(string(result)).Inc()
+	admissionAttemptDuration.WithLabelValues(string(result)).Observe(duration.Seconds())
+}
+
+func AdmittedWorkload(cqName kueue.ClusterQueueReference, waitTime time.Duration) {
+	AdmittedWorkloadsTotal.WithLabelValues(string(cqName)).Inc()
+	admissionWaitTime.WithLabelValues(string(cqName)).Observe(waitTime.Seconds())
+}
+
+func ClearQueueSystemMetrics(cqName string) {
+	PendingWorkloads.DeleteLabelValues(cqName)
+	AdmittedWorkloadsTotal.DeleteLabelValues(cqName)
+	admissionWaitTime.DeleteLabelValues(cqName)
 }
 
 func Register() {
 	metrics.Registry.MustRegister(
-		admissionAttempts,
-		admissionAttemptLatency,
+		admissionAttemptsTotal,
+		admissionAttemptDuration,
 		PendingWorkloads,
 		AdmittedActiveWorkloads,
+		AdmittedWorkloadsTotal,
+		admissionWaitTime,
 	)
 }

pkg/queue/manager.go

@@ -146,7 +146,7 @@ func (m *Manager) DeleteClusterQueue(cq *kueue.ClusterQueue) {
 		return
 	}
 	delete(m.clusterQueues, cq.Name)
-	metrics.PendingWorkloads.DeleteLabelValues(cq.Name)
+	metrics.ClearQueueSystemMetrics(cq.Name)
 
 	cohort := cq.Spec.Cohort
 	m.deleteCohort(cohort, cq.Name)

pkg/scheduler/scheduler.go

@@ -326,7 +326,9 @@ func (s *Scheduler) admit(ctx context.Context, e *entry) error {
 	s.admissionRoutineWrapper.Run(func() {
 		err := s.client.Update(ctx, newWorkload.DeepCopy())
 		if err == nil {
-			s.recorder.Eventf(newWorkload, corev1.EventTypeNormal, "Admitted", "Admitted by ClusterQueue %v", admission.ClusterQueue)
+			waitTime := time.Since(e.Obj.CreationTimestamp.Time)
+			s.recorder.Eventf(newWorkload, corev1.EventTypeNormal, "Admitted", "Admitted by ClusterQueue %v, wait time was %.3fs", admission.ClusterQueue, waitTime.Seconds())
+			metrics.AdmittedWorkload(admission.ClusterQueue, waitTime)
 			log.V(2).Info("Workload successfully admitted and assigned flavors")
 			return
 		}

test/integration/framework/framework.go

@@ -282,6 +282,15 @@ func ExpectAdmittedActiveWorkloadsMetric(cq *kueue.ClusterQueue, v int) {
 	}, Timeout, Interval).Should(gomega.Equal(v))
 }
 
+func ExpectAdmittedWorkloadsTotalMetric(cq *kueue.ClusterQueue, v int) {
+	metric := metrics.AdmittedWorkloadsTotal.WithLabelValues(cq.Name)
+	gomega.EventuallyWithOffset(1, func() int {
+		v, err := testutil.GetCounterMetricValue(metric)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+		return int(v)
+	}, Timeout, Interval).Should(gomega.Equal(v))
+}
+
 func UpdateWorkloadStatus(ctx context.Context, k8sClient client.Client, wl *kueue.Workload, update func(*kueue.Workload)) {
 	gomega.EventuallyWithOffset(1, func() error {
 		var updatedWl kueue.Workload