Expose resource metrics using labels in boskos

boskos/metrics/BUILD.bazel

@@ -8,7 +8,6 @@ load(
     "@io_bazel_rules_go//go:def.bzl",
     "go_binary",
     "go_library",
-    "go_test",
 )
 
 go_binary(
@@ -24,8 +23,10 @@ go_library(
     deps = [
         "//boskos/client:go_default_library",
         "//boskos/common:go_default_library",
+        "//prow/config:go_default_library",
+        "//prow/logrusutil:go_default_library",
+        "//prow/metrics:go_default_library",
         "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
-        "//vendor/github.com/prometheus/client_golang/prometheus/promhttp:go_default_library",
         "//vendor/github.com/sirupsen/logrus:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
     ],
@@ -48,10 +49,3 @@ filegroup(
     srcs = [":package-srcs"],
     tags = ["automanaged"],
 )
-
-go_test(
-    name = "go_default_test",
-    srcs = ["metrics_test.go"],
-    embed = [":go_default_library"],
-    deps = ["//boskos/common:go_default_library"],
-)

boskos/metrics/metrics.go

@@ -21,25 +21,24 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
-	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/sets"
+
 	"k8s.io/test-infra/boskos/client"
 	"k8s.io/test-infra/boskos/common"
+	"k8s.io/test-infra/prow/config"
+	"k8s.io/test-infra/prow/logrusutil"
+	"k8s.io/test-infra/prow/metrics"
 )
 
-type prometheusMetrics struct {
-	BoskosState map[string]map[string]prometheus.Gauge
-}
-
 var (
-	promMetrics = prometheusMetrics{
-		BoskosState: map[string]map[string]prometheus.Gauge{},
-	}
+	resourceMetric = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "boskos_resources",
+		Help: "Number of resources recorded in Boskos",
+	}, []string{"type", "state"})
 	resources, states common.CommaSeparatedStrings
 	defaultStates     = []string{
 		common.Busy,
@@ -53,35 +52,13 @@ var (
 )
 
 func init() {
-	flag.Var(&resources, "resource-type", "comma-separated list of resources need to have metrics collected")
-	flag.Var(&states, "resource-state", "comma-separated list of states need to have metrics collected")
-}
-
-func initMetrics() {
-	for _, resource := range resources {
-		promMetrics.BoskosState[resource] = map[string]prometheus.Gauge{}
-		for _, state := range states {
-			promMetrics.BoskosState[resource][state] = prometheus.NewGauge(prometheus.GaugeOpts{
-				Name: fmt.Sprintf("boskos_%s_%s", strings.Replace(resource, "-", "_", -1), state),
-				Help: fmt.Sprintf("Number of %s %s", state, resource),
-			})
-		}
-		// Adding other state for metrics that are not captured with existing state
-		promMetrics.BoskosState[resource][common.Other] = prometheus.NewGauge(prometheus.GaugeOpts{
-			Name: fmt.Sprintf("boskos_%s_%s", strings.Replace(resource, "-", "_", -1), common.Other),
-			Help: fmt.Sprintf("Number of %s %s", common.Other, resource),
-		})
-	}
-
-	for _, gauges := range promMetrics.BoskosState {
-		for _, gauge := range gauges {
-			prometheus.MustRegister(gauge)
-		}
-	}
+	flag.Var(&resources, "resource-type", "comma-separated list of resources need to have metrics collected.")
+	flag.Var(&states, "resource-state", "comma-separated list of states need to have metrics collected.")
+	prometheus.MustRegister(resourceMetric)
 }
 
 func main() {
-	logrus.SetFormatter(&logrus.JSONFormatter{})
+	logrusutil.ComponentInit("boskos-metrics")
 	boskos := client.NewClient("Metrics", "http://boskos")
 	logrus.Infof("Initialzied boskos client!")
 
@@ -90,51 +67,53 @@ func main() {
 		states = defaultStates
 	}
 
-	initMetrics()
-
-	http.Handle("/prometheus", promhttp.Handler())
-	http.Handle("/", handleMetric(boskos))
+	metrics.ExposeMetrics("boskos", config.PushGateway{})
 
 	go func() {
-		logTick := time.NewTicker(time.Minute).C
+		logTick := time.NewTicker(30 * time.Second).C
 		for range logTick {
 			if err := update(boskos); err != nil {
-				logrus.WithError(err).Warning("[Boskos Metrics]Update failed!")
+				logrus.WithError(err).Warning("Update failed!")
 			}
 		}
 	}()
 
 	logrus.Info("Start Service")
-	logrus.WithError(http.ListenAndServe(":8080", nil)).Fatal("ListenAndServe returned.")
+	metricsMux := http.NewServeMux()
+	metricsMux.Handle("/", handleMetric(boskos))
+	logrus.WithError(http.ListenAndServe(":8080", metricsMux)).Fatal("ListenAndServe returned.")
 }
 
-func filterMetrics(src map[string]int) map[string]int {
-	metricStates := sets.NewString(states...)
-	dest := map[string]int{}
-	// Making sure all metrics are created
-	for state := range metricStates {
-		dest[state] = 0
-	}
-	dest[common.Other] = 0
-	for state, value := range src {
-		if state != common.Other && metricStates.Has(state) {
-			dest[state] = value
-		} else {
-			dest[common.Other] += value
+func update(boskos *client.Client) error {
+	// initialize resources counted by type, then state
+	resourcesByState := map[string]map[string]float64{}
+	for _, resource := range resources {
+		resourcesByState[resource] = map[string]float64{}
+		for _, state := range states {
+			resourcesByState[resource][state] = 0
 		}
 	}
-	return dest
-}
 
-func update(boskos *client.Client) error {
+	// record current states
+	knownStates := sets.NewString(states...)
 	for _, resource := range resources {
 		metric, err := boskos.Metric(resource)
 		if err != nil {
 			return fmt.Errorf("fail to get metric for %s : %v", resource, err)
 		}
 		// Filtering metrics states
-		for state, value := range filterMetrics(metric.Current) {
-			promMetrics.BoskosState[resource][state].Set(float64(value))
+		for state, value := range metric.Current {
+			if !knownStates.Has(state) {
+				state = common.Other
+			}
+			resourcesByState[resource][state] = float64(value)
+		}
+	}
+
+	// expose current states
+	for resource, states := range resourcesByState {
+		for state, amount := range states {
+			resourceMetric.WithLabelValues(resource, state).Set(amount)
 		}
 	}
 	return nil