From 143325b254e243fc936cafaff22381d191477c70 Mon Sep 17 00:00:00 2001 From: Steve Kuznetsov Date: Mon, 5 Aug 2019 08:47:08 -0700 Subject: [PATCH] Expose resource metrics using labels in boskos Using one metric name for boskos resources allows for simpler queries that translate across resource types and states. For instance, instead of a query for `boskos_myType_myState` the query would be for `boskos_resources{type="myType",state="myState"}`. Furthermore, queries can now also be done across states for a type, or across types for a state. For instance, to see the states for a specific type: sum(boskos_resources{type="myThings"}) by (state) This allows for simpler dashboard creation as well. The current behavior of producing a "0" count for type/state combinations that do not have any record in Boskos but are requested via flag is preserved. Metrics serving is moved to `/metrics` on `:9090` as convention dictates. Signed-off-by: Steve Kuznetsov --- boskos/metrics/BUILD.bazel | 12 +--- boskos/metrics/metrics.go | 99 +++++++++++---------------- boskos/metrics/metrics_test.go | 120 --------------------------------- 3 files changed, 42 insertions(+), 189 deletions(-) delete mode 100644 boskos/metrics/metrics_test.go diff --git a/boskos/metrics/BUILD.bazel b/boskos/metrics/BUILD.bazel index fad12265db02..904243fe3f29 100644 --- a/boskos/metrics/BUILD.bazel +++ b/boskos/metrics/BUILD.bazel @@ -8,7 +8,6 @@ load( "@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", - "go_test", ) go_binary( @@ -24,8 +23,10 @@ go_library( deps = [ "//boskos/client:go_default_library", "//boskos/common:go_default_library", + "//prow/config:go_default_library", + "//prow/logrusutil:go_default_library", + "//prow/metrics:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", - "//vendor/github.com/prometheus/client_golang/prometheus/promhttp:go_default_library", "//vendor/github.com/sirupsen/logrus:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", ], @@ -48,10 +49,3 @@ filegroup( srcs = [":package-srcs"], tags = ["automanaged"], ) - -go_test( - name = "go_default_test", - srcs = ["metrics_test.go"], - embed = [":go_default_library"], - deps = ["//boskos/common:go_default_library"], -) diff --git a/boskos/metrics/metrics.go b/boskos/metrics/metrics.go index 61c2ddccd8b5..e6db7c20958e 100644 --- a/boskos/metrics/metrics.go +++ b/boskos/metrics/metrics.go @@ -21,25 +21,24 @@ import ( "flag" "fmt" "net/http" - "strings" "time" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/test-infra/boskos/client" "k8s.io/test-infra/boskos/common" + "k8s.io/test-infra/prow/config" + "k8s.io/test-infra/prow/logrusutil" + "k8s.io/test-infra/prow/metrics" ) -type prometheusMetrics struct { - BoskosState map[string]map[string]prometheus.Gauge -} - var ( - promMetrics = prometheusMetrics{ - BoskosState: map[string]map[string]prometheus.Gauge{}, - } + resourceMetric = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "boskos_resources", + Help: "Number of resources recorded in Boskos", + }, []string{"type", "state"}) resources, states common.CommaSeparatedStrings defaultStates = []string{ common.Busy, @@ -53,35 +52,13 @@ var ( ) func init() { - flag.Var(&resources, "resource-type", "comma-separated list of resources need to have metrics collected") - flag.Var(&states, "resource-state", "comma-separated list of states need to have metrics collected") -} - -func initMetrics() { - for _, resource := range resources { - promMetrics.BoskosState[resource] = map[string]prometheus.Gauge{} - for _, state := range states { - promMetrics.BoskosState[resource][state] = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: fmt.Sprintf("boskos_%s_%s", strings.Replace(resource, "-", "_", -1), state), - Help: fmt.Sprintf("Number of %s %s", state, resource), - }) - } - // Adding other state for metrics that are not captured with existing state - promMetrics.BoskosState[resource][common.Other] = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: fmt.Sprintf("boskos_%s_%s", strings.Replace(resource, "-", "_", -1), common.Other), - Help: fmt.Sprintf("Number of %s %s", common.Other, resource), - }) - } - - for _, gauges := range promMetrics.BoskosState { - for _, gauge := range gauges { - prometheus.MustRegister(gauge) - } - } + flag.Var(&resources, "resource-type", "comma-separated list of resources need to have metrics collected.") + flag.Var(&states, "resource-state", "comma-separated list of states need to have metrics collected.") + prometheus.MustRegister(resourceMetric) } func main() { - logrus.SetFormatter(&logrus.JSONFormatter{}) + logrusutil.ComponentInit("boskos-metrics") boskos := client.NewClient("Metrics", "http://boskos") logrus.Infof("Initialzied boskos client!") @@ -90,51 +67,53 @@ func main() { states = defaultStates } - initMetrics() - - http.Handle("/prometheus", promhttp.Handler()) - http.Handle("/", handleMetric(boskos)) + metrics.ExposeMetrics("boskos", config.PushGateway{}) go func() { - logTick := time.NewTicker(time.Minute).C + logTick := time.NewTicker(30 * time.Second).C for range logTick { if err := update(boskos); err != nil { - logrus.WithError(err).Warning("[Boskos Metrics]Update failed!") + logrus.WithError(err).Warning("Update failed!") } } }() logrus.Info("Start Service") - logrus.WithError(http.ListenAndServe(":8080", nil)).Fatal("ListenAndServe returned.") + metricsMux := http.NewServeMux() + metricsMux.Handle("/", handleMetric(boskos)) + logrus.WithError(http.ListenAndServe(":8080", metricsMux)).Fatal("ListenAndServe returned.") } -func filterMetrics(src map[string]int) map[string]int { - metricStates := sets.NewString(states...) - dest := map[string]int{} - // Making sure all metrics are created - for state := range metricStates { - dest[state] = 0 - } - dest[common.Other] = 0 - for state, value := range src { - if state != common.Other && metricStates.Has(state) { - dest[state] = value - } else { - dest[common.Other] += value +func update(boskos *client.Client) error { + // initialize resources counted by type, then state + resourcesByState := map[string]map[string]float64{} + for _, resource := range resources { + resourcesByState[resource] = map[string]float64{} + for _, state := range states { + resourcesByState[resource][state] = 0 } } - return dest -} -func update(boskos *client.Client) error { + // record current states + knownStates := sets.NewString(states...) for _, resource := range resources { metric, err := boskos.Metric(resource) if err != nil { return fmt.Errorf("fail to get metric for %s : %v", resource, err) } // Filtering metrics states - for state, value := range filterMetrics(metric.Current) { - promMetrics.BoskosState[resource][state].Set(float64(value)) + for state, value := range metric.Current { + if !knownStates.Has(state) { + state = common.Other + } + resourcesByState[resource][state] = float64(value) + } + } + + // expose current states + for resource, states := range resourcesByState { + for state, amount := range states { + resourceMetric.WithLabelValues(resource, state).Set(amount) } } return nil diff --git a/boskos/metrics/metrics_test.go b/boskos/metrics/metrics_test.go deleted file mode 100644 index 01813a1d015a..000000000000 --- a/boskos/metrics/metrics_test.go +++ /dev/null @@ -1,120 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "reflect" - "testing" - - "k8s.io/test-infra/boskos/common" -) - -func TestFilterMetric(t *testing.T) { - - testCases := []struct { - name string - states []string - src, dest map[string]int - }{ - { - name: "noOther", - states: []string{common.Dirty, common.Cleaning, common.Busy, common.Free}, - src: map[string]int{ - common.Dirty: 10, - common.Cleaning: 2, - common.Busy: 5, - common.Free: 3, - }, - dest: map[string]int{ - common.Dirty: 10, - common.Cleaning: 2, - common.Busy: 5, - common.Free: 3, - common.Other: 0, - }, - }, - { - name: "multipleOther", - states: []string{common.Dirty, common.Cleaning, common.Busy, common.Free}, - src: map[string]int{ - common.Dirty: 10, - common.Cleaning: 2, - common.Busy: 5, - common.Free: 3, - "test": 10, - "new": 14, - }, - dest: map[string]int{ - common.Dirty: 10, - common.Cleaning: 2, - common.Busy: 5, - common.Free: 3, - common.Other: 24, - }, - }, - { - name: "multipleOtherNoLeased", - states: []string{common.Dirty, common.Cleaning, common.Busy, common.Free, common.Leased}, - src: map[string]int{ - common.Dirty: 10, - common.Cleaning: 2, - common.Busy: 5, - common.Free: 3, - "test": 10, - "new": 14, - }, - dest: map[string]int{ - common.Dirty: 10, - common.Cleaning: 2, - common.Busy: 5, - common.Free: 3, - common.Leased: 0, - common.Other: 24, - }, - }, - { - name: "NoOtherLeased", - states: []string{common.Dirty, common.Cleaning, common.Busy, common.Free, common.Leased}, - src: map[string]int{ - common.Dirty: 10, - common.Cleaning: 2, - common.Busy: 5, - common.Free: 3, - common.Leased: 10, - }, - dest: map[string]int{ - common.Dirty: 10, - common.Cleaning: 2, - common.Busy: 5, - common.Free: 3, - common.Leased: 10, - common.Other: 0, - }, - }, - } - - for _, tc := range testCases { - test := func(t *testing.T) { - states = tc.states - dest := filterMetrics(tc.src) - if !reflect.DeepEqual(dest, tc.dest) { - t.Errorf("dest: %v is different than expected %v", dest, tc.dest) - } - } - t.Run(tc.name, test) - } -}