From a07e3cc4d435fae6175e73bfa0b21e951eb9d3db Mon Sep 17 00:00:00 2001 From: Alexander Greene Date: Wed, 19 Jan 2022 11:37:05 -0800 Subject: [PATCH] Introduce Experimental Metrics (#2435) This commit introduces a series of metrics that are only recorded when OLM is built with the experimental_metrics build tag. The metrics introduced in this commit includes namespace/name information of the request being reconciled by a series of controllers. These experimental metrics are enabled by default in the e2e-local target defined in the project's Makefile. Signed-off-by: Alexander Greene --- Makefile | 4 +- .../operators/adoption_controller.go | 3 ++ .../operators/operator_controller.go | 2 + .../operators/operatorcondition_controller.go | 2 + .../operatorconditiongenerator_controller.go | 2 + pkg/metrics/experimental_declarations.go | 42 +++++++++++++++++++ pkg/metrics/experimental_register.go | 41 ++++++++++++++++++ 7 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 pkg/metrics/experimental_declarations.go create mode 100644 pkg/metrics/experimental_register.go diff --git a/Makefile b/Makefile index 7247096f2c..40da0dea55 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ SHELL := /bin/bash ORG := github.com/operator-framework PKG := $(ORG)/operator-lifecycle-manager MOD_FLAGS := $(shell (go version | grep -q -E "1\.1[1-9]") && echo -mod=vendor) +BUILD_TAGS := "json1" CMDS := $(shell go list $(MOD_FLAGS) ./cmd/...) TCMDS := $(shell go list $(MOD_FLAGS) ./test/e2e/...) MOCKGEN := ./scripts/update_mockgen.sh @@ -101,7 +102,7 @@ bin/cpb: FORCE $(CMDS): version_flags=-ldflags "-X $(PKG)/pkg/version.GitCommit=$(GIT_COMMIT) -X $(PKG)/pkg/version.OLMVersion=`cat OLM_VERSION`" $(CMDS): - $(arch_flags) go $(build_cmd) $(MOD_FLAGS) $(version_flags) -tags "json1" -o bin/$(shell basename $@) $@ + $(arch_flags) go $(build_cmd) $(MOD_FLAGS) $(version_flags) -tags $(BUILD_TAGS) -o bin/$(shell basename $@) $@ build: clean $(CMDS) @@ -143,6 +144,7 @@ e2e: # See workflows/e2e-tests.yml See test/e2e/README.md for details. .PHONY: e2e-local +e2e-local: BUILD_TAGS="json1 experimental_metrics" e2e-local: extra_args=-kind.images=../test/e2e-local.image.tar e2e-local: run=bin/e2e-local.test e2e-local: bin/e2e-local.test test/e2e-local.image.tar diff --git a/pkg/controller/operators/adoption_controller.go b/pkg/controller/operators/adoption_controller.go index f0cd672893..1031be4359 100644 --- a/pkg/controller/operators/adoption_controller.go +++ b/pkg/controller/operators/adoption_controller.go @@ -30,6 +30,7 @@ import ( operatorsv2 "github.com/operator-framework/api/pkg/operators/v2" "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/decorators" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/ownerutil" + "github.com/operator-framework/operator-lifecycle-manager/pkg/metrics" ) // AdoptionReconciler automagically associates Operator components with their respective operator resource. @@ -113,6 +114,7 @@ func (r *AdoptionReconciler) ReconcileSubscription(ctx context.Context, req ctrl // Set up a convenient log object so we don't have to type request over and over again log := r.log.WithValues("request", req) log.V(1).Info("reconciling subscription") + metrics.EmitAdoptionSubscriptionReconcile(req.Namespace, req.Name) // Fetch the Subscription from the cache in := &operatorsv1alpha1.Subscription{} @@ -175,6 +177,7 @@ func (r *AdoptionReconciler) ReconcileClusterServiceVersion(ctx context.Context, // Set up a convenient log object so we don't have to type request over and over again log := r.log.WithValues("request", req) log.V(1).Info("reconciling csv") + metrics.EmitAdoptionCSVReconcile(req.Namespace, req.Name) // Fetch the CSV from the cache in := &operatorsv1alpha1.ClusterServiceVersion{} diff --git a/pkg/controller/operators/operator_controller.go b/pkg/controller/operators/operator_controller.go index 31d5c92c34..e39ecb7d9a 100644 --- a/pkg/controller/operators/operator_controller.go +++ b/pkg/controller/operators/operator_controller.go @@ -28,6 +28,7 @@ import ( operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" operatorsv2 "github.com/operator-framework/api/pkg/operators/v2" "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/decorators" + "github.com/operator-framework/operator-lifecycle-manager/pkg/metrics" ) var ( @@ -118,6 +119,7 @@ func (r *OperatorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c // Set up a convenient log object so we don't have to type request over and over again log := r.log.WithValues("request", req) log.V(1).Info("reconciling operator") + metrics.EmitOperatorReconcile(req.Namespace, req.Name) // Get the Operator create := false diff --git a/pkg/controller/operators/operatorcondition_controller.go b/pkg/controller/operators/operatorcondition_controller.go index 6b6ab867f5..122c7604a7 100644 --- a/pkg/controller/operators/operatorcondition_controller.go +++ b/pkg/controller/operators/operatorcondition_controller.go @@ -22,6 +22,7 @@ import ( operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" operatorsv2 "github.com/operator-framework/api/pkg/operators/v2" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/ownerutil" + "github.com/operator-framework/operator-lifecycle-manager/pkg/metrics" ) const ( @@ -94,6 +95,7 @@ func (r *OperatorConditionReconciler) Reconcile(ctx context.Context, req ctrl.Re // Set up a convenient log object so we don't have to type request over and over again log := r.log.WithValues("request", req) log.V(2).Info("reconciling operatorcondition") + metrics.EmitOperatorConditionReconcile(req.Namespace, req.Name) operatorCondition := &operatorsv2.OperatorCondition{} err := r.Client.Get(context.TODO(), req.NamespacedName, operatorCondition) diff --git a/pkg/controller/operators/operatorconditiongenerator_controller.go b/pkg/controller/operators/operatorconditiongenerator_controller.go index 1000b6b57d..dff9ae7a39 100644 --- a/pkg/controller/operators/operatorconditiongenerator_controller.go +++ b/pkg/controller/operators/operatorconditiongenerator_controller.go @@ -20,6 +20,7 @@ import ( operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" operatorsv2 "github.com/operator-framework/api/pkg/operators/v2" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/ownerutil" + "github.com/operator-framework/operator-lifecycle-manager/pkg/metrics" ) // OperatorConditionGeneratorReconciler reconciles a ClusterServiceVersion object and creates an OperatorCondition. @@ -90,6 +91,7 @@ var _ reconcile.Reconciler = &OperatorConditionGeneratorReconciler{} func (r *OperatorConditionGeneratorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { // Set up a convenient log object so we don't have to type request over and over again log := r.log.WithValues("request", req) + metrics.EmitOperatorConditionGeneratorReconcile(req.Namespace, req.Name) in := &operatorsv1alpha1.ClusterServiceVersion{} err := r.Client.Get(context.TODO(), req.NamespacedName, in) diff --git a/pkg/metrics/experimental_declarations.go b/pkg/metrics/experimental_declarations.go new file mode 100644 index 0000000000..30615c7ab6 --- /dev/null +++ b/pkg/metrics/experimental_declarations.go @@ -0,0 +1,42 @@ +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +const ( + // Controller names + operatorController = "operator" + adoptionCSVController = "adoption_csv" + adoptionSubscriptionController = "adoption_subscription" + operatorConditionController = "operator_condition" + operatorConditionGeneratorController = "operator_condition_generator" +) + +var ( + reconcileMetrics = map[string]*prometheus.CounterVec{} +) + +func EmitOperatorReconcile(namespace, name string) { + emitReconcile(operatorController, namespace, name) +} + +func EmitAdoptionCSVReconcile(namespace, name string) { + emitReconcile(adoptionCSVController, namespace, name) +} + +func EmitAdoptionSubscriptionReconcile(namespace, name string) { + emitReconcile(adoptionSubscriptionController, namespace, name) +} + +func EmitOperatorConditionReconcile(namespace, name string) { + emitReconcile(operatorConditionController, namespace, name) +} + +func EmitOperatorConditionGeneratorReconcile(namespace, name string) { + emitReconcile(operatorConditionGeneratorController, namespace, name) +} + +func emitReconcile(controllerName, namespace, name string) { + if counter, ok := reconcileMetrics[controllerName]; ok { + counter.WithLabelValues(namespace, name).Inc() + } +} diff --git a/pkg/metrics/experimental_register.go b/pkg/metrics/experimental_register.go new file mode 100644 index 0000000000..87d0e9bfee --- /dev/null +++ b/pkg/metrics/experimental_register.go @@ -0,0 +1,41 @@ +//go:build experimental_metrics +// +build experimental_metrics + +package metrics + +import ( + "fmt" + "strings" + + "github.com/prometheus/client_golang/prometheus" +) + +func init() { + // Register experimental metrics + reconcileMetrics = reconcileCounters(operatorController, adoptionCSVController, adoptionSubscriptionController, operatorConditionController, operatorConditionGeneratorController) + registerReconcileMetrics() +} + +func reconcileCounters(reconcilerNames ...string) map[string]*prometheus.CounterVec { + result := map[string]*prometheus.CounterVec{} + for _, s := range reconcilerNames { + result[s] = createReconcileCounterVec(s) + } + return result +} + +func createReconcileCounterVec(name string) *prometheus.CounterVec { + return prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "controller_reconcile_" + name, + Help: fmt.Sprintf("Count of reconcile events by the %s controller", strings.Replace(name, "_", " ", -1)), + }, + []string{NamespaceLabel, NameLabel}, + ) +} + +func registerReconcileMetrics() { + for _, v := range reconcileMetrics { + prometheus.MustRegister(v) + } +}