From 91b949ae2dda906dc18e80318d4a089763e98736 Mon Sep 17 00:00:00 2001 From: Josef Karasek Date: Fri, 3 Sep 2021 17:20:55 +0200 Subject: [PATCH 1/2] Emit CSV metric on startup (#2216) Signed-off-by: Josef Karasek Upstream-repository: operator-lifecycle-manager Upstream-commit: de4bebe06ba076f804d28c594d7dc52dfd95ef20 --- .../cmd/olm/main.go | 5 ++ .../pkg/controller/operators/olm/operator.go | 17 ++++ .../test/e2e/metrics_e2e_test.go | 84 +++++++++++++++++++ .../cmd/olm/main.go | 5 ++ .../pkg/controller/operators/olm/operator.go | 17 ++++ 5 files changed, 128 insertions(+) diff --git a/staging/operator-lifecycle-manager/cmd/olm/main.go b/staging/operator-lifecycle-manager/cmd/olm/main.go index 33fd6f709c..dd67a27a79 100644 --- a/staging/operator-lifecycle-manager/cmd/olm/main.go +++ b/staging/operator-lifecycle-manager/cmd/olm/main.go @@ -171,6 +171,11 @@ func main() { op.Run(ctx) <-op.Ready() + // Emit CSV metric + if err = op.EnsureCSVMetric(); err != nil { + logger.WithError(err).Fatalf("error emitting metrics for existing CSV") + } + if *writeStatusName != "" { reconciler, err := openshift.NewClusterOperatorReconciler( openshift.WithClient(mgr.GetClient()), diff --git a/staging/operator-lifecycle-manager/pkg/controller/operators/olm/operator.go b/staging/operator-lifecycle-manager/pkg/controller/operators/olm/operator.go index 2ab92f2a65..23a6e021c0 100644 --- a/staging/operator-lifecycle-manager/pkg/controller/operators/olm/operator.go +++ b/staging/operator-lifecycle-manager/pkg/controller/operators/olm/operator.go @@ -677,6 +677,23 @@ func (a *Operator) RegisterCSVWatchNotification(csvNotification csvutility.Watch a.csvNotification = csvNotification } +func (a *Operator) EnsureCSVMetric() error { + csvs, err := a.lister.OperatorsV1alpha1().ClusterServiceVersionLister().List(labels.Everything()) + if err != nil { + return err + } + for _, csv := range csvs { + logger := a.logger.WithFields(logrus.Fields{ + "name": csv.GetName(), + "namespace": csv.GetNamespace(), + "self": csv.GetSelfLink(), + }) + logger.Debug("emitting metrics for existing CSV") + metrics.EmitCSVMetric(csv, csv) + } + return nil +} + func (a *Operator) syncGCObject(obj interface{}) (syncError error) { metaObj, ok := obj.(metav1.Object) if !ok { diff --git a/staging/operator-lifecycle-manager/test/e2e/metrics_e2e_test.go b/staging/operator-lifecycle-manager/test/e2e/metrics_e2e_test.go index 56cef82d3e..2769091e3a 100644 --- a/staging/operator-lifecycle-manager/test/e2e/metrics_e2e_test.go +++ b/staging/operator-lifecycle-manager/test/e2e/metrics_e2e_test.go @@ -16,6 +16,7 @@ import ( . "github.com/onsi/gomega" io_prometheus_client "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -116,6 +117,44 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() { }) }) }) + + When("a CSV is created", func() { + var ( + cleanupCSV cleanupFunc + csv v1alpha1.ClusterServiceVersion + ) + BeforeEach(func() { + packageName := genName("csv-test-") + packageStable := fmt.Sprintf("%s-stable", packageName) + csv = newCSV(packageStable, testNamespace, "", semver.MustParse("0.1.0"), nil, nil, nil) + + var err error + _, err = createCSV(c, crc, csv, testNamespace, false, false) + Expect(err).ToNot(HaveOccurred()) + _, err = fetchCSV(crc, csv.Name, testNamespace, csvSucceededChecker) + Expect(err).ToNot(HaveOccurred()) + }) + AfterEach(func() { + if cleanupCSV != nil { + cleanupCSV() + } + }) + It("emits a CSV metrics", func() { + Expect(getMetricsFromPod(c, getPodWithLabel(c, "app=olm-operator"))).To( + ContainElement(LikeMetric(WithFamily("csv_succeeded"), WithName(csv.Name), WithValue(1))), + ) + }) + When("the OLM pod restarts", func() { + BeforeEach(func() { + restartDeploymentWithLabel(c, "app=olm-operator") + }) + It("CSV metric is preserved", func() { + Expect(getMetricsFromPod(c, getPodWithLabel(c, "app=olm-operator"))).To( + ContainElement(LikeMetric(WithFamily("csv_succeeded"), WithName(csv.Name), WithValue(1))), + ) + }) + }) + }) }) Context("Metrics emitted by objects during operator installation", func() { @@ -396,6 +435,51 @@ func getPodWithLabel(client operatorclient.ClientInterface, label string) *corev return &podList.Items[0] } +func getDeploymentWithLabel(client operatorclient.ClientInterface, label string) *appsv1.Deployment { + listOptions := metav1.ListOptions{LabelSelector: label} + var deploymentList *appsv1.DeploymentList + EventuallyWithOffset(1, func() (numDeps int, err error) { + deploymentList, err = client.KubernetesInterface().AppsV1().Deployments(operatorNamespace).List(context.TODO(), listOptions) + if deploymentList != nil { + numDeps = len(deploymentList.Items) + } + + return + }).Should(Equal(1), "expected exactly one Deployment") + + return &deploymentList.Items[0] +} + +func restartDeploymentWithLabel(client operatorclient.ClientInterface, l string) { + d := getDeploymentWithLabel(client, l) + z := int32(0) + oldZ := *d.Spec.Replicas + d.Spec.Replicas = &z + _, err := client.KubernetesInterface().AppsV1().Deployments(operatorNamespace).Update(context.TODO(), d, metav1.UpdateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + EventuallyWithOffset(1, func() (replicas int32, err error) { + deployment, err := client.KubernetesInterface().AppsV1().Deployments(operatorNamespace).Get(context.TODO(), d.Name, metav1.GetOptions{}) + if deployment != nil { + replicas = deployment.Status.Replicas + } + return + }).Should(Equal(int32(0)), "expected exactly 0 Deployments") + + updated := getDeploymentWithLabel(client, l) + updated.Spec.Replicas = &oldZ + _, err = client.KubernetesInterface().AppsV1().Deployments(operatorNamespace).Update(context.TODO(), updated, metav1.UpdateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + EventuallyWithOffset(1, func() (replicas int32, err error) { + deployment, err := client.KubernetesInterface().AppsV1().Deployments(operatorNamespace).Get(context.TODO(), d.Name, metav1.GetOptions{}) + if deployment != nil { + replicas = deployment.Status.Replicas + } + return + }).Should(Equal(oldZ), "expected exactly 1 Deployment") +} + func extractMetricPortFromPod(pod *corev1.Pod) string { for _, container := range pod.Spec.Containers { for _, port := range container.Ports { diff --git a/vendor/github.com/operator-framework/operator-lifecycle-manager/cmd/olm/main.go b/vendor/github.com/operator-framework/operator-lifecycle-manager/cmd/olm/main.go index 33fd6f709c..dd67a27a79 100644 --- a/vendor/github.com/operator-framework/operator-lifecycle-manager/cmd/olm/main.go +++ b/vendor/github.com/operator-framework/operator-lifecycle-manager/cmd/olm/main.go @@ -171,6 +171,11 @@ func main() { op.Run(ctx) <-op.Ready() + // Emit CSV metric + if err = op.EnsureCSVMetric(); err != nil { + logger.WithError(err).Fatalf("error emitting metrics for existing CSV") + } + if *writeStatusName != "" { reconciler, err := openshift.NewClusterOperatorReconciler( openshift.WithClient(mgr.GetClient()), diff --git a/vendor/github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/olm/operator.go b/vendor/github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/olm/operator.go index 2ab92f2a65..23a6e021c0 100644 --- a/vendor/github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/olm/operator.go +++ b/vendor/github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/olm/operator.go @@ -677,6 +677,23 @@ func (a *Operator) RegisterCSVWatchNotification(csvNotification csvutility.Watch a.csvNotification = csvNotification } +func (a *Operator) EnsureCSVMetric() error { + csvs, err := a.lister.OperatorsV1alpha1().ClusterServiceVersionLister().List(labels.Everything()) + if err != nil { + return err + } + for _, csv := range csvs { + logger := a.logger.WithFields(logrus.Fields{ + "name": csv.GetName(), + "namespace": csv.GetNamespace(), + "self": csv.GetSelfLink(), + }) + logger.Debug("emitting metrics for existing CSV") + metrics.EmitCSVMetric(csv, csv) + } + return nil +} + func (a *Operator) syncGCObject(obj interface{}) (syncError error) { metaObj, ok := obj.(metav1.Object) if !ok { From 7baca835c3e1b4da0040dd10fc20010c48d4364b Mon Sep 17 00:00:00 2001 From: "Akihiko (Aki) Kuroda" <16141898+akihikokuroda@users.noreply.github.com> Date: Mon, 20 Dec 2021 11:52:30 -0500 Subject: [PATCH 2/2] fix e2e CSV metric is preserved failure (#2530) Signed-off-by: akihikokuroda Upstream-repository: operator-lifecycle-manager Upstream-commit: 33b081aca79dcfc33895a5c114976c4f284f027f --- .../test/e2e/metrics_e2e_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/staging/operator-lifecycle-manager/test/e2e/metrics_e2e_test.go b/staging/operator-lifecycle-manager/test/e2e/metrics_e2e_test.go index 2769091e3a..ee03e3c30c 100644 --- a/staging/operator-lifecycle-manager/test/e2e/metrics_e2e_test.go +++ b/staging/operator-lifecycle-manager/test/e2e/metrics_e2e_test.go @@ -149,9 +149,13 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() { restartDeploymentWithLabel(c, "app=olm-operator") }) It("CSV metric is preserved", func() { - Expect(getMetricsFromPod(c, getPodWithLabel(c, "app=olm-operator"))).To( - ContainElement(LikeMetric(WithFamily("csv_succeeded"), WithName(csv.Name), WithValue(1))), - ) + Eventually(func() []Metric { + return getMetricsFromPod(c, getPodWithLabel(c, "app=olm-operator")) + }).Should(ContainElement(LikeMetric( + WithFamily("csv_succeeded"), + WithName(csv.Name), + WithValue(1), + ))) }) }) })