Skip to content

Commit

Permalink
Emit CSV metric on startup
Browse files Browse the repository at this point in the history
Signed-off-by: Josef Karasek <jkarasek@redhat.com>
  • Loading branch information
Josef Karasek committed Aug 24, 2021
1 parent 35deef2 commit 969ed27
Show file tree
Hide file tree
Showing 3 changed files with 106 additions and 0 deletions.
5 changes: 5 additions & 0 deletions cmd/olm/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,11 @@ func main() {
op.Run(ctx)
<-op.Ready()

// Emit CSV metric
if err = op.EnsureCSVMetric(); err != nil {
logger.WithError(err).Fatalf("error emitting metrics for existing CSV")
}

if *writeStatusName != "" {
reconciler, err := openshift.NewClusterOperatorReconciler(
openshift.WithClient(mgr.GetClient()),
Expand Down
17 changes: 17 additions & 0 deletions pkg/controller/operators/olm/operator.go
Original file line number Diff line number Diff line change
Expand Up @@ -655,6 +655,23 @@ func (a *Operator) RegisterCSVWatchNotification(csvNotification csvutility.Watch
a.csvNotification = csvNotification
}

func (a *Operator) EnsureCSVMetric() error {
csvs, err := a.lister.OperatorsV1alpha1().ClusterServiceVersionLister().List(labels.Everything())
if err != nil {
return err
}
for _, csv := range csvs {
logger := a.logger.WithFields(logrus.Fields{
"name": csv.GetName(),
"namespace": csv.GetNamespace(),
"self": csv.GetSelfLink(),
})
logger.Debug("emitting metrics for existing CSV")
metrics.EmitCSVMetric(csv, csv)
}
return nil
}

func (a *Operator) syncGCObject(obj interface{}) (syncError error) {
metaObj, ok := obj.(metav1.Object)
if !ok {
Expand Down
84 changes: 84 additions & 0 deletions test/e2e/metrics_e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
. "github.com/onsi/gomega"
io_prometheus_client "github.com/prometheus/client_model/go"
"github.com/prometheus/common/expfmt"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -116,6 +117,44 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
})
})
})

When("a CSV is created", func() {
var (
cleanupCSV cleanupFunc
csv v1alpha1.ClusterServiceVersion
)
BeforeEach(func() {
packageName := genName("csv-test-")
packageStable := fmt.Sprintf("%s-stable", packageName)
csv = newCSV(packageStable, testNamespace, "", semver.MustParse("0.1.0"), nil, nil, nil)

var err error
_, err = createCSV(c, crc, csv, testNamespace, false, false)
Expect(err).ToNot(HaveOccurred())
_, err = fetchCSV(crc, csv.Name, testNamespace, csvSucceededChecker)
Expect(err).ToNot(HaveOccurred())
})
AfterEach(func() {
if cleanupCSV != nil {
cleanupCSV()
}
})
It("emits a CSV metrics", func() {
Expect(getMetricsFromPod(c, getPodWithLabel(c, "app=olm-operator"))).To(
ContainElement(LikeMetric(WithFamily("csv_succeeded"), WithName(csv.Name), WithValue(1))),
)
})
When("the OLM pod restarts", func() {
BeforeEach(func() {
restartDeploymentWithLabel(c, "app=olm-operator")
})
It("CSV metric is preserved", func() {
Expect(getMetricsFromPod(c, getPodWithLabel(c, "app=olm-operator"))).To(
ContainElement(LikeMetric(WithFamily("csv_succeeded"), WithName(csv.Name), WithValue(1))),
)
})
})
})
})

Context("Metrics emitted by objects during operator installation", func() {
Expand Down Expand Up @@ -396,6 +435,51 @@ func getPodWithLabel(client operatorclient.ClientInterface, label string) *corev
return &podList.Items[0]
}

func getDeploymentWithLabel(client operatorclient.ClientInterface, label string) *appsv1.Deployment {
listOptions := metav1.ListOptions{LabelSelector: label}
var deploymentList *appsv1.DeploymentList
EventuallyWithOffset(1, func() (numDeps int, err error) {
deploymentList, err = client.KubernetesInterface().AppsV1().Deployments(operatorNamespace).List(context.TODO(), listOptions)
if deploymentList != nil {
numDeps = len(deploymentList.Items)
}

return
}).Should(Equal(1), "expected exactly one Deployment")

return &deploymentList.Items[0]
}

func restartDeploymentWithLabel(client operatorclient.ClientInterface, l string) {
d := getDeploymentWithLabel(client, l)
z := int32(0)
oldZ := *d.Spec.Replicas
d.Spec.Replicas = &z
_, err := client.KubernetesInterface().AppsV1().Deployments(operatorNamespace).Update(context.TODO(), d, metav1.UpdateOptions{})
Expect(err).ToNot(HaveOccurred())

EventuallyWithOffset(1, func() (replicas int32, err error) {
deployment, err := client.KubernetesInterface().AppsV1().Deployments(operatorNamespace).Get(context.TODO(), d.Name, metav1.GetOptions{})
if deployment != nil {
replicas = deployment.Status.Replicas
}
return
}).Should(Equal(int32(0)), "expected exactly 0 Deployments")

updated := getDeploymentWithLabel(client, l)
updated.Spec.Replicas = &oldZ
_, err = client.KubernetesInterface().AppsV1().Deployments(operatorNamespace).Update(context.TODO(), updated, metav1.UpdateOptions{})
Expect(err).ToNot(HaveOccurred())

EventuallyWithOffset(1, func() (replicas int32, err error) {
deployment, err := client.KubernetesInterface().AppsV1().Deployments(operatorNamespace).Get(context.TODO(), d.Name, metav1.GetOptions{})
if deployment != nil {
replicas = deployment.Status.Replicas
}
return
}).Should(Equal(oldZ), "expected exactly 1 Deployment")
}

func extractMetricPortFromPod(pod *corev1.Pod) string {
for _, container := range pod.Spec.Containers {
for _, port := range container.Ports {
Expand Down

0 comments on commit 969ed27

Please sign in to comment.