Update monitoring test to check operator generated metrics

tests/e2e-openshift/component-replicas/install-tempo-assert.yaml

@@ -233,6 +233,7 @@ spec:
       - args:
         - --log.level=warn
         - --opa.admin-groups=system:cluster-admins,cluster-admin,dedicated-admin
+        - --opa.matcher=kubernetes_namespace_name
         - --web.listen=:8082
         - --web.internal.listen=:8083
         - --web.healthchecks.url=http://localhost:8082

tests/e2e-openshift/monitoring/chainsaw-test.yaml

@@ -21,6 +21,31 @@ spec:
         file: 01-workload-monitoring.yaml
     - assert:
         file: 01-assert.yaml
+  - name: Enable OpenShift platform monitoring on the Tempo operator namespace
+    try:
+    - command:
+        entrypoint: oc
+        args:
+        - get
+        - pods
+        - -A
+        - -l control-plane=controller-manager
+        - -l app.kubernetes.io/name=tempo-operator
+        - -o
+        - jsonpath={.items[0].metadata.namespace}
+        outputs:
+        - name: TEMPO_NAMESPACE
+          value: ($stdout)
+    - command:
+        env:
+        - name: temponamespace
+          value: ($TEMPO_NAMESPACE)
+        entrypoint: oc
+        args:
+        - label
+        - namespace
+        - $temponamespace 
+        - openshift.io/cluster-monitoring=true
   - name: step-02
     try:
     - apply:
@@ -41,6 +66,21 @@ spec:
         file: 04-assert.yaml
   - name: step-05
     try:
+    - command:
+        entrypoint: oc
+        args:
+        - get
+        - pods
+        - -A
+        - -l control-plane=controller-manager
+        - -l app.kubernetes.io/name=tempo-operator
+        - -o 
+        - jsonpath={.items[0].metadata.namespace}
+        outputs:
+        - name: TEMPO_NAMESPACE
+          value: ($stdout)
     - script:
         timeout: 5m
         content: ./check_metrics.sh
+    - assert:
+        file: check_operator_servicemonitor.yaml

tests/e2e-openshift/monitoring/check_metrics.sh

@@ -4,7 +4,7 @@ TOKEN=$(oc create token prometheus-user-workload -n openshift-user-workload-moni
 THANOS_QUERIER_HOST=$(oc get route thanos-querier -n openshift-monitoring -o json | jq -r '.spec.host')
 
 #Check metrics used in the prometheus rules created for TempoStack. Refer issue https://issues.redhat.com/browse/TRACING-3399 for skipped metrics.
-metrics="tempo_request_duration_seconds_count tempo_request_duration_seconds_sum tempo_request_duration_seconds_bucket tempo_build_info tempo_ingester_bytes_received_total tempo_ingester_flush_failed_retries_total tempo_ingester_failed_flushes_total tempo_ring_members"
+metrics="tempo_request_duration_seconds_count tempo_request_duration_seconds_sum tempo_request_duration_seconds_bucket tempo_build_info tempo_ingester_bytes_received_total tempo_ingester_flush_failed_retries_total tempo_ingester_failed_flushes_total tempo_ring_members tempo_operator_tempostack_managed tempo_operator_tempostack_storage_backend tempo_operator_tempostack_multi_tenancy"
 
 for metric in $metrics; do
 query="$metric"

tests/e2e-openshift/monitoring/check_operator_servicemonitor.yaml

@@ -0,0 +1,50 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
+    app.kubernetes.io/name: tempo-operator
+    app.kubernetes.io/part-of: tempo-operator
+    control-plane: controller-manager
+    olm.managed: "true"
+  name: tempo-operator-controller-manager-metrics-service
+  namespace: ($TEMPO_NAMESPACE)
+spec:
+  ports:
+  - name: https
+    port: 8443
+    protocol: TCP
+    targetPort: https
+  selector:
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
+    app.kubernetes.io/name: tempo-operator
+    app.kubernetes.io/part-of: tempo-operator
+    control-plane: controller-manager
+  sessionAffinity: None
+  type: ClusterIP
+
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
+    app.kubernetes.io/name: tempo-operator
+    app.kubernetes.io/part-of: tempo-operator
+    control-plane: controller-manager
+  name: tempo-operator-controller-manager-metrics-monitor
+  namespace: ($TEMPO_NAMESPACE)
+spec:
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    path: /metrics
+    port: https
+    scheme: https
+    tlsConfig:
+      ca: {}
+      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+      cert: {}
+  namespaceSelector: {}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: tempo-operator