Check operator metrics (#3458)

tests/e2e-openshift/monitoring/03-assert.yaml

@@ -11,6 +11,7 @@ rules:
   - get
   - list
   - watch
+  - create
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

tests/e2e-openshift/monitoring/03-create-monitoring-roles.yaml

@@ -6,7 +6,7 @@ metadata:
 rules:
 - apiGroups: ["monitoring.coreos.com"]
   resources: ["prometheuses/api"]
-  verbs: ["get", "list", "watch"]
+  verbs: ["get", "list", "watch", "create"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

tests/e2e-openshift/monitoring/check_metrics.sh

@@ -3,23 +3,23 @@
 TOKEN=$(oc create token prometheus-user-workload -n openshift-user-workload-monitoring)
 THANOS_QUERIER_HOST=$(oc get route thanos-querier -n openshift-monitoring -o json | jq -r '.spec.host')
 
-#Check metrics for OpenTelemetry collector instance.
-metrics="otelcol_process_uptime otelcol_process_runtime_total_sys_memory_bytes otelcol_process_memory_rss otelcol_exporter_sent_spans otelcol_process_cpu_seconds otelcol_process_memory_rss otelcol_process_runtime_heap_alloc_bytes otelcol_process_runtime_total_alloc_bytes otelcol_process_runtime_total_sys_memory_bytes otelcol_process_uptime otelcol_receiver_accepted_spans otelcol_receiver_refused_spans opentelemetry_collector_info opentelemetry_collector_exporters opentelemetry_collector_receivers"
+# Check metrics for OpenTelemetry collector instance.
+metrics="otelcol_process_uptime otelcol_process_runtime_total_sys_memory_bytes otelcol_process_memory_rss otelcol_exporter_sent_spans otelcol_process_cpu_seconds otelcol_process_memory_rss otelcol_process_runtime_heap_alloc_bytes otelcol_process_runtime_total_alloc_bytes otelcol_process_runtime_total_sys_memory_bytes otelcol_process_uptime otelcol_receiver_accepted_spans otelcol_receiver_refused_spans controller_runtime_reconcile_time_seconds_count{controller=\"opentelemetrycollector\"} controller_runtime_reconcile_total{controller=\"opentelemetrycollector\",result=\"success\"} workqueue_work_duration_seconds_count{controller=\"opentelemetrycollector\",name=\"opentelemetrycollector\"}"
 
 for metric in $metrics; do
-query="$metric"
-count=0
+  query="$metric"
+  count=0
 
-# Keep fetching and checking the metrics until metrics with value is present.
-while [[ $count -eq 0 ]]; do
-    response=$(curl -k -H "Authorization: Bearer $TOKEN" -H "Content-type: application/json" "https://$THANOS_QUERIER_HOST/api/v1/query?query=$query")
-    count=$(echo "$response" | jq -r '.data.result | length')
+  # Keep fetching and checking the metrics until metrics with value is present.
+  while [[ $count -eq 0 ]]; do
+    response=$(curl -k -H "Authorization: Bearer $TOKEN" --data-urlencode "query=$query" "https://$THANOS_QUERIER_HOST/api/v1/query")
+    count=$(echo "$response" | jq -r '.data.result | length' | tr -d '\n' | tr -d ' ')
 
-    if [[ $count -eq 0 ]]; then
-    echo "No metric '$metric' with value present. Retrying..."
-    sleep 5  # Wait for 5 seconds before retrying
+    if [[ "$count" -eq 0 ]]; then
+      echo "No metric '$metric' with value present. Retrying..."
+      sleep 5  # Wait for 5 seconds before retrying
     else
-    echo "Metric '$metric' with value is present."
+      echo "Metric '$metric' with value is present."
     fi
   done
 done