Make eventing components observable (#1722)

* Make eventing controller observable - Enable scrapping of eventing-controller - Add eventing reconciler dasshboard * Retain vendor license * Removed eventing reconciler dashboard * Make broker filter and ingress scrapeable * Removed hard coded annotation for prometheus - Refer https://github.com/coreos/kube-prometheus/pull/16\#issuecomment-305933103 * Made sources-controller observable * Remove sources metrics service * Fixed as per PR comments
knative · Aug 27, 2019 · 359776f · 359776f
1 parent a6f4e5a
commit 359776f
Show file tree

Hide file tree

Showing 6 changed files with 148 additions and 59 deletions.
diff --git a/config/400-source-controller-service.yaml b/config/400-source-controller-service.yaml
diff --git a/config/monitoring/metrics/prometheus/100-prometheus-scrape-kn-eventing.yaml b/config/monitoring/metrics/prometheus/100-prometheus-scrape-kn-eventing.yaml
@@ -0,0 +1,72 @@
+# Eventing controller
+- job_name: eventing-controller
+  scrape_interval: 3s
+  scrape_timeout: 3s
+  kubernetes_sd_configs:
+  - role: pod
+  relabel_configs:
+  # Scrape only the the targets matching the following metadata
+  - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_label_app, __meta_kubernetes_pod_container_port_name]
+    action: keep
+    regex: knative-eventing;eventing-controller;metrics
+  # Rename metadata labels to be reader friendly
+  - source_labels: [__meta_kubernetes_namespace]
+    target_label: namespace
+  - source_labels: [__meta_kubernetes_pod_name]
+    target_label: pod
+  - source_labels: [__meta_kubernetes_service_name]
+    target_label: service
+# Broker filter
+- job_name: broker-filter
+  scrape_interval: 3s
+  scrape_timeout: 3s
+  kubernetes_sd_configs:
+  - role: pod
+  relabel_configs:
+  # Scrape only the the targets matching the following metadata
+  - source_labels: [__meta_kubernetes_pod_label_eventing_knative_dev_brokerRole, __meta_kubernetes_pod_container_port_name]
+    action: keep
+    regex: filter;metrics
+  # Rename metadata labels to be reader friendly
+  - source_labels: [__meta_kubernetes_namespace]
+    target_label: namespace
+  - source_labels: [__meta_kubernetes_pod_name]
+    target_label: pod
+  - source_labels: [__meta_kubernetes_service_name]
+    target_label: service
+# Broker ingress
+- job_name: broker-ingress
+  scrape_interval: 3s
+  scrape_timeout: 3s
+  kubernetes_sd_configs:
+  - role: pod
+  relabel_configs:
+  # Scrape only the the targets matching the following metadata
+  - source_labels: [__meta_kubernetes_pod_label_eventing_knative_dev_brokerRole, __meta_kubernetes_pod_container_port_name]
+    action: keep
+    regex: ingress;metrics
+  # Rename metadata labels to be reader friendly
+  - source_labels: [__meta_kubernetes_namespace]
+    target_label: namespace
+  - source_labels: [__meta_kubernetes_pod_name]
+    target_label: pod
+  - source_labels: [__meta_kubernetes_service_name]
+    target_label: service
+# Sources controller
+- job_name: sources-controller
+  scrape_interval: 3s
+  scrape_timeout: 3s
+  kubernetes_sd_configs:
+  - role: pod
+  relabel_configs:
+  # Scrape only the the targets matching the following metadata
+  - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_label_app, __meta_kubernetes_pod_container_port_name]
+    action: keep
+    regex: knative-eventing;sources-controller;metrics
+  # Rename metadata labels to be reader friendly
+  - source_labels: [__meta_kubernetes_namespace]
+    target_label: namespace
+  - source_labels: [__meta_kubernetes_pod_name]
+    target_label: pod
+  - source_labels: [__meta_kubernetes_service_name]
+    target_label: service
diff --git a/docs/metrics.md b/docs/metrics.md
@@ -28,6 +28,77 @@ Accessing metrics requires Prometheus and Grafana installed. Follow the
 [instructions to install Prometheus and Grafana](https://github.com/knative/docs/blob/master/docs/serving/installing-logging-metrics-traces.md)
 in namespace `knative-monitoring`.
 
+
+## Prometheus Collection
+> _All commands assume root of repo._ 
+
+1. Enable Knatives install of Prometheus to scrape Knative with GCP, run the
+   following:
+
+   ```shell
+   kubectl get configmap -n  knative-monitoring prometheus-scrape-config -oyaml > tmp.prometheus-scrape-config.yaml
+   sed -e 's/^/    /' config/monitoring/metrics/prometheus/100-prometheus-scrape-kn-eventing.yaml > tmp.prometheus-scrape-kn-eventing.yaml
+   sed -e '/    scrape_configs:/r tmp.prometheus-scrape-kn-eventing.yaml' tmp.prometheus-scrape-config.yaml \
+     | kubectl apply -f -
+   ```
+
+2. To verify, run the following to show the diff between what the resource was
+   original and is now (_note_: k8s will update `metadata.annotations`):
+
+   ```shell
+   kubectl get configmap -n  knative-monitoring prometheus-scrape-config -oyaml \
+     | diff - tmp.prometheus-scrape-config.yaml
+   ```
+
+   Or, to just see our changes (without `metadata.annotations`) run:
+
+   ```shell
+   CHANGED_LINES=$(echo $(cat config/monitoring/metrics/prometheus/100-prometheus-scrape-kn-eventing.yaml | wc -l) + 1 | bc)
+   kubectl get configmap -n  knative-monitoring prometheus-scrape-config -oyaml \
+     | diff - tmp.prometheus-scrape-config.yaml \
+     | head -n $CHANGED_LINES
+   ```
+
+3. Restart Prometheus
+
+   To pick up this new config, the pods that run Prometheus need to be
+   restarted, run:
+
+   ```shell
+   kubectl delete pods -n knative-monitoring prometheus-system-0 prometheus-system-1
+   ```
+
+   And they will come back:
+
+   ```shell
+   $ kubectl get pods -n knative-monitoring
+   NAME                                 READY   STATUS    RESTARTS   AGE
+   grafana-d7478555c-8qgf7              1/1     Running   0          22h
+   kube-state-metrics-765d876c6-z7dfn   4/4     Running   0          22h
+   node-exporter-5m9cz                  2/2     Running   0          22h
+   node-exporter-z59gz                  2/2     Running   0          22h
+   prometheus-system-0                  1/1     Running   0          32s
+   prometheus-system-1                  1/1     Running   0          36s
+   ```
+
+4. To remove the temp files, run:
+
+   ```shell
+   rm tmp.prometheus-scrape-kn-eventing.yaml
+   rm tmp.prometheus-scrape-config.yaml
+   ```
+
+#### Remove Scrape Config
+
+Remove the text related to Cloud Run Events from `prometheus-scrape-config`, 
+
+```shell
+kubectl edit configmap -n  knative-monitoring prometheus-scrape-config
+```
+
+And then restart Prometheus.
+
+
 ### Prometheus
 
 Follow the

diff --git a/pkg/reconciler/broker/broker_test.go b/pkg/reconciler/broker/broker_test.go
@@ -277,7 +277,6 @@ func TestReconcileCRD(t *testing.T) {
 			WantCreates: []runtime.Object{
 				NewService(filterServiceName, testNS,
 					WithServiceOwnerReferences(ownerReferences()),
-					WithServiceAnnotations(resources.FilterAnnotations()),
 					WithServiceLabels(resources.FilterLabels(brokerName)),
 					WithServicePorts(servicePorts(filterContainerName, 8080))),
 			},
@@ -453,7 +452,6 @@ func TestReconcileCRD(t *testing.T) {
 			WantCreates: []runtime.Object{
 				NewService(ingressServiceName, testNS,
 					WithServiceOwnerReferences(ownerReferences()),
-					WithServiceAnnotations(resources.IngressAnnotations()),
 					WithServiceLabels(resources.IngressLabels(brokerName)),
 					WithServicePorts(servicePorts(ingressContainerName, 8080))),
 			},

diff --git a/pkg/reconciler/broker/resources/filter.go b/pkg/reconciler/broker/resources/filter.go
@@ -126,10 +126,9 @@ func MakeFilterDeployment(args *FilterArgs) *appsv1.Deployment {
 func MakeFilterService(b *eventingv1alpha1.Broker) *corev1.Service {
 	return &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Namespace:   b.Namespace,
-			Name:        fmt.Sprintf("%s-broker-filter", b.Name),
-			Annotations: FilterAnnotations(),
-			Labels:      FilterLabels(b.Name),
+			Namespace: b.Namespace,
+			Name:      fmt.Sprintf("%s-broker-filter", b.Name),
+			Labels:    FilterLabels(b.Name),
 			OwnerReferences: []metav1.OwnerReference{
 				*kmeta.NewControllerRef(b),
 			},
@@ -159,12 +158,3 @@ func FilterLabels(brokerName string) map[string]string {
 		"eventing.knative.dev/brokerRole": "filter",
 	}
 }
-
-// FilterAnnotations generates the annotation that allow Prometheus to scrape the metrics exposed
-// by this service.
-func FilterAnnotations() map[string]string {
-	return map[string]string{
-		"prometheus.io/scrape": "true",
-		"prometheus.io/port":   "9090",
-	}
-}
diff --git a/pkg/reconciler/broker/resources/ingress.go b/pkg/reconciler/broker/resources/ingress.go
@@ -127,9 +127,8 @@ func MakeIngressService(b *eventingv1alpha1.Broker) *corev1.Service {
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: b.Namespace,
 			// TODO add -ingress to the name to be consistent with the filter service naming.
-			Name:        fmt.Sprintf("%s-broker", b.Name),
-			Annotations: IngressAnnotations(),
-			Labels:      IngressLabels(b.Name),
+			Name:   fmt.Sprintf("%s-broker", b.Name),
+			Labels: IngressLabels(b.Name),
 			OwnerReferences: []metav1.OwnerReference{
 				*kmeta.NewControllerRef(b),
 			},
@@ -159,12 +158,3 @@ func IngressLabels(brokerName string) map[string]string {
 		"eventing.knative.dev/brokerRole": "ingress",
 	}
 }
-
-// IngressAnnotations generates the annotation that allow Prometheus to scrape the metrics exposed
-// by this service.
-func IngressAnnotations() map[string]string {
-	return map[string]string{
-		"prometheus.io/scrape": "true",
-		"prometheus.io/port":   "9090",
-	}
-}