Add k8s manifest leveraging leaderelection (#20512) (#20600)

(cherry picked from commit 7b7fb3b)

deploy/kubernetes/metricbeat-kubernetes.yaml

@@ -14,12 +14,47 @@ data:
       # Reload module configs as they change:
       reload.enabled: false
 
-    # To enable hints based autodiscover uncomment this:
-    #metricbeat.autodiscover:
-    #  providers:
-    #    - type: kubernetes
-    #      node: ${NODE_NAME}
-    #      hints.enabled: true
+    metricbeat.autodiscover:
+      providers:
+        # To enable hints based autodiscover uncomment this:
+        #- type: kubernetes
+        #  node: ${NODE_NAME}
+        #  hints.enabled: true
+        # Uncomment the following to enable leader election provider that handles
+        # singleton instance configuration across the Daemonset Pods of the whole cluster
+        # in order to monitor some unique data sources, like kube-state-metrics.
+        # When enabling this remember to also delete the Deployment or just set the replicas of the
+        # Deployment to 0.
+        #- type: kubernetes
+        #  scope: cluster
+        #  node: ${NODE_NAME}
+        #  unique: true
+        #  # identifier: <lease_lock_name>
+        #  templates:
+        #    - config:
+        #        - module: kubernetes
+        #          hosts: ["kube-state-metrics:8080"]
+        #          period: 10s
+        #          add_metadata: true
+        #          metricsets:
+        #            - state_node
+        #            - state_deployment
+        #            - state_replicaset
+        #            - state_pod
+        #            - state_container
+        #            - state_cronjob
+        #            - state_resourcequota
+        #            - state_statefulset
+        #            # Uncomment this to get k8s events:
+        #            #- event
+        #        - module: kubernetes
+        #          metricsets:
+        #            - apiserver
+        #          hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"]
+        #          bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+        #          ssl.certificate_authorities:
+        #            - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        #          period: 30s
 
     processors:
       - add_cloud_metadata:
@@ -258,6 +293,8 @@ metadata:
   labels:
     k8s-app: metricbeat
 spec:
+  # Set to 0 if using leader election provider with the Daemonset
+  replicas: 1
   selector:
     matchLabels:
       k8s-app: metricbeat

deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml

@@ -14,12 +14,47 @@ data:
       # Reload module configs as they change:
       reload.enabled: false
 
-    # To enable hints based autodiscover uncomment this:
-    #metricbeat.autodiscover:
-    #  providers:
-    #    - type: kubernetes
-    #      node: ${NODE_NAME}
-    #      hints.enabled: true
+    metricbeat.autodiscover:
+      providers:
+        # To enable hints based autodiscover uncomment this:
+        #- type: kubernetes
+        #  node: ${NODE_NAME}
+        #  hints.enabled: true
+        # Uncomment the following to enable leader election provider that handles
+        # singleton instance configuration across the Daemonset Pods of the whole cluster
+        # in order to monitor some unique data sources, like kube-state-metrics.
+        # When enabling this remember to also delete the Deployment or just set the replicas of the
+        # Deployment to 0.
+        #- type: kubernetes
+        #  scope: cluster
+        #  node: ${NODE_NAME}
+        #  unique: true
+        #  # identifier: <lease_lock_name>
+        #  templates:
+        #    - config:
+        #        - module: kubernetes
+        #          hosts: ["kube-state-metrics:8080"]
+        #          period: 10s
+        #          add_metadata: true
+        #          metricsets:
+        #            - state_node
+        #            - state_deployment
+        #            - state_replicaset
+        #            - state_pod
+        #            - state_container
+        #            - state_cronjob
+        #            - state_resourcequota
+        #            - state_statefulset
+        #            # Uncomment this to get k8s events:
+        #            #- event
+        #        - module: kubernetes
+        #          metricsets:
+        #            - apiserver
+        #          hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"]
+        #          bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+        #          ssl.certificate_authorities:
+        #            - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        #          period: 30s
 
     processors:
       - add_cloud_metadata:

deploy/kubernetes/metricbeat/metricbeat-deployment.yaml

@@ -7,6 +7,8 @@ metadata:
   labels:
     k8s-app: metricbeat
 spec:
+  # Set to 0 if using leader election provider with the Daemonset
+  replicas: 1
   selector:
     matchLabels:
       k8s-app: metricbeat

metricbeat/docs/running-on-kubernetes.asciidoc

@@ -182,3 +182,44 @@ metricbeat                1         1         1            1           1m
 ------------------------------------------------
 
 Metrics should start flowing to Elasticsearch.
+
+
+[float]
+==== Deploying Metricbeat Daemonset with Leader Election enabled
+
+In some cases users may want to avoid deploying both a Deployment and a Daemonset
+to collect cluser-wide metrics and node-level metrics.
+For this case, we provide the option to deploy Metricbeat only as a Daemonset
+and leverage the leader election feature which allows to define configurations
+that are enabled only by the leader Pod. The leader Pod is automatically chosen
+between the Pods of the Daemonset.
+Here is an example of a configuration that enables leader election:
+[source,yaml]
+-----
+metricbeat.autodiscover:
+  providers:
+  - type: kubernetes
+    scope: cluster
+    node: ${NODE_NAME}
+    unique: true
+    identifier: leaderelectionmetricbeat
+    templates:
+      - config:
+          - module: kubernetes
+            hosts: ["kube-state-metrics:8080"]
+            period: 10s
+            add_metadata: true
+            metricsets:
+              - state_node
+-----
+Users can find more info about the `unique` and `identifier` options at <<configuration-autodiscover>>.
+
+Users can enable the respective parts the Daemonset ConfigMap and
+set the `replicas` of the Deployment to `0` in order to only deploy
+the Daemonset on the cluster with the leader election provider enabled
+in order to collect cluster-wide metrics:
+["source", "sh", subs="attributes"]
+------------------------------------------------
+curl -L -O https://raw.githubusercontent.com/elastic/beats/{branch}/deploy/kubernetes/metricbeat-kubernetes.yaml
+kubectl create -f metricbeat-kubernetes.yaml
+------------------------------------------------