From 8ea42c86fd2971d0d7ee488cd4fdb97f84cc3462 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Wed, 2 Sep 2020 12:55:42 +0300 Subject: [PATCH] Improve docs of leaderelection configuration (#20601) (cherry picked from commit 6d7213ff717785febd904dd3d8d39ccb2d246c9e) --- deploy/kubernetes/metricbeat-kubernetes.yaml | 196 +++--------------- .../metricbeat-daemonset-configmap.yaml | 64 +++--- .../metricbeat-deployment-configmap.yaml | 59 ------ .../metricbeat/metricbeat-deployment.yaml | 71 ------- .../docs/running-on-kubernetes.asciidoc | 53 ++--- 5 files changed, 70 insertions(+), 373 deletions(-) delete mode 100644 deploy/kubernetes/metricbeat/metricbeat-deployment-configmap.yaml delete mode 100644 deploy/kubernetes/metricbeat/metricbeat-deployment.yaml diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml index 6baf35b6809..1cb34fdffff 100644 --- a/deploy/kubernetes/metricbeat-kubernetes.yaml +++ b/deploy/kubernetes/metricbeat-kubernetes.yaml @@ -16,45 +16,39 @@ data: metricbeat.autodiscover: providers: + - type: kubernetes + scope: cluster + node: ${NODE_NAME} + unique: true + templates: + - config: + - module: kubernetes + hosts: ["kube-state-metrics:8080"] + period: 10s + add_metadata: true + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_pod + - state_container + - state_cronjob + - state_resourcequota + - state_statefulset + # Uncomment this to get k8s events: + #- event + - module: kubernetes + metricsets: + - apiserver + hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.certificate_authorities: + - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + period: 30s # To enable hints based autodiscover uncomment this: #- type: kubernetes # node: ${NODE_NAME} # hints.enabled: true - # Uncomment the following to enable leader election provider that handles - # singleton instance configuration across the Daemonset Pods of the whole cluster - # in order to monitor some unique data sources, like kube-state-metrics. - # When enabling this remember to also delete the Deployment or just set the replicas of the - # Deployment to 0. - #- type: kubernetes - # scope: cluster - # node: ${NODE_NAME} - # unique: true - # # identifier: - # templates: - # - config: - # - module: kubernetes - # hosts: ["kube-state-metrics:8080"] - # period: 10s - # add_metadata: true - # metricsets: - # - state_node - # - state_deployment - # - state_replicaset - # - state_pod - # - state_container - # - state_cronjob - # - state_resourcequota - # - state_statefulset - # # Uncomment this to get k8s events: - # #- event - # - module: kubernetes - # metricsets: - # - apiserver - # hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] - # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - # ssl.certificate_authorities: - # - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - # period: 30s processors: - add_cloud_metadata: @@ -224,138 +218,6 @@ spec: path: /var/lib/metricbeat-data type: DirectoryOrCreate --- -apiVersion: v1 -kind: ConfigMap -metadata: - name: metricbeat-deployment-config - namespace: kube-system - labels: - k8s-app: metricbeat -data: - metricbeat.yml: |- - metricbeat.config.modules: - # Mounted `metricbeat-daemonset-modules` configmap: - path: ${path.config}/modules.d/*.yml - # Reload module configs as they change: - reload.enabled: false - - processors: - - add_cloud_metadata: - - cloud.id: ${ELASTIC_CLOUD_ID} - cloud.auth: ${ELASTIC_CLOUD_AUTH} - - output.elasticsearch: - hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] - username: ${ELASTICSEARCH_USERNAME} - password: ${ELASTICSEARCH_PASSWORD} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: metricbeat-deployment-modules - namespace: kube-system - labels: - k8s-app: metricbeat -data: - # This module requires `kube-state-metrics` up and running under `kube-system` namespace - kubernetes.yml: |- - - module: kubernetes - metricsets: - - state_node - - state_deployment - - state_replicaset - - state_pod - - state_container - - state_cronjob - - state_resourcequota - - state_statefulset - # Uncomment this to get k8s events: - #- event - period: 10s - host: ${NODE_NAME} - hosts: ["kube-state-metrics:8080"] - #- module: kubernetes - # metricsets: - # - apiserver - # hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] - # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - # ssl.certificate_authorities: - # - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - # period: 30s ---- -# Deploy singleton instance in the whole cluster for some unique data sources, like kube-state-metrics -apiVersion: apps/v1 -kind: Deployment -metadata: - name: metricbeat - namespace: kube-system - labels: - k8s-app: metricbeat -spec: - # Set to 0 if using leader election provider with the Daemonset - replicas: 1 - selector: - matchLabels: - k8s-app: metricbeat - template: - metadata: - labels: - k8s-app: metricbeat - spec: - serviceAccountName: metricbeat - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: metricbeat - image: docker.elastic.co/beats/metricbeat:7.10.0 - args: [ - "-c", "/etc/metricbeat.yml", - "-e", - ] - env: - - name: ELASTICSEARCH_HOST - value: elasticsearch - - name: ELASTICSEARCH_PORT - value: "9200" - - name: ELASTICSEARCH_USERNAME - value: elastic - - name: ELASTICSEARCH_PASSWORD - value: changeme - - name: ELASTIC_CLOUD_ID - value: - - name: ELASTIC_CLOUD_AUTH - value: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - runAsUser: 0 - resources: - limits: - memory: 200Mi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - name: config - mountPath: /etc/metricbeat.yml - readOnly: true - subPath: metricbeat.yml - - name: modules - mountPath: /usr/share/metricbeat/modules.d - readOnly: true - volumes: - - name: config - configMap: - defaultMode: 0640 - name: metricbeat-deployment-config - - name: modules - configMap: - defaultMode: 0640 - name: metricbeat-deployment-modules ---- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml index 3b0da9cc503..34448da650f 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml @@ -16,45 +16,39 @@ data: metricbeat.autodiscover: providers: + - type: kubernetes + scope: cluster + node: ${NODE_NAME} + unique: true + templates: + - config: + - module: kubernetes + hosts: ["kube-state-metrics:8080"] + period: 10s + add_metadata: true + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_pod + - state_container + - state_cronjob + - state_resourcequota + - state_statefulset + # Uncomment this to get k8s events: + #- event + - module: kubernetes + metricsets: + - apiserver + hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.certificate_authorities: + - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + period: 30s # To enable hints based autodiscover uncomment this: #- type: kubernetes # node: ${NODE_NAME} # hints.enabled: true - # Uncomment the following to enable leader election provider that handles - # singleton instance configuration across the Daemonset Pods of the whole cluster - # in order to monitor some unique data sources, like kube-state-metrics. - # When enabling this remember to also delete the Deployment or just set the replicas of the - # Deployment to 0. - #- type: kubernetes - # scope: cluster - # node: ${NODE_NAME} - # unique: true - # # identifier: - # templates: - # - config: - # - module: kubernetes - # hosts: ["kube-state-metrics:8080"] - # period: 10s - # add_metadata: true - # metricsets: - # - state_node - # - state_deployment - # - state_replicaset - # - state_pod - # - state_container - # - state_cronjob - # - state_resourcequota - # - state_statefulset - # # Uncomment this to get k8s events: - # #- event - # - module: kubernetes - # metricsets: - # - apiserver - # hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] - # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - # ssl.certificate_authorities: - # - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - # period: 30s processors: - add_cloud_metadata: diff --git a/deploy/kubernetes/metricbeat/metricbeat-deployment-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-deployment-configmap.yaml deleted file mode 100644 index 1e492e61259..00000000000 --- a/deploy/kubernetes/metricbeat/metricbeat-deployment-configmap.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: metricbeat-deployment-config - namespace: kube-system - labels: - k8s-app: metricbeat -data: - metricbeat.yml: |- - metricbeat.config.modules: - # Mounted `metricbeat-daemonset-modules` configmap: - path: ${path.config}/modules.d/*.yml - # Reload module configs as they change: - reload.enabled: false - - processors: - - add_cloud_metadata: - - cloud.id: ${ELASTIC_CLOUD_ID} - cloud.auth: ${ELASTIC_CLOUD_AUTH} - - output.elasticsearch: - hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] - username: ${ELASTICSEARCH_USERNAME} - password: ${ELASTICSEARCH_PASSWORD} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: metricbeat-deployment-modules - namespace: kube-system - labels: - k8s-app: metricbeat -data: - # This module requires `kube-state-metrics` up and running under `kube-system` namespace - kubernetes.yml: |- - - module: kubernetes - metricsets: - - state_node - - state_deployment - - state_replicaset - - state_pod - - state_container - - state_cronjob - - state_resourcequota - - state_statefulset - # Uncomment this to get k8s events: - #- event - period: 10s - host: ${NODE_NAME} - hosts: ["kube-state-metrics:8080"] - #- module: kubernetes - # metricsets: - # - apiserver - # hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] - # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - # ssl.certificate_authorities: - # - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - # period: 30s diff --git a/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml b/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml deleted file mode 100644 index f82b204b63d..00000000000 --- a/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml +++ /dev/null @@ -1,71 +0,0 @@ -# Deploy singleton instance in the whole cluster for some unique data sources, like kube-state-metrics -apiVersion: apps/v1 -kind: Deployment -metadata: - name: metricbeat - namespace: kube-system - labels: - k8s-app: metricbeat -spec: - # Set to 0 if using leader election provider with the Daemonset - replicas: 1 - selector: - matchLabels: - k8s-app: metricbeat - template: - metadata: - labels: - k8s-app: metricbeat - spec: - serviceAccountName: metricbeat - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: metricbeat - image: docker.elastic.co/beats/metricbeat:%VERSION% - args: [ - "-c", "/etc/metricbeat.yml", - "-e", - ] - env: - - name: ELASTICSEARCH_HOST - value: elasticsearch - - name: ELASTICSEARCH_PORT - value: "9200" - - name: ELASTICSEARCH_USERNAME - value: elastic - - name: ELASTICSEARCH_PASSWORD - value: changeme - - name: ELASTIC_CLOUD_ID - value: - - name: ELASTIC_CLOUD_AUTH - value: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - runAsUser: 0 - resources: - limits: - memory: 200Mi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - name: config - mountPath: /etc/metricbeat.yml - readOnly: true - subPath: metricbeat.yml - - name: modules - mountPath: /usr/share/metricbeat/modules.d - readOnly: true - volumes: - - name: config - configMap: - defaultMode: 0640 - name: metricbeat-deployment-config - - name: modules - configMap: - defaultMode: 0640 - name: metricbeat-deployment-modules diff --git a/metricbeat/docs/running-on-kubernetes.asciidoc b/metricbeat/docs/running-on-kubernetes.asciidoc index 8e5226431b3..c360394522a 100644 --- a/metricbeat/docs/running-on-kubernetes.asciidoc +++ b/metricbeat/docs/running-on-kubernetes.asciidoc @@ -17,18 +17,21 @@ endif::[] [float] ==== Kubernetes deploy manifests -You deploy {beatname_uc} in two different ways at the same time: - -* As a https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/[DaemonSet] +You deploy {beatname_uc} as a https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/[DaemonSet] to ensure that there's a running instance on each node of the cluster. These instances are used to retrieve most metrics from the host, such as system metrics, Docker stats, and metrics from all the services running on top of Kubernetes. -* As a single {beatname_uc} instance created using a https://kubernetes.io/docs/concepts/workloads/controllers/Deployment/[Deployment]. +In addition, one of the Pods in the DaemonSet will constantly hold a _leader lock_ which makes it responsible for +handling cluster-wide monitoring. This instance is used to retrieve metrics that are unique for the whole cluster, such as Kubernetes events or https://github.com/kubernetes/kube-state-metrics[kube-state-metrics]. +You can find more information about leader election configuration options at <>. + +Note: If you are upgrading from older versions, please make sure there are no redundant parts +as left-overs from the old manifests. Deployment specification and its ConfigMaps might be the case. Everything is deployed under the `kube-system` namespace by default. To change the namespace, modify the manifest file. @@ -185,41 +188,9 @@ Metrics should start flowing to Elasticsearch. [float] -==== Deploying Metricbeat Daemonset with Leader Election enabled - -In some cases users may want to avoid deploying both a Deployment and a Daemonset -to collect cluser-wide metrics and node-level metrics. -For this case, we provide the option to deploy Metricbeat only as a Daemonset -and leverage the leader election feature which allows to define configurations -that are enabled only by the leader Pod. The leader Pod is automatically chosen -between the Pods of the Daemonset. -Here is an example of a configuration that enables leader election: -[source,yaml] ------ -metricbeat.autodiscover: - providers: - - type: kubernetes - scope: cluster - node: ${NODE_NAME} - unique: true - identifier: leaderelectionmetricbeat - templates: - - config: - - module: kubernetes - hosts: ["kube-state-metrics:8080"] - period: 10s - add_metadata: true - metricsets: - - state_node ------ -Users can find more info about the `unique` and `identifier` options at <>. +==== Deploying Metricbeat to collect cluster-level metrics in large clusters -Users can enable the respective parts the Daemonset ConfigMap and -set the `replicas` of the Deployment to `0` in order to only deploy -the Daemonset on the cluster with the leader election provider enabled -in order to collect cluster-wide metrics: -["source", "sh", subs="attributes"] ------------------------------------------------- -curl -L -O https://raw.githubusercontent.com/elastic/beats/{branch}/deploy/kubernetes/metricbeat-kubernetes.yaml -kubectl create -f metricbeat-kubernetes.yaml ------------------------------------------------- +The size and the number of nodes in a Kubernetes cluster can be fairly large at times, and in such cases +the Pod that will be collecting cluster level metrics might face performance issues due to +resources limitations. In this case users might consider to avoid using the leader election strategy +and instead run a dedicated, standalone Metribceat instance using a Deployment in addition to the DaemonSet.