From 32c3b216deb4a6d34decdfb7b949b8c6b977f085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Mon, 1 Jul 2019 18:28:02 +0200 Subject: [PATCH] Use CRI paths in kubernetes manifests (#12632) We added a new `container` input in #12162, this change makes use of it to read logs from their CRI paths. Making Filebeat work with deployments that are not using Docker runtime. --- CHANGELOG.next.asciidoc | 1 + deploy/kubernetes/filebeat-kubernetes.yaml | 61 ++++++++----------- .../filebeat/filebeat-configmap.yaml | 44 ++++++------- .../filebeat/filebeat-daemonset.yaml | 17 +++--- 4 files changed, 55 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 28abef111b4..a48a27bb2c6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -211,6 +211,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add specific date processor to convert timezones so same pipeline can be used when convert_timezone is enabled or disabled. {pull}12253[12253] - Add MSSQL module {pull}12079[12079] - Add ISO8601 date parsing support for system module. {pull}12568[12568] {pull}12578[12579] +- Update Kubernetes deployment manifest to use `container` input. {pull}12632[12632] - Use correct OS path separator in `add_kubernetes_metadata` to support Windows nodes. {pull}9205[9205] - Add support for client addresses with port in Apache error logs {pull}12695[12695] diff --git a/deploy/kubernetes/filebeat-kubernetes.yaml b/deploy/kubernetes/filebeat-kubernetes.yaml index 1695f25e887..17b59df6c0f 100644 --- a/deploy/kubernetes/filebeat-kubernetes.yaml +++ b/deploy/kubernetes/filebeat-kubernetes.yaml @@ -8,22 +8,28 @@ metadata: k8s-app: filebeat data: filebeat.yml: |- - filebeat.config: - inputs: - # Mounted `filebeat-inputs` configmap: - path: ${path.config}/inputs.d/*.yml - # Reload inputs configs as they change: - reload.enabled: false - modules: - path: ${path.config}/modules.d/*.yml - # Reload module configs as they change: - reload.enabled: false + filebeat.inputs: + - type: container + paths: + - /var/log/containers/*.log + processors: + - add_kubernetes_metadata: + in_cluster: true + host: ${NODE_NAME} + matchers: + - logs_path: + logs_path: "/var/log/containers/" - # To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this: + # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this: #filebeat.autodiscover: # providers: # - type: kubernetes + # host: ${NODE_NAME} # hints.enabled: true + # hints.default_config: + # type: container + # paths: + # - /var/log/containers/*${data.kubernetes.container.id}.log processors: - add_cloud_metadata: @@ -36,22 +42,6 @@ data: username: ${ELASTICSEARCH_USERNAME} password: ${ELASTICSEARCH_PASSWORD} --- -apiVersion: v1 -kind: ConfigMap -metadata: - name: filebeat-inputs - namespace: kube-system - labels: - k8s-app: filebeat -data: - kubernetes.yml: |- - - type: docker - containers.ids: - - "*" - processors: - - add_kubernetes_metadata: - in_cluster: true ---- apiVersion: extensions/v1beta1 kind: DaemonSet metadata: @@ -87,6 +77,10 @@ spec: value: - name: ELASTIC_CLOUD_AUTH value: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName securityContext: runAsUser: 0 # If using Red Hat OpenShift uncomment this: @@ -102,14 +96,14 @@ spec: mountPath: /etc/filebeat.yml readOnly: true subPath: filebeat.yml - - name: inputs - mountPath: /usr/share/filebeat/inputs.d - readOnly: true - name: data mountPath: /usr/share/filebeat/data - name: varlibdockercontainers mountPath: /var/lib/docker/containers readOnly: true + - name: varlog + mountPath: /var/log + readOnly: true volumes: - name: config configMap: @@ -118,10 +112,9 @@ spec: - name: varlibdockercontainers hostPath: path: /var/lib/docker/containers - - name: inputs - configMap: - defaultMode: 0600 - name: filebeat-inputs + - name: varlog + hostPath: + path: /var/log # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart - name: data hostPath: diff --git a/deploy/kubernetes/filebeat/filebeat-configmap.yaml b/deploy/kubernetes/filebeat/filebeat-configmap.yaml index 0b7f3dd5024..e37a759e60e 100644 --- a/deploy/kubernetes/filebeat/filebeat-configmap.yaml +++ b/deploy/kubernetes/filebeat/filebeat-configmap.yaml @@ -8,22 +8,28 @@ metadata: k8s-app: filebeat data: filebeat.yml: |- - filebeat.config: - inputs: - # Mounted `filebeat-inputs` configmap: - path: ${path.config}/inputs.d/*.yml - # Reload inputs configs as they change: - reload.enabled: false - modules: - path: ${path.config}/modules.d/*.yml - # Reload module configs as they change: - reload.enabled: false + filebeat.inputs: + - type: container + paths: + - /var/log/containers/*.log + processors: + - add_kubernetes_metadata: + in_cluster: true + host: ${NODE_NAME} + matchers: + - logs_path: + logs_path: "/var/log/containers/" - # To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this: + # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this: #filebeat.autodiscover: # providers: # - type: kubernetes + # host: ${NODE_NAME} # hints.enabled: true + # hints.default_config: + # type: container + # paths: + # - /var/log/containers/*${data.kubernetes.container.id}.log processors: - add_cloud_metadata: @@ -35,19 +41,3 @@ data: hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] username: ${ELASTICSEARCH_USERNAME} password: ${ELASTICSEARCH_PASSWORD} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: filebeat-inputs - namespace: kube-system - labels: - k8s-app: filebeat -data: - kubernetes.yml: |- - - type: docker - containers.ids: - - "*" - processors: - - add_kubernetes_metadata: - in_cluster: true diff --git a/deploy/kubernetes/filebeat/filebeat-daemonset.yaml b/deploy/kubernetes/filebeat/filebeat-daemonset.yaml index f554dee185b..98fd31b73d4 100644 --- a/deploy/kubernetes/filebeat/filebeat-daemonset.yaml +++ b/deploy/kubernetes/filebeat/filebeat-daemonset.yaml @@ -33,6 +33,10 @@ spec: value: - name: ELASTIC_CLOUD_AUTH value: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName securityContext: runAsUser: 0 # If using Red Hat OpenShift uncomment this: @@ -48,14 +52,14 @@ spec: mountPath: /etc/filebeat.yml readOnly: true subPath: filebeat.yml - - name: inputs - mountPath: /usr/share/filebeat/inputs.d - readOnly: true - name: data mountPath: /usr/share/filebeat/data - name: varlibdockercontainers mountPath: /var/lib/docker/containers readOnly: true + - name: varlog + mountPath: /var/log + readOnly: true volumes: - name: config configMap: @@ -64,10 +68,9 @@ spec: - name: varlibdockercontainers hostPath: path: /var/lib/docker/containers - - name: inputs - configMap: - defaultMode: 0600 - name: filebeat-inputs + - name: varlog + hostPath: + path: /var/log # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart - name: data hostPath: