Cherry-pick #12632 to 7.2: Use CRI paths in kubernetes manifests (#12742

)

* Use CRI paths in kubernetes manifests (#12632)

We added a new `container` input in #12162, this change makes use of it
to read logs from their CRI paths. Making Filebeat work with deployments that
are not using Docker runtime.

(cherry picked from commit 4933680)

* Update CHANGELOG.next.asciidoc

CHANGELOG.next.asciidoc

@@ -83,6 +83,8 @@ https://github.com/elastic/beats/compare/v7.2.0...7.2[Check the HEAD diff]
 
 *Filebeat*
 
+- Update Kubernetes deployment manifest to use `container` input. {pull}12632[12632]
+
 *Heartbeat*
 
 *Journalbeat*

deploy/kubernetes/filebeat-kubernetes.yaml

@@ -8,22 +8,28 @@ metadata:
  k8s-app: filebeat
 data:
  filebeat.yml: |-
- filebeat.config:
- inputs:
- # Mounted `filebeat-inputs` configmap:
- path: ${path.config}/inputs.d/*.yml
- # Reload inputs configs as they change:
- reload.enabled: false
- modules:
- path: ${path.config}/modules.d/*.yml
- # Reload module configs as they change:
- reload.enabled: false
+ filebeat.inputs:
+ - type: container
+ paths:
+ - /var/log/containers/*.log
+ processors:
+ - add_kubernetes_metadata:
+ in_cluster: true
+ host: ${NODE_NAME}
+ matchers:
+ - logs_path:
+ logs_path: "/var/log/containers/"
 
- # To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this:
+ # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this:
  #filebeat.autodiscover:
  # providers:
  # - type: kubernetes
+ # host: ${NODE_NAME}
  # hints.enabled: true
+ # hints.default_config:
+ # type: container
+ # paths:
+ # - /var/log/containers/*${data.kubernetes.container.id}.log
 
  processors:
  - add_cloud_metadata:
@@ -36,22 +42,6 @@ data:
  username: ${ELASTICSEARCH_USERNAME}
  password: ${ELASTICSEARCH_PASSWORD}
 ---
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: filebeat-inputs
- namespace: kube-system
- labels:
- k8s-app: filebeat
-data:
- kubernetes.yml: |-
- - type: docker
- containers.ids:
- - "*"
- processors:
- - add_kubernetes_metadata:
- in_cluster: true
----
 apiVersion: extensions/v1beta1
 kind: DaemonSet
 metadata:
@@ -87,6 +77,10 @@ spec:
  value:
  - name: ELASTIC_CLOUD_AUTH
  value:
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
  securityContext:
  runAsUser: 0
  # If using Red Hat OpenShift uncomment this:
@@ -102,14 +96,14 @@ spec:
  mountPath: /etc/filebeat.yml
  readOnly: true
  subPath: filebeat.yml
- - name: inputs
- mountPath: /usr/share/filebeat/inputs.d
- readOnly: true
  - name: data
  mountPath: /usr/share/filebeat/data
  - name: varlibdockercontainers
  mountPath: /var/lib/docker/containers
  readOnly: true
+ - name: varlog
+ mountPath: /var/log
+ readOnly: true
  volumes:
  - name: config
  configMap:
@@ -118,10 +112,9 @@ spec:
  - name: varlibdockercontainers
  hostPath:
  path: /var/lib/docker/containers
- - name: inputs
- configMap:
- defaultMode: 0600
- name: filebeat-inputs
+ - name: varlog
+ hostPath:
+ path: /var/log
  # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
  - name: data
  hostPath:

deploy/kubernetes/filebeat/filebeat-configmap.yaml

@@ -8,22 +8,28 @@ metadata:
  k8s-app: filebeat
 data:
  filebeat.yml: |-
- filebeat.config:
- inputs:
- # Mounted `filebeat-inputs` configmap:
- path: ${path.config}/inputs.d/*.yml
- # Reload inputs configs as they change:
- reload.enabled: false
- modules:
- path: ${path.config}/modules.d/*.yml
- # Reload module configs as they change:
- reload.enabled: false
+ filebeat.inputs:
+ - type: container
+ paths:
+ - /var/log/containers/*.log
+ processors:
+ - add_kubernetes_metadata:
+ in_cluster: true
+ host: ${NODE_NAME}
+ matchers:
+ - logs_path:
+ logs_path: "/var/log/containers/"
 
- # To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this:
+ # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this:
  #filebeat.autodiscover:
  # providers:
  # - type: kubernetes
+ # host: ${NODE_NAME}
  # hints.enabled: true
+ # hints.default_config:
+ # type: container
+ # paths:
+ # - /var/log/containers/*${data.kubernetes.container.id}.log
 
  processors:
  - add_cloud_metadata:
@@ -35,19 +41,3 @@ data:
  hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
  username: ${ELASTICSEARCH_USERNAME}
  password: ${ELASTICSEARCH_PASSWORD}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: filebeat-inputs
- namespace: kube-system
- labels:
- k8s-app: filebeat
-data:
- kubernetes.yml: |-
- - type: docker
- containers.ids:
- - "*"
- processors:
- - add_kubernetes_metadata:
- in_cluster: true

deploy/kubernetes/filebeat/filebeat-daemonset.yaml

@@ -33,6 +33,10 @@ spec:
  value:
  - name: ELASTIC_CLOUD_AUTH
  value:
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
  securityContext:
  runAsUser: 0
  # If using Red Hat OpenShift uncomment this:
@@ -48,14 +52,14 @@ spec:
  mountPath: /etc/filebeat.yml
  readOnly: true
  subPath: filebeat.yml
- - name: inputs
- mountPath: /usr/share/filebeat/inputs.d
- readOnly: true
  - name: data
  mountPath: /usr/share/filebeat/data
  - name: varlibdockercontainers
  mountPath: /var/lib/docker/containers
  readOnly: true
+ - name: varlog
+ mountPath: /var/log
+ readOnly: true
  volumes:
  - name: config
  configMap:
@@ -64,10 +68,9 @@ spec:
  - name: varlibdockercontainers
  hostPath:
  path: /var/lib/docker/containers
- - name: inputs
- configMap:
- defaultMode: 0600
- name: filebeat-inputs
+ - name: varlog
+ hostPath:
+ path: /var/log
  # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
  - name: data
  hostPath: