Use CRI paths in kubernetes manifests (elastic#12632)

We added a new `container` input in elastic#12162, this change makes use of it
to read logs from their CRI paths. Making Filebeat work with deployments that
are not using Docker runtime.

(cherry picked from commit 4933680)

CHANGELOG.next.asciidoc

@@ -54,6 +54,33 @@ https://github.com/elastic/beats/compare/v7.2.0...7.2[Check the HEAD diff]
 
 *Filebeat*
 
+- Add more info to message logged when a duplicated symlink file is found {pull}10845[10845]
+- Add option to configure docker input with paths {pull}10687[10687]
+- Add Netflow module to enrich flow events with geoip data. {pull}10877[10877]
+- Set `event.category: network_traffic` for Suricata. {pull}10882[10882]
+- Allow custom default settings with autodiscover (for example, use of CRI paths for logs). {pull}12193[12193]
+- Allow to disable hints based autodiscover default behavior (fetching all logs). {pull}12193[12193]
+- Change Suricata module pipeline to handle `destination.domain` being set if a reverse DNS processor is used. {issue}10510[10510]
+- Add the `network.community_id` flow identifier to field to the IPTables, Suricata, and Zeek modules. {pull}11005[11005]
+- New Filebeat coredns module to ingest coredns logs. It supports both native coredns deployment and coredns deployment in kubernetes. {pull}11200[11200]
+- New module for Cisco ASA logs. {issue}9200[9200] {pull}11171[11171]
+- Added support for Cisco ASA fields to the netflow input. {pull}11201[11201]
+- Configurable line terminator. {pull}11015[11015]
+- Add Filebeat envoyproxy module. {pull}11700[11700]
+- Add apache2(httpd) log path (`/var/log/httpd`) to make apache2 module work out of the box on Redhat-family OSes. {issue}11887[11887] {pull}11888[11888]
+- Add support to new MongoDB additional diagnostic information {pull}11952[11952]
+- New module `panw` for Palo Alto Networks PAN-OS logs. {pull}11999[11999]
+- Add RabbitMQ module. {pull}12032[12032]
+- Add new `container` input. {pull}12162[12162]
+- Add timeouts on communication with docker daemon. {pull}12310[12310]
+- `container` and `docker` inputs now support reading of labels and env vars written by docker JSON file logging driver. {issue}8358[8358]
+- Add specific date processor to convert timezones so same pipeline can be used when convert_timezone is enabled or disabled. {pull}12253[12253]
+- Add MSSQL module {pull}12079[12079]
+- Add ISO8601 date parsing support for system module. {pull}12568[12568] {pull}12578[12579]
+- Update Kubernetes deployment manifest to use `container` input. {pull}12632[12632]
+- Use correct OS path separator in `add_kubernetes_metadata` to support Windows nodes. {pull}9205[9205]
+- Add support for client addresses with port in Apache error logs {pull}12695[12695]
+
 *Heartbeat*
 
 *Journalbeat*

deploy/kubernetes/filebeat-kubernetes.yaml

@@ -8,22 +8,28 @@ metadata:
  k8s-app: filebeat
 data:
  filebeat.yml: |-
- filebeat.config:
- inputs:
- # Mounted `filebeat-inputs` configmap:
- path: ${path.config}/inputs.d/*.yml
- # Reload inputs configs as they change:
- reload.enabled: false
- modules:
- path: ${path.config}/modules.d/*.yml
- # Reload module configs as they change:
- reload.enabled: false
+ filebeat.inputs:
+ - type: container
+ paths:
+ - /var/log/containers/*.log
+ processors:
+ - add_kubernetes_metadata:
+ in_cluster: true
+ host: ${NODE_NAME}
+ matchers:
+ - logs_path:
+ logs_path: "/var/log/containers/"
 
- # To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this:
+ # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this:
  #filebeat.autodiscover:
  # providers:
  # - type: kubernetes
+ # host: ${NODE_NAME}
  # hints.enabled: true
+ # hints.default_config:
+ # type: container
+ # paths:
+ # - /var/log/containers/*${data.kubernetes.container.id}.log
 
  processors:
  - add_cloud_metadata:
@@ -36,22 +42,6 @@ data:
  username: ${ELASTICSEARCH_USERNAME}
  password: ${ELASTICSEARCH_PASSWORD}
 ---
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: filebeat-inputs
- namespace: kube-system
- labels:
- k8s-app: filebeat
-data:
- kubernetes.yml: |-
- - type: docker
- containers.ids:
- - "*"
- processors:
- - add_kubernetes_metadata:
- in_cluster: true
----
 apiVersion: extensions/v1beta1
 kind: DaemonSet
 metadata:
@@ -87,6 +77,10 @@ spec:
  value:
  - name: ELASTIC_CLOUD_AUTH
  value:
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
  securityContext:
  runAsUser: 0
  # If using Red Hat OpenShift uncomment this:
@@ -102,14 +96,14 @@ spec:
  mountPath: /etc/filebeat.yml
  readOnly: true
  subPath: filebeat.yml
- - name: inputs
- mountPath: /usr/share/filebeat/inputs.d
- readOnly: true
  - name: data
  mountPath: /usr/share/filebeat/data
  - name: varlibdockercontainers
  mountPath: /var/lib/docker/containers
  readOnly: true
+ - name: varlog
+ mountPath: /var/log
+ readOnly: true
  volumes:
  - name: config
  configMap:
@@ -118,10 +112,9 @@ spec:
  - name: varlibdockercontainers
  hostPath:
  path: /var/lib/docker/containers
- - name: inputs
- configMap:
- defaultMode: 0600
- name: filebeat-inputs
+ - name: varlog
+ hostPath:
+ path: /var/log
  # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
  - name: data
  hostPath:

deploy/kubernetes/filebeat/filebeat-configmap.yaml

@@ -8,22 +8,28 @@ metadata:
  k8s-app: filebeat
 data:
  filebeat.yml: |-
- filebeat.config:
- inputs:
- # Mounted `filebeat-inputs` configmap:
- path: ${path.config}/inputs.d/*.yml
- # Reload inputs configs as they change:
- reload.enabled: false
- modules:
- path: ${path.config}/modules.d/*.yml
- # Reload module configs as they change:
- reload.enabled: false
+ filebeat.inputs:
+ - type: container
+ paths:
+ - /var/log/containers/*.log
+ processors:
+ - add_kubernetes_metadata:
+ in_cluster: true
+ host: ${NODE_NAME}
+ matchers:
+ - logs_path:
+ logs_path: "/var/log/containers/"
 
- # To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this:
+ # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this:
  #filebeat.autodiscover:
  # providers:
  # - type: kubernetes
+ # host: ${NODE_NAME}
  # hints.enabled: true
+ # hints.default_config:
+ # type: container
+ # paths:
+ # - /var/log/containers/*${data.kubernetes.container.id}.log
 
  processors:
  - add_cloud_metadata:
@@ -35,19 +41,3 @@ data:
  hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
  username: ${ELASTICSEARCH_USERNAME}
  password: ${ELASTICSEARCH_PASSWORD}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: filebeat-inputs
- namespace: kube-system
- labels:
- k8s-app: filebeat
-data:
- kubernetes.yml: |-
- - type: docker
- containers.ids:
- - "*"
- processors:
- - add_kubernetes_metadata:
- in_cluster: true

deploy/kubernetes/filebeat/filebeat-daemonset.yaml

@@ -33,6 +33,10 @@ spec:
  value:
  - name: ELASTIC_CLOUD_AUTH
  value:
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
  securityContext:
  runAsUser: 0
  # If using Red Hat OpenShift uncomment this:
@@ -48,14 +52,14 @@ spec:
  mountPath: /etc/filebeat.yml
  readOnly: true
  subPath: filebeat.yml
- - name: inputs
- mountPath: /usr/share/filebeat/inputs.d
- readOnly: true
  - name: data
  mountPath: /usr/share/filebeat/data
  - name: varlibdockercontainers
  mountPath: /var/lib/docker/containers
  readOnly: true
+ - name: varlog
+ mountPath: /var/log
+ readOnly: true
  volumes:
  - name: config
  configMap:
@@ -64,10 +68,9 @@ spec:
  - name: varlibdockercontainers
  hostPath:
  path: /var/lib/docker/containers
- - name: inputs
- configMap:
- defaultMode: 0600
- name: filebeat-inputs
+ - name: varlog
+ hostPath:
+ path: /var/log
  # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
  - name: data
  hostPath: