From 076be691322228ebe07c351e7fbfe202f95bc361 Mon Sep 17 00:00:00 2001 From: David Kowalski Date: Wed, 8 Jul 2020 10:01:12 +0200 Subject: [PATCH 1/9] Add Beat config library --- config/dev/elastic-psp.yaml | 2 +- config/recipes/beats/0_ns.yaml | 4 - config/recipes/beats/1_monitor.yaml | 38 -- .../recipes/beats/2_filebeat-kubernetes.yaml | 158 ------- .../beats/3_metricbeat-kubernetes.yaml | 385 ------------------ config/recipes/beats/auditbeat_hosts.yaml | 133 ++++++ .../recipes/beats/filebeat_autodiscover.yaml | 113 +++++ .../filebeat_autodiscover_by_metadata.yaml | 115 ++++++ .../beats/filebeat_no_autodiscover.yaml | 71 ++++ .../recipes/beats/heartbeat_es_kb_health.yaml | 49 +++ config/recipes/beats/journalbeat_hosts.yaml | 84 ++++ config/recipes/beats/metricbeat_hosts.yaml | 188 +++++++++ config/recipes/beats/packetbeat_dns_http.yaml | 69 ++++ test/e2e/beat/config_test.go | 6 +- test/e2e/beat/recipes_test.go | 197 +++++++++ test/e2e/beat/setup_test.go | 3 + test/e2e/samples_test.go | 15 +- test/e2e/test/beat/builder.go | 24 +- test/e2e/test/beat/checks.go | 30 +- test/e2e/test/helper/yaml.go | 239 ++++++++++- test/e2e/test/run.go | 37 ++ 21 files changed, 1338 insertions(+), 622 deletions(-) delete mode 100644 config/recipes/beats/0_ns.yaml delete mode 100644 config/recipes/beats/1_monitor.yaml delete mode 100644 config/recipes/beats/2_filebeat-kubernetes.yaml delete mode 100644 config/recipes/beats/3_metricbeat-kubernetes.yaml create mode 100644 config/recipes/beats/auditbeat_hosts.yaml create mode 100644 config/recipes/beats/filebeat_autodiscover.yaml create mode 100644 config/recipes/beats/filebeat_autodiscover_by_metadata.yaml create mode 100644 config/recipes/beats/filebeat_no_autodiscover.yaml create mode 100644 config/recipes/beats/heartbeat_es_kb_health.yaml create mode 100644 config/recipes/beats/journalbeat_hosts.yaml create mode 100644 config/recipes/beats/metricbeat_hosts.yaml create mode 100644 config/recipes/beats/packetbeat_dns_http.yaml create mode 100644 test/e2e/beat/recipes_test.go diff --git a/config/dev/elastic-psp.yaml b/config/dev/elastic-psp.yaml index 85c5e4cfd1..6a923b8de5 100644 --- a/config/dev/elastic-psp.yaml +++ b/config/dev/elastic-psp.yaml @@ -227,7 +227,7 @@ - SETPCAP - AUDIT_WRITE - NET_BIND_SERVICE - hostNetwork: false + hostNetwork: true hostIPC: false hostPID: false runAsUser: diff --git a/config/recipes/beats/0_ns.yaml b/config/recipes/beats/0_ns.yaml deleted file mode 100644 index 350ad314cf..0000000000 --- a/config/recipes/beats/0_ns.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: beats \ No newline at end of file diff --git a/config/recipes/beats/1_monitor.yaml b/config/recipes/beats/1_monitor.yaml deleted file mode 100644 index 5daaec54f0..0000000000 --- a/config/recipes/beats/1_monitor.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# This sample sets up an Elasticsearch cluster and a Kibana instance preconfigured for that cluster. -# The hints based autodiscover annotations for Filebeat are managed by ECK. -apiVersion: elasticsearch.k8s.elastic.co/v1 -kind: Elasticsearch -metadata: - name: monitor - namespace: beats -spec: - version: 7.8.0 - nodeSets: - - name: mdi - count: 3 - config: - node.store.allow_mmap: false - volumeClaimTemplates: - - metadata: - name: elasticsearch-data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Gi ---- -apiVersion: kibana.k8s.elastic.co/v1 -kind: Kibana -metadata: - name: monitor - namespace: beats -spec: - version: 7.8.0 - count: 1 - elasticsearchRef: - name: "monitor" - http: - service: - spec: - type: LoadBalancer \ No newline at end of file diff --git a/config/recipes/beats/2_filebeat-kubernetes.yaml b/config/recipes/beats/2_filebeat-kubernetes.yaml deleted file mode 100644 index e0ac2198f2..0000000000 --- a/config/recipes/beats/2_filebeat-kubernetes.yaml +++ /dev/null @@ -1,158 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: filebeat-config - namespace: beats - labels: - k8s-app: filebeat -data: - filebeat.yml: |- - filebeat.autodiscover: - providers: - - type: kubernetes - host: ${NODE_NAME} - hints.enabled: true - hints.default_config: - type: container - paths: - - /var/log/containers/*${data.kubernetes.container.id}.log - - processors: - - add_cloud_metadata: - - add_host_metadata: - - output.elasticsearch: - hosts: ['https://${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] - username: ${ELASTICSEARCH_USERNAME} - password: ${ELASTICSEARCH_PASSWORD} - ssl.certificate_authorities: - - /mnt/elastic/tls.crt ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: filebeat - namespace: beats - labels: - k8s-app: filebeat -spec: - selector: - matchLabels: - k8s-app: filebeat - template: - metadata: - labels: - k8s-app: filebeat - spec: - serviceAccountName: filebeat - terminationGracePeriodSeconds: 30 - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: filebeat - image: docker.elastic.co/beats/filebeat:7.8.0 - args: [ - "-c", "/etc/filebeat.yml", - "-e", - ] - env: - - name: ELASTICSEARCH_HOST - value: monitor-es-http - - name: ELASTICSEARCH_PORT - value: "9200" - - name: ELASTICSEARCH_USERNAME - value: elastic - - name: ELASTICSEARCH_PASSWORD - valueFrom: - secretKeyRef: - key: elastic - name: monitor-es-elastic-user - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - runAsUser: 0 - # If using Red Hat OpenShift uncomment this: - #privileged: true - resources: - limits: - memory: 200Mi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - name: config - mountPath: /etc/filebeat.yml - readOnly: true - subPath: filebeat.yml - - name: data - mountPath: /usr/share/filebeat/data - - name: varlibdockercontainers - mountPath: /var/lib/docker/containers - readOnly: true - - name: varlog - mountPath: /var/log - readOnly: true - - name: es-certs - mountPath: /mnt/elastic/tls.crt - readOnly: true - subPath: tls.crt - volumes: - - name: config - configMap: - defaultMode: 0600 - name: filebeat-config - - name: varlibdockercontainers - hostPath: - path: /var/lib/docker/containers - - name: varlog - hostPath: - path: /var/log - # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart - - name: data - hostPath: - path: /var/lib/filebeat-data - type: DirectoryOrCreate - - name: es-certs - secret: - secretName: monitor-es-http-certs-public ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: filebeat -subjects: -- kind: ServiceAccount - name: filebeat - namespace: beats -roleRef: - kind: ClusterRole - name: filebeat - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: filebeat - labels: - k8s-app: filebeat -rules: -- apiGroups: [""] # "" indicates the core API group - resources: - - namespaces - - pods - verbs: - - get - - watch - - list ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: filebeat - namespace: beats - labels: - k8s-app: filebeat ---- diff --git a/config/recipes/beats/3_metricbeat-kubernetes.yaml b/config/recipes/beats/3_metricbeat-kubernetes.yaml deleted file mode 100644 index 52399e912a..0000000000 --- a/config/recipes/beats/3_metricbeat-kubernetes.yaml +++ /dev/null @@ -1,385 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: metricbeat-daemonset-config - namespace: beats - labels: - k8s-app: metricbeat -data: - metricbeat.yml: |- - metricbeat.config.modules: - # Mounted `metricbeat-daemonset-modules` configmap: - path: ${path.config}/modules.d/*.yml - # Reload module configs as they change: - reload.enabled: false - - # To enable hints based autodiscover uncomment this: - metricbeat.autodiscover: - providers: - - type: kubernetes - host: ${NODE_NAME} - hints.enabled: true - - processors: - - add_cloud_metadata: - - output.elasticsearch: - hosts: ['https://${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] - username: ${ELASTICSEARCH_USERNAME} - password: ${ELASTICSEARCH_PASSWORD} - ssl.certificate_authorities: - - /mnt/elastic/tls.crt ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: metricbeat-daemonset-modules - namespace: beats - labels: - k8s-app: metricbeat -data: - system.yml: |- - - module: system - period: 10s - metricsets: - - cpu - - load - - memory - - network - - process - - process_summary - #- core - #- diskio - #- socket - processes: ['.*'] - process.include_top_n: - by_cpu: 5 # include top 5 processes by CPU - by_memory: 5 # include top 5 processes by memory - - - module: system - period: 1m - metricsets: - - filesystem - - fsstat - processors: - - drop_event.when.regexp: - system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' - kubernetes.yml: |- - - module: kubernetes - metricsets: - - node - - system - - pod - - container - - volume - period: 10s - host: ${NODE_NAME} - hosts: ["https://${HOSTNAME}:10250"] - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - ssl.verification_mode: "none" - # If using Red Hat OpenShift remove ssl.verification_mode entry and - # uncomment these settings: - #ssl.certificate_authorities: - #- /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - - module: kubernetes - metricsets: - - proxy - period: 10s - host: ${NODE_NAME} - hosts: ["localhost:10249"] ---- -# Deploy a Metricbeat instance per node for node metrics retrieval -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: metricbeat - namespace: beats - labels: - k8s-app: metricbeat -spec: - selector: - matchLabels: - k8s-app: metricbeat - template: - metadata: - labels: - k8s-app: metricbeat - spec: - serviceAccountName: metricbeat - terminationGracePeriodSeconds: 30 - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: metricbeat - image: docker.elastic.co/beats/metricbeat:7.8.0 - args: [ - "-c", "/etc/metricbeat.yml", - "-e", - "-system.hostfs=/hostfs", - "-d", "autodiscover", - "-d", "kubernetes", - ] - env: - - name: ELASTICSEARCH_HOST - value: monitor-es-http - - name: ELASTICSEARCH_PORT - value: "9200" - - name: ELASTICSEARCH_USERNAME - value: elastic - - name: ELASTICSEARCH_PASSWORD - valueFrom: - secretKeyRef: - key: elastic - name: monitor-es-elastic-user - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - runAsUser: 0 - resources: - limits: - memory: 200Mi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - name: config - mountPath: /etc/metricbeat.yml - readOnly: true - subPath: metricbeat.yml - - name: modules - mountPath: /usr/share/metricbeat/modules.d - readOnly: true - - name: dockersock - mountPath: /var/run/docker.sock - - name: proc - mountPath: /hostfs/proc - readOnly: true - - name: cgroup - mountPath: /hostfs/sys/fs/cgroup - readOnly: true - - name: es-certs - mountPath: /mnt/elastic/tls.crt - readOnly: true - subPath: tls.crt - volumes: - - name: proc - hostPath: - path: /proc - - name: cgroup - hostPath: - path: /sys/fs/cgroup - - name: dockersock - hostPath: - path: /var/run/docker.sock - - name: config - configMap: - defaultMode: 0600 - name: metricbeat-daemonset-config - - name: modules - configMap: - defaultMode: 0600 - name: metricbeat-daemonset-modules - - name: es-certs - secret: - secretName: monitor-es-http-certs-public ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: metricbeat-deployment-config - namespace: beats - labels: - k8s-app: metricbeat -data: - metricbeat.yml: |- - metricbeat.config.modules: - # Mounted `metricbeat-daemonset-modules` configmap: - path: ${path.config}/modules.d/*.yml - # Reload module configs as they change: - reload.enabled: false - - processors: - - add_cloud_metadata: - - setup.dashboards.enabled: true - - setup.kibana: - host: "https://${KIBANA_HOST:kibana}:${KIBANA_PORT:5601}" - ssl.enabled: true - ssl.certificate_authorities: - - /mnt/kibana/ca.crt - - output.elasticsearch: - hosts: ['https://${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] - username: ${ELASTICSEARCH_USERNAME} - password: ${ELASTICSEARCH_PASSWORD} - ssl.certificate_authorities: - - /mnt/elastic/tls.crt - ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: metricbeat-deployment-modules - namespace: beats - labels: - k8s-app: metricbeat -data: - # This module requires `kube-state-metrics` up and running under `kube-system` namespace - kubernetes.yml: |- - - module: kubernetes - metricsets: - - state_node - - state_deployment - - state_replicaset - - state_pod - - state_container - # Uncomment this to get k8s events: - #- event - period: 10s - host: ${NODE_NAME} - hosts: ["kube-state-metrics.kube-system:8080"] ---- -# Deploy singleton instance in the whole cluster for some unique data sources, like kube-state-metrics -apiVersion: apps/v1 -kind: Deployment -metadata: - name: metricbeat - namespace: beats - labels: - k8s-app: metricbeat -spec: - selector: - matchLabels: - k8s-app: metricbeat - template: - metadata: - labels: - k8s-app: metricbeat - spec: - serviceAccountName: metricbeat - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: metricbeat - image: docker.elastic.co/beats/metricbeat:7.8.0 - args: [ - "-c", "/etc/metricbeat.yml", - "-e", - "-d", "autodiscover", - ] - env: - - name: ELASTICSEARCH_HOST - value: monitor-es-http - - name: ELASTICSEARCH_PORT - value: "9200" - - name: ELASTICSEARCH_USERNAME - value: elastic - - name: ELASTICSEARCH_PASSWORD - valueFrom: - secretKeyRef: - key: elastic - name: monitor-es-elastic-user - - name: KIBANA_HOST - value: monitor-kb-http - - name: KIBANA_PORT - value: "5601" - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - runAsUser: 0 - resources: - limits: - memory: 200Mi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - name: config - mountPath: /etc/metricbeat.yml - readOnly: true - subPath: metricbeat.yml - - name: modules - mountPath: /usr/share/metricbeat/modules.d - readOnly: true - - name: es-certs - mountPath: /mnt/elastic/tls.crt - readOnly: true - subPath: tls.crt - - name: kb-certs - mountPath: /mnt/kibana/ca.crt - readOnly: true - subPath: ca.crt - volumes: - - name: config - configMap: - defaultMode: 0600 - name: metricbeat-deployment-config - - name: modules - configMap: - defaultMode: 0600 - name: metricbeat-deployment-modules - - name: es-certs - secret: - secretName: monitor-es-http-certs-public - - name: kb-certs - secret: - secretName: monitor-kb-http-certs-public ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: metricbeat -subjects: -- kind: ServiceAccount - name: metricbeat - namespace: beats -roleRef: - kind: ClusterRole - name: metricbeat - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: metricbeat - labels: - k8s-app: metricbeat -rules: -- apiGroups: [""] - resources: - - nodes - - namespaces - - events - - pods - verbs: ["get", "list", "watch"] -- apiGroups: ["extensions"] - resources: - - replicasets - verbs: ["get", "list", "watch"] -- apiGroups: ["apps"] - resources: - - statefulsets - - deployments - - replicasets - verbs: ["get", "list", "watch"] -- apiGroups: - - "" - resources: - - nodes/stats - verbs: - - get ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: metricbeat - namespace: beats - labels: - k8s-app: metricbeat ---- diff --git a/config/recipes/beats/auditbeat_hosts.yaml b/config/recipes/beats/auditbeat_hosts.yaml new file mode 100644 index 0000000000..05ef2e7e54 --- /dev/null +++ b/config/recipes/beats/auditbeat_hosts.yaml @@ -0,0 +1,133 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: auditbeat +spec: + type: auditbeat + version: 7.8.0 + elasticsearchRef: + name: elasticsearch + kibanaRef: + name: kibana + config: + auditbeat.modules: + - module: file_integrity + paths: + - /hostfs/bin + - /hostfs/usr/bin + - /hostfs/sbin + - /hostfs/usr/sbin + - /hostfs/etc + exclude_files: + - '(?i)\.sw[nop]$' + - '~$' + - '/\.git($|/)' + scan_at_start: true + scan_rate_per_sec: 50 MiB + max_file_size: 100 MiB + hash_types: [sha1] + recursive: true + - module: auditd + audit_rules: | + # Executions + -a always,exit -F arch=b64 -S execve,execveat -k exec + + # Unauthorized access attempts + -a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EACCES -k access + -a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access + + processors: + - add_cloud_metadata: {} + - add_process_metadata: + match_pids: ['process.pid'] + - add_kubernetes_metadata: + host: ${HOSTNAME} + default_indexers.enabled: false + default_matchers.enabled: false + indexers: + - container: + matchers: + - fields.lookup_fields: ['container.id'] + daemonSet: + podTemplate: + spec: + hostPID: true # Required by auditd module + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + automountServiceAccountToken: true # some older Beat versions are depending on this settings presence in k8s context + securityContext: + runAsUser: 0 + volumes: + - name: bin + hostPath: + path: /bin + - name: usrbin + hostPath: + path: /usr/bin + - name: sbin + hostPath: + path: /sbin + - name: usrsbin + hostPath: + path: /usr/sbin + - name: etc + hostPath: + path: /etc + - name: run-containerd + hostPath: + path: /run/containerd + type: DirectoryOrCreate + containers: + - name: auditbeat + securityContext: + capabilities: + add: + # Capabilities needed for auditd module + - 'AUDIT_READ' + - 'AUDIT_WRITE' + - 'AUDIT_CONTROL' + volumeMounts: + - name: bin + mountPath: /hostfs/bin + readOnly: true + - name: sbin + mountPath: /hostfs/sbin + readOnly: true + - name: usrbin + mountPath: /hostfs/usr/bin + readOnly: true + - name: usrsbin + mountPath: /hostfs/usr/sbin + readOnly: true + - name: etc + mountPath: /hostfs/etc + readOnly: true + # Directory with root filesystems of containers executed with containerd, this can be + # different with other runtimes. This volume is needed to monitor the file integrity + # of files in containers. + - name: run-containerd + mountPath: /run/containerd + readOnly: true +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: elasticsearch +spec: + version: 7.8.0 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana +spec: + version: 7.8.0 + count: 1 + elasticsearchRef: + name: elasticsearch +--- diff --git a/config/recipes/beats/filebeat_autodiscover.yaml b/config/recipes/beats/filebeat_autodiscover.yaml new file mode 100644 index 0000000000..06323c0b73 --- /dev/null +++ b/config/recipes/beats/filebeat_autodiscover.yaml @@ -0,0 +1,113 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: filebeat +spec: + type: filebeat + version: 7.8.0 + elasticsearchRef: + name: elasticsearch + kibanaRef: + name: kibana + config: + filebeat: + autodiscover: + providers: + - type: kubernetes + host: ${HOSTNAME} + hints: + enabled: true + default_config: + type: container + paths: + - /var/log/containers/*${data.kubernetes.container.id}.log + processors: + - add_cloud_metadata: {} + - add_host_metadata: {} + daemonSet: + podTemplate: + spec: + serviceAccountName: filebeat + automountServiceAccountToken: true + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + securityContext: + runAsUser: 0 + # If using Red Hat OpenShift uncomment this: + #privileged: true + containers: + - name: filebeat + volumeMounts: + - name: varlogcontainers + mountPath: /var/log/containers + - name: varlogpods + mountPath: /var/log/pods + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + volumes: + - name: varlogcontainers + hostPath: + path: /var/log/containers + - name: varlogpods + hostPath: + path: /var/log/pods + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + verbs: + - get + - watch + - list +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: filebeat +subjects: +- kind: ServiceAccount + name: filebeat + namespace: default +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: elasticsearch +spec: + version: 7.8.0 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana +spec: + version: 7.8.0 + count: 1 + elasticsearchRef: + name: elasticsearch +--- diff --git a/config/recipes/beats/filebeat_autodiscover_by_metadata.yaml b/config/recipes/beats/filebeat_autodiscover_by_metadata.yaml new file mode 100644 index 0000000000..a82f590044 --- /dev/null +++ b/config/recipes/beats/filebeat_autodiscover_by_metadata.yaml @@ -0,0 +1,115 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: filebeat +spec: + type: filebeat + version: 7.8.0 + elasticsearchRef: + name: elasticsearch + kibanaRef: + name: kibana + config: + filebeat.autodiscover.providers: + - node: ${HOSTNAME} + type: kubernetes + hints.default_config.enabled: "false" + templates: + - condition.equals.kubernetes.namespace: log-namespace + config: + - paths: ["/var/log/containers/*${data.kubernetes.container.id}.log"] + type: container + - condition.equals.kubernetes.labels.log-label: "true" + config: + - paths: ["/var/log/containers/*${data.kubernetes.container.id}.log"] + type: container + processors: + - add_cloud_metadata: {} + - add_host_metadata: {} + daemonSet: + podTemplate: + spec: + serviceAccountName: filebeat + automountServiceAccountToken: true + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + securityContext: + runAsUser: 0 + # If using Red Hat OpenShift uncomment this: + #privileged: true + containers: + - name: filebeat + volumeMounts: + - name: varlogcontainers + mountPath: /var/log/containers + - name: varlogpods + mountPath: /var/log/pods + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + volumes: + - name: varlogcontainers + hostPath: + path: /var/log/containers + - name: varlogpods + hostPath: + path: /var/log/pods + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + verbs: + - get + - watch + - list +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: filebeat +subjects: +- kind: ServiceAccount + name: filebeat + namespace: default +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: elasticsearch +spec: + version: 7.8.0 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana +spec: + version: 7.8.0 + count: 1 + elasticsearchRef: + name: elasticsearch +--- diff --git a/config/recipes/beats/filebeat_no_autodiscover.yaml b/config/recipes/beats/filebeat_no_autodiscover.yaml new file mode 100644 index 0000000000..9603fac8dc --- /dev/null +++ b/config/recipes/beats/filebeat_no_autodiscover.yaml @@ -0,0 +1,71 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: filebeat +spec: + type: filebeat + version: 7.8.0 + elasticsearchRef: + name: elasticsearch + kibanaRef: + name: kibana + config: + filebeat.inputs: + - type: container + paths: + - /var/log/containers/*.log + processors: + - add_host_metadata: {} + - add_cloud_metadata: {} + daemonSet: + podTemplate: + spec: + automountServiceAccountToken: true + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + securityContext: + runAsUser: 0 + # If using Red Hat OpenShift uncomment this: + #privileged: true + containers: + - name: filebeat + volumeMounts: + - name: varlogcontainers + mountPath: /var/log/containers + - name: varlogpods + mountPath: /var/log/pods + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + volumes: + - name: varlogcontainers + hostPath: + path: /var/log/containers + - name: varlogpods + hostPath: + path: /var/log/pods + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: elasticsearch +spec: + version: 7.8.0 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana +spec: + version: 7.8.0 + count: 1 + elasticsearchRef: + name: elasticsearch diff --git a/config/recipes/beats/heartbeat_es_kb_health.yaml b/config/recipes/beats/heartbeat_es_kb_health.yaml new file mode 100644 index 0000000000..91bdb50c26 --- /dev/null +++ b/config/recipes/beats/heartbeat_es_kb_health.yaml @@ -0,0 +1,49 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: heartbeat +spec: + type: heartbeat + version: 7.8.0 + elasticsearchRef: + name: elasticsearch + kibanaRef: + name: kibana + config: + heartbeat.monitors: + - type: tcp + schedule: '@every 5s' + hosts: ["elasticsearch-es-http.default.svc:9200"] + - type: tcp + schedule: '@every 5s' + hosts: ["kibana-kb-http.default.svc:5601"] + deployment: + podTemplate: + spec: + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + securityContext: + runAsUser: 0 +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: elasticsearch +spec: + version: 7.8.0 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana +spec: + version: 7.8.0 + count: 1 + elasticsearchRef: + name: elasticsearch +--- diff --git a/config/recipes/beats/journalbeat_hosts.yaml b/config/recipes/beats/journalbeat_hosts.yaml new file mode 100644 index 0000000000..bbd716dcf9 --- /dev/null +++ b/config/recipes/beats/journalbeat_hosts.yaml @@ -0,0 +1,84 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: journalbeat +spec: + type: journalbeat + version: 7.8.0 + elasticsearchRef: + name: elasticsearch + kibanaRef: + name: kibana + config: + journalbeat.inputs: + - paths: [] + seek: cursor + cursor_seek_fallback: tail + processors: + - add_kubernetes_metadata: + host: "${HOSTNAME}" + in_cluster: true + default_indexers.enabled: false + default_matchers.enabled: false + indexers: + - container: + matchers: + - fields: + lookup_fields: ["container.id"] + - decode_json_fields: + fields: ["message"] + process_array: false + max_depth: 1 + target: "" + overwrite_keys: true + daemonSet: + podTemplate: + spec: + automountServiceAccountToken: true # some older Beat versions are depending on this settings presence in k8s context + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: journalbeat + volumeMounts: + - mountPath: /var/log/journal + name: var-journal + - mountPath: /run/log/journal + name: run-journal + - mountPath: /etc/machine-id + name: machine-id + hostNetwork: true + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 30 + volumes: + - hostPath: + path: /var/log/journal + name: var-journal + - hostPath: + path: /run/log/journal + name: run-journal + - hostPath: + path: /etc/machine-id + name: machine-id +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: elasticsearch +spec: + version: 7.8.0 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana +spec: + version: 7.8.0 + count: 1 + elasticsearchRef: + name: elasticsearch +--- diff --git a/config/recipes/beats/metricbeat_hosts.yaml b/config/recipes/beats/metricbeat_hosts.yaml new file mode 100644 index 0000000000..4ece5e9ded --- /dev/null +++ b/config/recipes/beats/metricbeat_hosts.yaml @@ -0,0 +1,188 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: metricbeat +spec: + type: metricbeat + version: 7.8.0 + elasticsearchRef: + name: elasticsearch + kibanaRef: + name: kibana + config: + metricbeat: + autodiscover: + providers: + - hints: + default_config: {} + enabled: "true" + host: ${HOSTNAME} + type: kubernetes + modules: + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + process: + include_top_n: + by_cpu: 5 + by_memory: 5 + processes: + - .* + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event: + when: + regexp: + system: + filesystem: + mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib)($|/) + - module: kubernetes + period: 10s + host: ${HOSTNAME} + hosts: + - https://${HOSTNAME}:10250 + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl: + verification_mode: none + metricsets: + - node + - system + - pod + - container + - volume + processors: + - add_cloud_metadata: {} + - add_host_metadata: {} + daemonSet: + podTemplate: + spec: + serviceAccountName: metricbeat + automountServiceAccountToken: true # some older Beat versions are depending on this settings presence in k8s context + containers: + - args: + - -e + - -c + - /etc/beat.yml + - -system.hostfs=/hostfs + name: metricbeat + volumeMounts: + - mountPath: /hostfs/sys/fs/cgroup + name: cgroup + - mountPath: /var/run/docker.sock + name: dockersock + - mountPath: /hostfs/proc + name: proc + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 30 + volumes: + - hostPath: + path: /sys/fs/cgroup + name: cgroup + - hostPath: + path: /var/run/docker.sock + name: dockersock + - hostPath: + path: /proc + name: proc +--- +# permissions needed for metricbeat +# source: https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-module-kubernetes.html +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metricbeat +rules: +- apiGroups: + - "" + resources: + - nodes + - namespaces + - events + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "extensions" + resources: + - replicasets + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - statefulsets + - deployments + - replicasets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metricbeat + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metricbeat +subjects: +- kind: ServiceAccount + name: metricbeat + namespace: default +roleRef: + kind: ClusterRole + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: elasticsearch +spec: + version: 7.8.0 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana +spec: + version: 7.8.0 + count: 1 + elasticsearchRef: + name: elasticsearch +--- diff --git a/config/recipes/beats/packetbeat_dns_http.yaml b/config/recipes/beats/packetbeat_dns_http.yaml new file mode 100644 index 0000000000..2158c9684d --- /dev/null +++ b/config/recipes/beats/packetbeat_dns_http.yaml @@ -0,0 +1,69 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: packetbeat +spec: + type: packetbeat + version: 7.8.0 + elasticsearchRef: + name: elasticsearch + kibanaRef: + name: kibana + config: + packetbeat.interfaces.device: any + packetbeat.protocols: + - type: dns + ports: [53] + include_authorities: true + include_additionals: true + - type: http + ports: [80, 8000, 8080, 9200] + packetbeat.flows: + timeout: 30s + period: 10s + processors: + - add_cloud_metadata: + - add_kubernetes_metadata: + host: ${HOSTNAME} + indexers: + - ip_port: + matchers: + - field_format: + format: '%{[ip]}:%{[port]}' + daemonSet: + podTemplate: + spec: + terminationGracePeriodSeconds: 30 + hostNetwork: true + automountServiceAccountToken: true # some older Beat versions are depending on this settings presence in k8s context + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: packetbeat + securityContext: + runAsUser: 0 + capabilities: + add: + - NET_ADMIN +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: elasticsearch +spec: + version: 7.8.0 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana +spec: + version: 7.8.0 + count: 1 + elasticsearchRef: + name: elasticsearch +--- diff --git a/test/e2e/beat/config_test.go b/test/e2e/beat/config_test.go index c10b76fbfe..263f76cc63 100644 --- a/test/e2e/beat/config_test.go +++ b/test/e2e/beat/config_test.go @@ -37,6 +37,7 @@ func TestFilebeatDefaultConfig(t *testing.T) { testPodBuilder := beat.NewPodBuilder(name) fbBuilder := beat.NewBuilder(name). + WithRoles(beat.PSPClusterRoleName, beat.AutodiscoverClusterRoleName). WithType(filebeat.Type). WithElasticsearchRef(esBuilder.Ref()). WithESValidations( @@ -59,7 +60,7 @@ func TestMetricbeatDefaultConfig(t *testing.T) { mbBuilder := beat.NewBuilder(name). WithType(metricbeat.Type). - WithRoles(beat.MetricbeatClusterRoleName). + WithRoles(beat.MetricbeatClusterRoleName, beat.PSPClusterRoleName, beat.AutodiscoverClusterRoleName). WithElasticsearchRef(esBuilder.Ref()). WithESValidations( beat.HasEventFromBeat(metricbeat.Type), @@ -85,6 +86,7 @@ func TestHeartbeatConfig(t *testing.T) { hbBuilder := beat.NewBuilder(name). WithType(heartbeat.Type). + WithRoles(beat.PSPClusterRoleName). WithDeployment(). WithElasticsearchRef(esBuilder.Ref()). WithESValidations( @@ -120,6 +122,7 @@ func TestBeatSecureSettings(t *testing.T) { fbBuilder := beat.NewBuilder(name). WithType(filebeat.Type). + WithRoles(beat.PSPClusterRoleName, beat.AutodiscoverClusterRoleName). WithElasticsearchRef(esBuilder.Ref()). WithSecureSettings(secretName). WithObjects(secret). @@ -191,6 +194,7 @@ processors: fbBuilder := beat.NewBuilder(name). WithType(filebeat.Type). + WithRoles(beat.PSPClusterRoleName, beat.AutodiscoverClusterRoleName). WithElasticsearchRef(esBuilder.Ref()). WithConfigRef(secretName). WithObjects(secret). diff --git a/test/e2e/beat/recipes_test.go b/test/e2e/beat/recipes_test.go new file mode 100644 index 0000000000..cc40a6a0fe --- /dev/null +++ b/test/e2e/beat/recipes_test.go @@ -0,0 +1,197 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package beat + +import ( + "fmt" + "path" + "strings" + "testing" + + commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" + esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" + beatcommon "github.com/elastic/cloud-on-k8s/pkg/controller/beat/common" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/settings" + "github.com/elastic/cloud-on-k8s/pkg/controller/kibana" + "github.com/elastic/cloud-on-k8s/test/e2e/test" + "github.com/elastic/cloud-on-k8s/test/e2e/test/beat" + "github.com/elastic/cloud-on-k8s/test/e2e/test/helper" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/rand" +) + +func TestFilebeatNoAutodiscoverRecipe(t *testing.T) { + name := "fb-no-autodiscover" + pod, loggedString := loggingTestPod(name) + customize := func(builder beat.Builder) beat.Builder { + return builder. + WithRoles(beat.PSPClusterRoleName). + WithESValidations( + beat.HasMessageContaining(loggedString), + ) + } + + runBeatRecipe(t, "filebeat_no_autodiscover.yaml", customize, pod) +} + +func TestFilebeatAutodiscoverRecipe(t *testing.T) { + name := "fb-autodiscover" + pod, loggedString := loggingTestPod(name) + customize := func(builder beat.Builder) beat.Builder { + return builder. + WithRoles(beat.PSPClusterRoleName). + WithESValidations( + beat.HasEventFromPod(pod.Name), + beat.HasMessageContaining(loggedString), + ) + } + + runBeatRecipe(t, "filebeat_autodiscover.yaml", customize, pod) +} + +func TestFilebeatAutodiscoverByMetadataRecipe(t *testing.T) { + name := "fb-autodiscover-meta" + podBad, badLog := loggingTestPod(name + "-bad") + podLabel, goodLog := loggingTestPod(name + "-label") + podLabel.Labels["log-label"] = "true" + + customize := func(builder beat.Builder) beat.Builder { + return builder. + WithRoles(beat.PSPClusterRoleName, beat.AutodiscoverClusterRoleName). + WithESValidations( + beat.HasEventFromPod(podLabel.Name), + beat.HasMessageContaining(goodLog), + beat.NoMessageContaining(badLog), + ) + } + + runBeatRecipe(t, "filebeat_autodiscover_by_metadata.yaml", customize, podLabel, podBad) +} + +func TestMetricbeatHostsRecipe(t *testing.T) { + customize := func(builder beat.Builder) beat.Builder { + return builder. + WithRoles(beat.PSPClusterRoleName). + WithESValidations( + beat.HasEvent("event.dataset:system.cpu"), + beat.HasEvent("event.dataset:system.load"), + beat.HasEvent("event.dataset:system.memory"), + beat.HasEvent("event.dataset:system.network"), + beat.HasEvent("event.dataset:system.process"), + beat.HasEvent("event.dataset:system.process.summary"), + beat.HasEvent("event.dataset:system.fsstat"), + ) + } + + runBeatRecipe(t, "metricbeat_hosts.yaml", customize) +} + +func TestHeartbeatEsKbHealthRecipe(t *testing.T) { + customize := func(builder beat.Builder) beat.Builder { + cfg := settings.MustCanonicalConfig(builder.Beat.Spec.Config.Data) + yamlBytes, err := cfg.Render() + require.NoError(t, err) + + spec := builder.Beat.Spec + newEsHost := fmt.Sprintf("%s.%s.svc", esv1.HTTPService(spec.ElasticsearchRef.Name), builder.Beat.Namespace) + newKbHost := fmt.Sprintf("%s.%s.svc", kibana.HTTPService(spec.KibanaRef.Name), builder.Beat.Namespace) + + yaml := string(yamlBytes) + yaml = strings.ReplaceAll(yaml, "elasticsearch-es-http.default.svc", newEsHost) + yaml = strings.ReplaceAll(yaml, "kibana-kb-http.default.svc", newKbHost) + + builder.Beat.Spec.Config = &commonv1.Config{} + err = settings.MustParseConfig([]byte(yaml)).Unpack(&builder.Beat.Spec.Config.Data) + require.NoError(t, err) + + return builder. + WithRoles(beat.PSPClusterRoleName). + WithESValidations( + beat.HasEvent("monitor.status:up"), + ) + } + + runBeatRecipe(t, "heartbeat_es_kb_health.yaml", customize) +} + +func TestAuditbeatHostsRecipe(t *testing.T) { + if test.Ctx().Provider == "kind" { + // kind doesn't support configuring required settings + // see https://github.com/elastic/cloud-on-k8s/issues/3328 for more context + t.SkipNow() + } + + customize := func(builder beat.Builder) beat.Builder { + return builder. + WithRoles(beat.AuditbeatPSPClusterRoleName). + WithESValidations( + beat.HasEvent("event.dataset:file"), + beat.HasEvent("event.module:file_integrity"), + ) + } + + runBeatRecipe(t, "auditbeat_hosts.yaml", customize) +} + +func TestPacketbeatDnsHttpRecipe(t *testing.T) { + customize := func(builder beat.Builder) beat.Builder { + if !(test.Ctx().Provider == "kind" && test.Ctx().KubernetesVersion == "1.12") { + // there are some issues with kind 1.12 and tracking http traffic + builder = builder.WithESValidations(beat.HasEvent("event.dataset:http")) + } + + return builder. + WithRoles(beat.PacketbeatPSPClusterRoleName). + WithESValidations( + beat.HasEvent("event.dataset:flow"), + beat.HasEvent("event.dataset:dns"), + ) + } + + runBeatRecipe(t, "packetbeat_dns_http.yaml", customize) +} + +func TestJournalbeatHostsRecipe(t *testing.T) { + customize := func(builder beat.Builder) beat.Builder { + return builder. + WithRoles(beat.JournalbeatPSPClusterRoleName) + } + + runBeatRecipe(t, "journalbeat_hosts.yaml", customize) +} + +func runBeatRecipe( + t *testing.T, + fileName string, + customize func(builder beat.Builder) beat.Builder, + additionalObjects ...runtime.Object, +) { + filePath := path.Join("../../../config/recipes/beats", fileName) + namespace := test.Ctx().ManagedNamespace(0) + suffix := rand.String(4) + + transformationsWrapped := func(builder test.Builder) test.Builder { + beatBuilder, ok := builder.(beat.Builder) + if !ok { + return builder + } + + if customize != nil { + beatBuilder = customize(beatBuilder) + } + + return beatBuilder. + WithESValidations(beat.HasEventFromBeat(beatcommon.Type(beatBuilder.Beat.Spec.Type))) + } + + helper.RunFile(t, filePath, namespace, suffix, additionalObjects, transformationsWrapped) +} + +func loggingTestPod(name string) (*corev1.Pod, string) { + podBuilder := beat.NewPodBuilder(name) + return &podBuilder.Pod, podBuilder.Logged +} diff --git a/test/e2e/beat/setup_test.go b/test/e2e/beat/setup_test.go index 1d40274ac2..c6a4139995 100644 --- a/test/e2e/beat/setup_test.go +++ b/test/e2e/beat/setup_test.go @@ -50,6 +50,7 @@ func TestBeatKibanaRef(t *testing.T) { fbBuilder := beat.NewBuilder(name). WithType(filebeat.Type). + WithRoles(beat.PSPClusterRoleName, beat.AutodiscoverClusterRoleName). WithElasticsearchRef(esBuilder.Ref()). WithKibanaRef(kbBuilder.Ref()) @@ -57,6 +58,7 @@ func TestBeatKibanaRef(t *testing.T) { mbBuilder := beat.NewBuilder(name). WithType(metricbeat.Type). + WithRoles(beat.PSPClusterRoleName, beat.AutodiscoverClusterRoleName). WithElasticsearchRef(esBuilder.Ref()). WithKibanaRef(kbBuilder.Ref()). WithRoles(beat.MetricbeatClusterRoleName) @@ -65,6 +67,7 @@ func TestBeatKibanaRef(t *testing.T) { hbBuilder := beat.NewBuilder(name). WithType(heartbeat.Type). + WithRoles(beat.PSPClusterRoleName). WithDeployment(). WithElasticsearchRef(esBuilder.Ref()) diff --git a/test/e2e/samples_test.go b/test/e2e/samples_test.go index a7549e4b4c..cdce2cf5a7 100644 --- a/test/e2e/samples_test.go +++ b/test/e2e/samples_test.go @@ -8,7 +8,6 @@ import ( "bufio" "os" "path/filepath" - "strings" "testing" commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" @@ -31,7 +30,7 @@ func TestSamples(t *testing.T) { decoder := helper.NewYAMLDecoder() for _, sample := range sampleFiles { - testName := mkTestName(t, sample) + testName := helper.MkTestName(t, sample) builders := createBuilders(t, decoder, sample, testName) t.Run(testName, func(t *testing.T) { test.Sequence(nil, test.EmptySteps, builders...).RunSequential(t) @@ -39,18 +38,6 @@ func TestSamples(t *testing.T) { } } -func mkTestName(t *testing.T, path string) string { - t.Helper() - - baseName := filepath.Base(path) - baseName = strings.TrimSuffix(baseName, ".yaml") - parentDir := filepath.Base(filepath.Dir(path)) - testName := filepath.Join(parentDir, baseName) - - // testName will be used as label, so avoid using illegal chars - return strings.ReplaceAll(testName, "/", "-") -} - func createBuilders(t *testing.T, decoder *helper.YAMLDecoder, sampleFile, testName string) []test.Builder { t.Helper() diff --git a/test/e2e/test/beat/builder.go b/test/e2e/test/beat/builder.go index f4436cd893..002aa4ccec 100644 --- a/test/e2e/test/beat/builder.go +++ b/test/e2e/test/beat/builder.go @@ -48,8 +48,20 @@ func (b Builder) SkipTest() bool { } -func NewBuilderWithoutSuffix(name string) Builder { - return newBuilder(name, "") +// NewBuilderFromBeat creates a Beat builder from an existing Beat config. Sets all additional Builder fields +// appropriately. +func NewBuilderFromBeat(beat *beatv1beta1.Beat) Builder { + var podTemplate *corev1.PodTemplateSpec + if beat.Spec.DaemonSet != nil { + podTemplate = &beat.Spec.DaemonSet.PodTemplate + } else if beat.Spec.Deployment != nil { + podTemplate = &beat.Spec.Deployment.PodTemplate + } + + return Builder{ + Beat: *beat, + PodTemplate: podTemplate, + } } func NewBuilder(name string) Builder { @@ -73,8 +85,7 @@ func newBuilder(name string, suffix string) Builder { }. WithSuffix(suffix). WithLabel(run.TestNameLabel, name). - WithDaemonSet(). - WithRoles(AutodiscoverClusterRoleName, PSPClusterRoleName) + WithDaemonSet() } type ValidationFunc func(client.Client) error @@ -196,9 +207,10 @@ func (b Builder) WithRoles(clusterRoleNames ...string) Builder { } func bind(b Builder, clusterRoleName string) Builder { - saName := fmt.Sprintf("%s-sa", b.Beat.Name) + saName := b.PodTemplate.Spec.ServiceAccountName - if b.PodTemplate.Spec.ServiceAccountName != saName { + if saName == "" { + saName = fmt.Sprintf("%s-sa", b.Beat.Name) b = b.WithPodTemplateServiceAccount(saName) sa := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/e2e/test/beat/checks.go b/test/e2e/test/beat/checks.go index 9ec1ed0623..cda35e9a45 100644 --- a/test/e2e/test/beat/checks.go +++ b/test/e2e/test/beat/checks.go @@ -27,11 +27,37 @@ func HasMessageContaining(message string) ValidationFunc { return HasEvent(fmt.Sprintf("message:%s", message)) } +func NoMessageContaining(message string) ValidationFunc { + return NoEvent(fmt.Sprintf("message:%s", message)) +} + func HasEvent(query string) ValidationFunc { return hasEvent(fmt.Sprintf("/*beat*/_search?q=%s", query)) } +func NoEvent(query string) ValidationFunc { + return noEvent(fmt.Sprintf("/*beat*/_search?q=%s", query)) +} + func hasEvent(url string) ValidationFunc { + return checkEvent(url, func(hitsCount int) error { + if hitsCount == 0 { + return fmt.Errorf("hit count should be more than 0 for %s", url) + } + return nil + }) +} + +func noEvent(url string) ValidationFunc { + return checkEvent(url, func(hitsCount int) error { + if hitsCount != 0 { + return fmt.Errorf("hit count should be 0 for %s", url) + } + return nil + }) +} + +func checkEvent(url string, check func(int) error) ValidationFunc { return func(esClient client.Client) error { req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { @@ -52,8 +78,8 @@ func hasEvent(url string) ValidationFunc { if err != nil { return err } - if len(results.Hits.Hits) == 0 { - return fmt.Errorf("hit count should be more than 0 for %s", url) + if err := check(len(results.Hits.Hits)); err != nil { + return err } return nil diff --git a/test/e2e/test/helper/yaml.go b/test/e2e/test/helper/yaml.go index 7d2ad0933a..1abca7b7f4 100644 --- a/test/e2e/test/helper/yaml.go +++ b/test/e2e/test/helper/yaml.go @@ -8,18 +8,30 @@ import ( "bufio" "fmt" "io" + "os" + "path/filepath" + "strings" + "testing" apmv1 "github.com/elastic/cloud-on-k8s/pkg/apis/apm/v1" beatv1beta1 "github.com/elastic/cloud-on-k8s/pkg/apis/beat/v1beta1" + commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" entv1beta1 "github.com/elastic/cloud-on-k8s/pkg/apis/enterprisesearch/v1beta1" kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1" + beatcommon "github.com/elastic/cloud-on-k8s/pkg/controller/beat/common" + "github.com/elastic/cloud-on-k8s/test/e2e/cmd/run" "github.com/elastic/cloud-on-k8s/test/e2e/test" "github.com/elastic/cloud-on-k8s/test/e2e/test/apmserver" "github.com/elastic/cloud-on-k8s/test/e2e/test/beat" "github.com/elastic/cloud-on-k8s/test/e2e/test/elasticsearch" "github.com/elastic/cloud-on-k8s/test/e2e/test/enterprisesearch" "github.com/elastic/cloud-on-k8s/test/e2e/test/kibana" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + meta2 "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/yaml" @@ -39,6 +51,10 @@ func NewYAMLDecoder() *YAMLDecoder { scheme.AddKnownTypes(apmv1.GroupVersion, &apmv1.ApmServer{}, &apmv1.ApmServerList{}) scheme.AddKnownTypes(beatv1beta1.GroupVersion, &beatv1beta1.Beat{}, &beatv1beta1.BeatList{}) scheme.AddKnownTypes(entv1beta1.GroupVersion, &entv1beta1.EnterpriseSearch{}, &entv1beta1.EnterpriseSearchList{}) + + scheme.AddKnownTypes(rbacv1.SchemeGroupVersion, &rbacv1.ClusterRoleBinding{}, &rbacv1.ClusterRoleBindingList{}) + scheme.AddKnownTypes(rbacv1.SchemeGroupVersion, &rbacv1.ClusterRole{}, &rbacv1.ClusterRoleList{}) + scheme.AddKnownTypes(corev1.SchemeGroupVersion, &corev1.ServiceAccount{}, &corev1.ServiceAccountList{}) decoder := serializer.NewCodecFactory(scheme).UniversalDeserializer() return &YAMLDecoder{decoder: decoder} @@ -77,19 +93,7 @@ func (yd *YAMLDecoder) ToBuilders(reader *bufio.Reader, transform BuilderTransfo b.ApmServer = *decodedObj builder = transform(b) case *beatv1beta1.Beat: - b := beat.NewBuilderWithoutSuffix(decodedObj.Name) - b.Beat = *decodedObj - // adjust builder to compensate for overwriting Beat struct - // RBAC objects were populated using a name that doesn't have proper test suffix, - // hence clearing them here so the next calls can populate them properly - b.AdditionalObjects = nil - - // Since b.Beat was overwritten, b.PodTemplate is pointing to wrong struct, fixing it here - if b.Beat.Spec.DaemonSet != nil { - b.PodTemplate = &b.Beat.Spec.DaemonSet.PodTemplate - } else if b.Beat.Spec.Deployment != nil { - b.PodTemplate = &b.Beat.Spec.Deployment.PodTemplate - } + b := beat.NewBuilderFromBeat(decodedObj) builder = transform(b) case *entv1beta1.EnterpriseSearch: b := enterprisesearch.NewBuilderWithoutSuffix(decodedObj.Name) @@ -104,3 +108,212 @@ func (yd *YAMLDecoder) ToBuilders(reader *bufio.Reader, transform BuilderTransfo return builders, nil } + +func (yd *YAMLDecoder) ToObjects(reader *bufio.Reader) ([]runtime.Object, error) { + var objects []runtime.Object + + yamlReader := yaml.NewYAMLReader(reader) + for { + yamlBytes, err := yamlReader.Read() + if err != nil { + if err == io.EOF { + break + } + return nil, fmt.Errorf("failed to read YAML: %w", err) + } + obj, _, err := yd.decoder.Decode(yamlBytes, nil, nil) + if err != nil { + return nil, fmt.Errorf("failed to decode YAML: %w", err) + } + + objects = append(objects, obj) + } + + return objects, nil +} + +// RunFile runs builder workflow for all known resources in a yaml file, all other objects are created before and deleted +// after. Resources will be created in a given namespace and with a given suffix. Additional objects to be created and deleted +// can be passed as well as set of optional transformations to apply to all Builders. +func RunFile( + t *testing.T, + filePath, namespace, suffix string, + additionalObjects []runtime.Object, + transformations ...BuilderTransform) { + builders, objects, err := extractFromFile(t, filePath, namespace, suffix, MkTestName(t, filePath), transformations...) + if err != nil { + panic(err) + } + + objects = append(objects, additionalObjects...) + + creates, deletes := makeObjectSteps(t, objects) + + test.BeforeAfterSequence(creates, deletes, builders...).RunSequential(t) +} + +func extractFromFile( + t *testing.T, + filePath, namespace, suffix, fullTestName string, + transformations ...BuilderTransform, +) ([]test.Builder, []runtime.Object, error) { + f, err := os.Open(filePath) + require.NoError(t, err, "Failed to open file %s", filePath) + defer f.Close() + + decoder := NewYAMLDecoder() + objects, err := decoder.ToObjects(bufio.NewReader(f)) + if err != nil { + return nil, nil, err + } + + builders, objects := transform(namespace, fullTestName, suffix, transformations, objects) + return builders, objects, nil +} + +func makeObjectSteps( + t *testing.T, + objects []runtime.Object, +) (func(k *test.K8sClient) test.StepList, func(k *test.K8sClient) test.StepList) { + return func(k *test.K8sClient) test.StepList { + steps := test.StepList{} + for i := range objects { + ii := i + meta, err := meta2.Accessor(objects[ii]) + require.NoError(t, err) + steps = steps.WithStep(test.Step{ + Name: fmt.Sprintf("Create %s/%s", meta.GetNamespace(), meta.GetNamespace()), + Test: func(t *testing.T) { + err := k.Client.Create(objects[ii]) + if !k8serrors.IsAlreadyExists(err) { + require.NoError(t, err) + } + }, + }) + } + return steps + }, func(k *test.K8sClient) test.StepList { + steps := test.StepList{} + for i := range objects { + ii := i + meta, err := meta2.Accessor(objects[ii]) + require.NoError(t, err) + steps = steps.WithStep(test.Step{ + Name: fmt.Sprintf("Delete %s/%s", meta.GetNamespace(), meta.GetNamespace()), + Test: func(t *testing.T) { + err := k.Client.Delete(objects[ii]) + if !k8serrors.IsNotFound(err) { + require.NoError(t, err) + } + }, + }) + } + return steps + } +} + +func transform(namespace, fullTestName, suffix string, transformers []BuilderTransform, objects []runtime.Object) ([]test.Builder, []runtime.Object) { + var builders []test.Builder + var otherObjects []runtime.Object + for _, object := range objects { + var builder test.Builder + switch decodedObj := object.(type) { + case *esv1.Elasticsearch: + b := elasticsearch.NewBuilderWithoutSuffix(decodedObj.Name) + b.Elasticsearch = *decodedObj + builder = b.WithNamespace(namespace). + WithSuffix(suffix). + WithRestrictedSecurityContext(). + WithLabel(run.TestNameLabel, fullTestName). + WithPodLabel(run.TestNameLabel, fullTestName) + case *kbv1.Kibana: + b := kibana.NewBuilderWithoutSuffix(decodedObj.Name) + b.Kibana = *decodedObj + builder = b.WithNamespace(namespace). + WithSuffix(suffix). + WithElasticsearchRef(tweakServiceRef(b.Kibana.Spec.ElasticsearchRef, suffix)). + WithRestrictedSecurityContext(). + WithLabel(run.TestNameLabel, fullTestName). + WithPodLabel(run.TestNameLabel, fullTestName) + case *apmv1.ApmServer: + b := apmserver.NewBuilderWithoutSuffix(decodedObj.Name) + b.ApmServer = *decodedObj + builder = b.WithNamespace(namespace). + WithSuffix(suffix). + WithElasticsearchRef(tweakServiceRef(b.ApmServer.Spec.ElasticsearchRef, suffix)). + WithKibanaRef(tweakServiceRef(b.ApmServer.Spec.KibanaRef, suffix)). + WithConfig(map[string]interface{}{"apm-server.ilm.enabled": false}). + WithRestrictedSecurityContext(). + WithLabel(run.TestNameLabel, fullTestName). + WithPodLabel(run.TestNameLabel, fullTestName) + case *beatv1beta1.Beat: + b := beat.NewBuilderFromBeat(decodedObj) + + builder = b.WithNamespace(namespace). + WithSuffix(suffix). + WithElasticsearchRef(tweakServiceRef(b.Beat.Spec.ElasticsearchRef, suffix)). + WithLabel(run.TestNameLabel, fullTestName). + WithPodLabel(run.TestNameLabel, fullTestName). + WithESValidations(beat.HasEventFromBeat(beatcommon.Type(b.Beat.Spec.Type))). + WithKibanaRef(tweakServiceRef(b.Beat.Spec.KibanaRef, suffix)) + + if b.PodTemplate.Spec.ServiceAccountName != "" { + b = b.WithPodTemplateServiceAccount(b.PodTemplate.Spec.ServiceAccountName + "-" + suffix) + } + case *entv1beta1.EnterpriseSearch: + b := enterprisesearch.NewBuilderWithoutSuffix(decodedObj.Name) + b.EnterpriseSearch = *decodedObj + builder = b.WithNamespace(namespace). + WithSuffix(suffix). + WithElasticsearchRef(tweakServiceRef(b.EnterpriseSearch.Spec.ElasticsearchRef, suffix)). + WithRestrictedSecurityContext(). + WithLabel(run.TestNameLabel, fullTestName). + WithPodLabel(run.TestNameLabel, fullTestName) + case *corev1.ServiceAccount: + decodedObj.Namespace = namespace + decodedObj.Name = decodedObj.Name + "-" + suffix + case *rbacv1.ClusterRoleBinding: + decodedObj.Subjects[0].Namespace = namespace + decodedObj.Subjects[0].Name = decodedObj.Subjects[0].Name + "-" + suffix + decodedObj.RoleRef.Name = decodedObj.RoleRef.Name + "-" + suffix + decodedObj.Name = decodedObj.Name + "-" + suffix + case *rbacv1.ClusterRole: + decodedObj.Name = decodedObj.Name + "-" + suffix + } + + if builder != nil { + // ECK driven resources can be further transformed + for _, transformer := range transformers { + builder = transformer(builder) + } + builders = append(builders, builder) + } else { + // built-in in resources are separated as they are treated differently + otherObjects = append(otherObjects, object) + } + } + + return builders, otherObjects +} + +func tweakServiceRef(ref commonv1.ObjectSelector, suffix string) commonv1.ObjectSelector { + // All the objects defined in the YAML file will have a random test suffix added to prevent clashes with previous runs. + // This necessitates changing the Elasticsearch reference to match the suffixed name. + if ref.Name != "" { + ref.Name = ref.Name + "-" + suffix + } + + return ref +} + +func MkTestName(t *testing.T, path string) string { + t.Helper() + + baseName := filepath.Base(path) + baseName = strings.TrimSuffix(baseName, ".yaml") + parentDir := filepath.Base(filepath.Dir(path)) + testName := filepath.Join(parentDir, baseName) + + // testName will be used as label, so avoid using illegal chars + return strings.ReplaceAll(testName, "/", "-") +} diff --git a/test/e2e/test/run.go b/test/e2e/test/run.go index cc168c8124..3470e2f165 100644 --- a/test/e2e/test/run.go +++ b/test/e2e/test/run.go @@ -40,3 +40,40 @@ func Sequence(before StepsFunc, f StepsFunc, builders ...Builder) StepList { return steps } + +// BeforeAfterSequence returns a list of steps corresponding to a workflow that allows defining list of step to execute +// before and after builder workflow (before steps, init, create, checks, deletes, after steps) +func BeforeAfterSequence(before StepsFunc, after StepsFunc, builders ...Builder) StepList { + steps := StepList{} + for _, b := range builders { + // ignore the test if some builders cannot be tested + if b.SkipTest() { + return steps + } + } + + k := NewK8sClientOrFatal() + + if before != nil { + steps = steps.WithSteps(before(k)) + } + + for _, b := range builders { + steps = steps.WithSteps(b.InitTestSteps(k)) + } + for _, b := range builders { + steps = steps.WithSteps(b.CreationTestSteps(k)) + } + for _, b := range builders { + steps = steps.WithSteps(CheckTestSteps(b, k)) + } + for _, b := range builders { + steps = steps.WithSteps(b.DeletionTestSteps(k)) + } + + if after != nil { + steps = steps.WithSteps(after(k)) + } + + return steps +} From 248adb9b57db34b09d44cedf628b2a574345eb9a Mon Sep 17 00:00:00 2001 From: David Kowalski Date: Thu, 9 Jul 2020 09:01:14 +0200 Subject: [PATCH 2/9] Fix commented security context placement --- config/recipes/beats/filebeat_no_autodiscover.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/recipes/beats/filebeat_no_autodiscover.yaml b/config/recipes/beats/filebeat_no_autodiscover.yaml index 9603fac8dc..6867655cd1 100644 --- a/config/recipes/beats/filebeat_no_autodiscover.yaml +++ b/config/recipes/beats/filebeat_no_autodiscover.yaml @@ -26,10 +26,11 @@ spec: hostNetwork: true securityContext: runAsUser: 0 - # If using Red Hat OpenShift uncomment this: - #privileged: true containers: - name: filebeat + # If using Red Hat OpenShift uncomment this: + #securityContext: + #privileged: true volumeMounts: - name: varlogcontainers mountPath: /var/log/containers From 7aef6418d4ad35834047464a9344b933cd631104 Mon Sep 17 00:00:00 2001 From: David Kowalski <50632861+david-kow@users.noreply.github.com> Date: Thu, 9 Jul 2020 16:05:43 +0200 Subject: [PATCH 3/9] Apply suggestions from code review Co-authored-by: Peter Brachwitz --- test/e2e/test/helper/yaml.go | 2 +- test/e2e/test/run.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/test/helper/yaml.go b/test/e2e/test/helper/yaml.go index 1abca7b7f4..80ef673fb1 100644 --- a/test/e2e/test/helper/yaml.go +++ b/test/e2e/test/helper/yaml.go @@ -132,7 +132,7 @@ func (yd *YAMLDecoder) ToObjects(reader *bufio.Reader) ([]runtime.Object, error) return objects, nil } -// RunFile runs builder workflow for all known resources in a yaml file, all other objects are created before and deleted +// RunFile runs the builder workflow for all known resources in a yaml file, all other objects are created before and deleted // after. Resources will be created in a given namespace and with a given suffix. Additional objects to be created and deleted // can be passed as well as set of optional transformations to apply to all Builders. func RunFile( diff --git a/test/e2e/test/run.go b/test/e2e/test/run.go index 3470e2f165..70ccba0f7c 100644 --- a/test/e2e/test/run.go +++ b/test/e2e/test/run.go @@ -41,7 +41,7 @@ func Sequence(before StepsFunc, f StepsFunc, builders ...Builder) StepList { return steps } -// BeforeAfterSequence returns a list of steps corresponding to a workflow that allows defining list of step to execute +// BeforeAfterSequence returns a list of steps corresponding to a workflow that allows defining a list of steps to execute // before and after builder workflow (before steps, init, create, checks, deletes, after steps) func BeforeAfterSequence(before StepsFunc, after StepsFunc, builders ...Builder) StepList { steps := StepList{} From 48092644866c9be37d19ce3be8fdb1a1ed58a1cf Mon Sep 17 00:00:00 2001 From: David Kowalski Date: Tue, 14 Jul 2020 09:23:36 +0200 Subject: [PATCH 4/9] Add beats config library readme --- config/recipes/beats/README.md | 39 +++++++++++++++++++ .../recipes/beats/heartbeat_es_kb_health.yaml | 1 + test/e2e/test/helper/yaml.go | 4 +- 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 config/recipes/beats/README.md diff --git a/config/recipes/beats/README.md b/config/recipes/beats/README.md new file mode 100644 index 0000000000..74bf3441b8 --- /dev/null +++ b/config/recipes/beats/README.md @@ -0,0 +1,39 @@ +# Beats Configuration Library + +This directory contains yaml manifests with example configurations for Beats. These manifests are self-contained and work out-of-the-box on any non-secured Kubernetes cluster. All of them contain three-node Elasticsearch cluster and single Kibana instance. All Beat configurations set up Kibana dashboards if they are available for a given Beat and all required RBAC resources. + +#### Metricbeat for Kubernetes monitoring - `metricbeat_hosts.yaml` + +Deploys Metricbeat as a DaemonSet that monitors the host resource usage (cpu, memory, network, filesystem) and Kubernetes resources (nodes, pods, containers, volumes). + +#### Filebeat with autodiscover - `filebeat_autodiscover.yaml` + +Deploys Filebeat as DaemonSet with autodiscover feature enabled. All pods in all namespace will have logs shipped to Elasticsearch cluster. + +#### Filebeat with autodiscover for metadata - `filebeat_autodiscover_by_metadata.yaml` + +Deploys Filebeat as a DaemonSet with autodiscover feature enabled. Fullfilling any of the two conditions below will cause a given Pod logs to be shipped to Elasticsearch cluster: + +- Pod is in `log-namespace` namespace +- Pod has `log-label: "true"` label + +#### Filebeat without autodiscover - `filebeat_no_autodiscover.yaml` + +Deploys Filebeat as a DaemonSet with autodiscover feature disabled. Uses entire logs directory on the host as the input. Doesn't require any RBAC resources as no Kubernetes APIs are used. + +#### Heartbeat monitoring Elasticsearch and Kibana health - `heartbeat_es_kb_heatlh.yaml` + +Deploys Heartbeat as a single Pod deployment that monitors the health of Elasticsearch and Kibana by TCP probing their Service endpoints. + +#### Auditbeat - `auditbeat_hosts.yaml` + +Deploys Auditbeat as DaemonSet that checks file integrity and audits file operations on the host system. + +#### Journalbeat - `journalbeat_hosts.yaml` + +Deploys Journalbeat as a DaemonSet that ships data from systemd journals. + + +#### Packetbeat monitoring DNS and HTTP traffic - `packetbeat_dns_http.yaml` + +Deploys Packetbeat as a DaemonSet that monitors DNS on port `53` and HTTP(S) traffic on ports `80`, `8000`, `8080` and `9200`. diff --git a/config/recipes/beats/heartbeat_es_kb_health.yaml b/config/recipes/beats/heartbeat_es_kb_health.yaml index 91bdb50c26..95a6a456dc 100644 --- a/config/recipes/beats/heartbeat_es_kb_health.yaml +++ b/config/recipes/beats/heartbeat_es_kb_health.yaml @@ -18,6 +18,7 @@ spec: schedule: '@every 5s' hosts: ["kibana-kb-http.default.svc:5601"] deployment: + replicas: 1 podTemplate: spec: dnsPolicy: ClusterFirstWithHostNet diff --git a/test/e2e/test/helper/yaml.go b/test/e2e/test/helper/yaml.go index 80ef673fb1..a4b9cd54df 100644 --- a/test/e2e/test/helper/yaml.go +++ b/test/e2e/test/helper/yaml.go @@ -167,7 +167,7 @@ func extractFromFile( return nil, nil, err } - builders, objects := transform(namespace, fullTestName, suffix, transformations, objects) + builders, objects := transformToE2E(namespace, fullTestName, suffix, transformations, objects) return builders, objects, nil } @@ -212,7 +212,7 @@ func makeObjectSteps( } } -func transform(namespace, fullTestName, suffix string, transformers []BuilderTransform, objects []runtime.Object) ([]test.Builder, []runtime.Object) { +func transformToE2E(namespace, fullTestName, suffix string, transformers []BuilderTransform, objects []runtime.Object) ([]test.Builder, []runtime.Object) { var builders []test.Builder var otherObjects []runtime.Object for _, object := range objects { From 7b1e9295833e510ae601a7623c10f40c70163bd6 Mon Sep 17 00:00:00 2001 From: David Kowalski <50632861+david-kow@users.noreply.github.com> Date: Tue, 14 Jul 2020 14:47:16 +0200 Subject: [PATCH 5/9] Apply suggestions from code review Co-authored-by: Peter Brachwitz --- config/recipes/beats/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/recipes/beats/README.md b/config/recipes/beats/README.md index 74bf3441b8..a9c7d40a45 100644 --- a/config/recipes/beats/README.md +++ b/config/recipes/beats/README.md @@ -8,11 +8,11 @@ Deploys Metricbeat as a DaemonSet that monitors the host resource usage (cpu, me #### Filebeat with autodiscover - `filebeat_autodiscover.yaml` -Deploys Filebeat as DaemonSet with autodiscover feature enabled. All pods in all namespace will have logs shipped to Elasticsearch cluster. +Deploys Filebeat as DaemonSet with autodiscover feature enabled. All pods in all namespace will have logs shipped to a Elasticsearch cluster. #### Filebeat with autodiscover for metadata - `filebeat_autodiscover_by_metadata.yaml` -Deploys Filebeat as a DaemonSet with autodiscover feature enabled. Fullfilling any of the two conditions below will cause a given Pod logs to be shipped to Elasticsearch cluster: +Deploys Filebeat as a DaemonSet with autodiscover feature enabled. Fullfilling any of the two conditions below will cause a given Pod logs to be shipped to a Elasticsearch cluster: - Pod is in `log-namespace` namespace - Pod has `log-label: "true"` label @@ -27,7 +27,7 @@ Deploys Heartbeat as a single Pod deployment that monitors the health of Elastic #### Auditbeat - `auditbeat_hosts.yaml` -Deploys Auditbeat as DaemonSet that checks file integrity and audits file operations on the host system. +Deploys Auditbeat as a DaemonSet that checks file integrity and audits file operations on the host system. #### Journalbeat - `journalbeat_hosts.yaml` From 8e3789a717f78a4dfc8ed3f3da2419530e6be703 Mon Sep 17 00:00:00 2001 From: David Kowalski Date: Tue, 14 Jul 2020 16:08:57 +0200 Subject: [PATCH 6/9] Remove add_kubernetes_metadata processor --- config/recipes/beats/auditbeat_hosts.yaml | 9 +-------- config/recipes/beats/journalbeat_hosts.yaml | 18 ++---------------- config/recipes/beats/packetbeat_dns_http.yaml | 10 ++-------- 3 files changed, 5 insertions(+), 32 deletions(-) diff --git a/config/recipes/beats/auditbeat_hosts.yaml b/config/recipes/beats/auditbeat_hosts.yaml index 05ef2e7e54..7b505a82c1 100644 --- a/config/recipes/beats/auditbeat_hosts.yaml +++ b/config/recipes/beats/auditbeat_hosts.yaml @@ -38,16 +38,9 @@ spec: processors: - add_cloud_metadata: {} + - add_host_metadata: {} - add_process_metadata: match_pids: ['process.pid'] - - add_kubernetes_metadata: - host: ${HOSTNAME} - default_indexers.enabled: false - default_matchers.enabled: false - indexers: - - container: - matchers: - - fields.lookup_fields: ['container.id'] daemonSet: podTemplate: spec: diff --git a/config/recipes/beats/journalbeat_hosts.yaml b/config/recipes/beats/journalbeat_hosts.yaml index bbd716dcf9..a5ac5e33c5 100644 --- a/config/recipes/beats/journalbeat_hosts.yaml +++ b/config/recipes/beats/journalbeat_hosts.yaml @@ -15,22 +15,8 @@ spec: seek: cursor cursor_seek_fallback: tail processors: - - add_kubernetes_metadata: - host: "${HOSTNAME}" - in_cluster: true - default_indexers.enabled: false - default_matchers.enabled: false - indexers: - - container: - matchers: - - fields: - lookup_fields: ["container.id"] - - decode_json_fields: - fields: ["message"] - process_array: false - max_depth: 1 - target: "" - overwrite_keys: true + - add_cloud_metadata: {} + - add_host_metadata: {} daemonSet: podTemplate: spec: diff --git a/config/recipes/beats/packetbeat_dns_http.yaml b/config/recipes/beats/packetbeat_dns_http.yaml index 2158c9684d..7ad9a56283 100644 --- a/config/recipes/beats/packetbeat_dns_http.yaml +++ b/config/recipes/beats/packetbeat_dns_http.yaml @@ -22,14 +22,8 @@ spec: timeout: 30s period: 10s processors: - - add_cloud_metadata: - - add_kubernetes_metadata: - host: ${HOSTNAME} - indexers: - - ip_port: - matchers: - - field_format: - format: '%{[ip]}:%{[port]}' + - add_cloud_metadata: {} + - add_host_metadata: {} daemonSet: podTemplate: spec: From c1fc46420390dc6392815ee28de076908c95a1c4 Mon Sep 17 00:00:00 2001 From: David Kowalski Date: Thu, 16 Jul 2020 08:20:25 +0200 Subject: [PATCH 7/9] PR fixes --- config/recipes/beats/README.md | 10 +++++----- config/recipes/beats/auditbeat_hosts.yaml | 2 +- config/recipes/beats/filebeat_autodiscover.yaml | 2 +- .../beats/filebeat_autodiscover_by_metadata.yaml | 2 +- config/recipes/beats/filebeat_no_autodiscover.yaml | 1 + config/recipes/beats/heartbeat_es_kb_health.yaml | 2 +- config/recipes/beats/journalbeat_hosts.yaml | 2 +- config/recipes/beats/metricbeat_hosts.yaml | 2 +- config/recipes/beats/packetbeat_dns_http.yaml | 2 +- 9 files changed, 13 insertions(+), 12 deletions(-) diff --git a/config/recipes/beats/README.md b/config/recipes/beats/README.md index a9c7d40a45..1deeda457d 100644 --- a/config/recipes/beats/README.md +++ b/config/recipes/beats/README.md @@ -1,18 +1,19 @@ -# Beats Configuration Library +# Beats Configuration Examples This directory contains yaml manifests with example configurations for Beats. These manifests are self-contained and work out-of-the-box on any non-secured Kubernetes cluster. All of them contain three-node Elasticsearch cluster and single Kibana instance. All Beat configurations set up Kibana dashboards if they are available for a given Beat and all required RBAC resources. + #### Metricbeat for Kubernetes monitoring - `metricbeat_hosts.yaml` -Deploys Metricbeat as a DaemonSet that monitors the host resource usage (cpu, memory, network, filesystem) and Kubernetes resources (nodes, pods, containers, volumes). +Deploys Metricbeat as a DaemonSet that monitors the host resource usage (cpu, memory, network, filesystem) and Kubernetes resources (Nodes, Pods, Containers, Volumes). #### Filebeat with autodiscover - `filebeat_autodiscover.yaml` -Deploys Filebeat as DaemonSet with autodiscover feature enabled. All pods in all namespace will have logs shipped to a Elasticsearch cluster. +Deploys Filebeat as DaemonSet with autodiscover feature enabled. All pods in all namespaces will have logs shipped to an Elasticsearch cluster. #### Filebeat with autodiscover for metadata - `filebeat_autodiscover_by_metadata.yaml` -Deploys Filebeat as a DaemonSet with autodiscover feature enabled. Fullfilling any of the two conditions below will cause a given Pod logs to be shipped to a Elasticsearch cluster: +Deploys Filebeat as a DaemonSet with autodiscover feature enabled. Fullfilling any of the two conditions below will cause a given Pod's logs to be shipped to an Elasticsearch cluster: - Pod is in `log-namespace` namespace - Pod has `log-label: "true"` label @@ -33,7 +34,6 @@ Deploys Auditbeat as a DaemonSet that checks file integrity and audits file oper Deploys Journalbeat as a DaemonSet that ships data from systemd journals. - #### Packetbeat monitoring DNS and HTTP traffic - `packetbeat_dns_http.yaml` Deploys Packetbeat as a DaemonSet that monitors DNS on port `53` and HTTP(S) traffic on ports `80`, `8000`, `8080` and `9200`. diff --git a/config/recipes/beats/auditbeat_hosts.yaml b/config/recipes/beats/auditbeat_hosts.yaml index 7b505a82c1..a92ac096a2 100644 --- a/config/recipes/beats/auditbeat_hosts.yaml +++ b/config/recipes/beats/auditbeat_hosts.yaml @@ -123,4 +123,4 @@ spec: count: 1 elasticsearchRef: name: elasticsearch ---- +... diff --git a/config/recipes/beats/filebeat_autodiscover.yaml b/config/recipes/beats/filebeat_autodiscover.yaml index 06323c0b73..e419c7d12d 100644 --- a/config/recipes/beats/filebeat_autodiscover.yaml +++ b/config/recipes/beats/filebeat_autodiscover.yaml @@ -110,4 +110,4 @@ spec: count: 1 elasticsearchRef: name: elasticsearch ---- +... diff --git a/config/recipes/beats/filebeat_autodiscover_by_metadata.yaml b/config/recipes/beats/filebeat_autodiscover_by_metadata.yaml index a82f590044..e68111d7ae 100644 --- a/config/recipes/beats/filebeat_autodiscover_by_metadata.yaml +++ b/config/recipes/beats/filebeat_autodiscover_by_metadata.yaml @@ -112,4 +112,4 @@ spec: count: 1 elasticsearchRef: name: elasticsearch ---- +... diff --git a/config/recipes/beats/filebeat_no_autodiscover.yaml b/config/recipes/beats/filebeat_no_autodiscover.yaml index 6867655cd1..e79593aee1 100644 --- a/config/recipes/beats/filebeat_no_autodiscover.yaml +++ b/config/recipes/beats/filebeat_no_autodiscover.yaml @@ -70,3 +70,4 @@ spec: count: 1 elasticsearchRef: name: elasticsearch +... diff --git a/config/recipes/beats/heartbeat_es_kb_health.yaml b/config/recipes/beats/heartbeat_es_kb_health.yaml index 95a6a456dc..b69b1e8e60 100644 --- a/config/recipes/beats/heartbeat_es_kb_health.yaml +++ b/config/recipes/beats/heartbeat_es_kb_health.yaml @@ -47,4 +47,4 @@ spec: count: 1 elasticsearchRef: name: elasticsearch ---- +... diff --git a/config/recipes/beats/journalbeat_hosts.yaml b/config/recipes/beats/journalbeat_hosts.yaml index a5ac5e33c5..43f48b8133 100644 --- a/config/recipes/beats/journalbeat_hosts.yaml +++ b/config/recipes/beats/journalbeat_hosts.yaml @@ -67,4 +67,4 @@ spec: count: 1 elasticsearchRef: name: elasticsearch ---- +... diff --git a/config/recipes/beats/metricbeat_hosts.yaml b/config/recipes/beats/metricbeat_hosts.yaml index 4ece5e9ded..13c5ea0068 100644 --- a/config/recipes/beats/metricbeat_hosts.yaml +++ b/config/recipes/beats/metricbeat_hosts.yaml @@ -185,4 +185,4 @@ spec: count: 1 elasticsearchRef: name: elasticsearch ---- +... diff --git a/config/recipes/beats/packetbeat_dns_http.yaml b/config/recipes/beats/packetbeat_dns_http.yaml index 7ad9a56283..171352ab30 100644 --- a/config/recipes/beats/packetbeat_dns_http.yaml +++ b/config/recipes/beats/packetbeat_dns_http.yaml @@ -60,4 +60,4 @@ spec: count: 1 elasticsearchRef: name: elasticsearch ---- +... From 07d764cfa6516a0615c02238e1ea3feb11fdef89 Mon Sep 17 00:00:00 2001 From: David Kowalski Date: Thu, 16 Jul 2020 15:57:59 +0200 Subject: [PATCH 8/9] Add note about namespace expectations for heartbeat example --- config/recipes/beats/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/recipes/beats/README.md b/config/recipes/beats/README.md index 1deeda457d..8d35ecf76e 100644 --- a/config/recipes/beats/README.md +++ b/config/recipes/beats/README.md @@ -24,7 +24,7 @@ Deploys Filebeat as a DaemonSet with autodiscover feature disabled. Uses entire #### Heartbeat monitoring Elasticsearch and Kibana health - `heartbeat_es_kb_heatlh.yaml` -Deploys Heartbeat as a single Pod deployment that monitors the health of Elasticsearch and Kibana by TCP probing their Service endpoints. +Deploys Heartbeat as a single Pod deployment that monitors the health of Elasticsearch and Kibana by TCP probing their Service endpoints. Note that Heartbeat expects that Elasticsearch and Kibana are deployed in the `default` namespace. #### Auditbeat - `auditbeat_hosts.yaml` From 385391e39086733503eb24474143fbede8a3e499 Mon Sep 17 00:00:00 2001 From: David Kowalski Date: Fri, 17 Jul 2020 10:00:05 +0200 Subject: [PATCH 9/9] Add hostNetwork comments, add fix for GKE --- config/recipes/beats/auditbeat_hosts.yaml | 13 ++++++++++++- config/recipes/beats/filebeat_autodiscover.yaml | 2 +- .../beats/filebeat_autodiscover_by_metadata.yaml | 2 +- config/recipes/beats/filebeat_no_autodiscover.yaml | 2 +- config/recipes/beats/heartbeat_es_kb_health.yaml | 2 -- config/recipes/beats/journalbeat_hosts.yaml | 2 +- config/recipes/beats/metricbeat_hosts.yaml | 2 +- 7 files changed, 17 insertions(+), 8 deletions(-) diff --git a/config/recipes/beats/auditbeat_hosts.yaml b/config/recipes/beats/auditbeat_hosts.yaml index a92ac096a2..32bfcf3213 100644 --- a/config/recipes/beats/auditbeat_hosts.yaml +++ b/config/recipes/beats/auditbeat_hosts.yaml @@ -46,7 +46,7 @@ spec: spec: hostPID: true # Required by auditd module dnsPolicy: ClusterFirstWithHostNet - hostNetwork: true + hostNetwork: true # Allows to provide richer host metadata automountServiceAccountToken: true # some older Beat versions are depending on this settings presence in k8s context securityContext: runAsUser: 0 @@ -70,6 +70,17 @@ spec: hostPath: path: /run/containerd type: DirectoryOrCreate + # Uncomment the below when running on GKE. See https://github.com/elastic/beats/issues/8523 for more context. + #- name: run + # hostPath: + # path: /run + #initContainers: + #- name: cos-init + # image: docker.elastic.co/beats/auditbeat:7.8.0 + # volumeMounts: + # - name: run + # mountPath: /run + # command: ['sh', '-c', 'export SYSTEMD_IGNORE_CHROOT=1 && systemctl stop systemd-journald-audit.socket && systemctl mask systemd-journald-audit.socket && systemctl restart systemd-journald'] containers: - name: auditbeat securityContext: diff --git a/config/recipes/beats/filebeat_autodiscover.yaml b/config/recipes/beats/filebeat_autodiscover.yaml index e419c7d12d..86ec4cc92d 100644 --- a/config/recipes/beats/filebeat_autodiscover.yaml +++ b/config/recipes/beats/filebeat_autodiscover.yaml @@ -31,7 +31,7 @@ spec: automountServiceAccountToken: true terminationGracePeriodSeconds: 30 dnsPolicy: ClusterFirstWithHostNet - hostNetwork: true + hostNetwork: true # Allows to provide richer host metadata securityContext: runAsUser: 0 # If using Red Hat OpenShift uncomment this: diff --git a/config/recipes/beats/filebeat_autodiscover_by_metadata.yaml b/config/recipes/beats/filebeat_autodiscover_by_metadata.yaml index e68111d7ae..8709c00169 100644 --- a/config/recipes/beats/filebeat_autodiscover_by_metadata.yaml +++ b/config/recipes/beats/filebeat_autodiscover_by_metadata.yaml @@ -33,7 +33,7 @@ spec: automountServiceAccountToken: true terminationGracePeriodSeconds: 30 dnsPolicy: ClusterFirstWithHostNet - hostNetwork: true + hostNetwork: true # Allows to provide richer host metadata securityContext: runAsUser: 0 # If using Red Hat OpenShift uncomment this: diff --git a/config/recipes/beats/filebeat_no_autodiscover.yaml b/config/recipes/beats/filebeat_no_autodiscover.yaml index e79593aee1..736925613b 100644 --- a/config/recipes/beats/filebeat_no_autodiscover.yaml +++ b/config/recipes/beats/filebeat_no_autodiscover.yaml @@ -23,7 +23,7 @@ spec: automountServiceAccountToken: true terminationGracePeriodSeconds: 30 dnsPolicy: ClusterFirstWithHostNet - hostNetwork: true + hostNetwork: true # Allows to provide richer host metadata securityContext: runAsUser: 0 containers: diff --git a/config/recipes/beats/heartbeat_es_kb_health.yaml b/config/recipes/beats/heartbeat_es_kb_health.yaml index b69b1e8e60..98d236f8fe 100644 --- a/config/recipes/beats/heartbeat_es_kb_health.yaml +++ b/config/recipes/beats/heartbeat_es_kb_health.yaml @@ -21,8 +21,6 @@ spec: replicas: 1 podTemplate: spec: - dnsPolicy: ClusterFirstWithHostNet - hostNetwork: true securityContext: runAsUser: 0 --- diff --git a/config/recipes/beats/journalbeat_hosts.yaml b/config/recipes/beats/journalbeat_hosts.yaml index 43f48b8133..b0e646f4d4 100644 --- a/config/recipes/beats/journalbeat_hosts.yaml +++ b/config/recipes/beats/journalbeat_hosts.yaml @@ -31,7 +31,7 @@ spec: name: run-journal - mountPath: /etc/machine-id name: machine-id - hostNetwork: true + hostNetwork: true # Allows to provide richer host metadata securityContext: runAsUser: 0 terminationGracePeriodSeconds: 30 diff --git a/config/recipes/beats/metricbeat_hosts.yaml b/config/recipes/beats/metricbeat_hosts.yaml index 13c5ea0068..06bc0ad5d3 100644 --- a/config/recipes/beats/metricbeat_hosts.yaml +++ b/config/recipes/beats/metricbeat_hosts.yaml @@ -83,7 +83,7 @@ spec: - mountPath: /hostfs/proc name: proc dnsPolicy: ClusterFirstWithHostNet - hostNetwork: true + hostNetwork: true # Allows to provide richer host metadata securityContext: runAsUser: 0 terminationGracePeriodSeconds: 30