From 366f455e0ba6a541225301c60a77c9b8752f8d45 Mon Sep 17 00:00:00 2001 From: David Kowalski <50632861+david-kow@users.noreply.github.com> Date: Fri, 17 Jul 2020 10:36:17 +0200 Subject: [PATCH] Add Beat config library (#3406) * Add Beat config library * Fix commented security context placement * Apply suggestions from code review Co-authored-by: Peter Brachwitz * Add beats config library readme * Apply suggestions from code review Co-authored-by: Peter Brachwitz * Remove add_kubernetes_metadata processor * PR fixes * Add note about namespace expectations for heartbeat example * Add hostNetwork comments, add fix for GKE Co-authored-by: Peter Brachwitz --- config/dev/elastic-psp.yaml | 2 +- config/recipes/beats/0_ns.yaml | 4 - config/recipes/beats/1_monitor.yaml | 38 -- .../recipes/beats/2_filebeat-kubernetes.yaml | 158 ------- .../beats/3_metricbeat-kubernetes.yaml | 385 ------------------ config/recipes/beats/README.md | 39 ++ config/recipes/beats/auditbeat_hosts.yaml | 137 +++++++ .../recipes/beats/filebeat_autodiscover.yaml | 113 +++++ .../filebeat_autodiscover_by_metadata.yaml | 115 ++++++ .../beats/filebeat_no_autodiscover.yaml | 73 ++++ .../recipes/beats/heartbeat_es_kb_health.yaml | 48 +++ config/recipes/beats/journalbeat_hosts.yaml | 70 ++++ config/recipes/beats/metricbeat_hosts.yaml | 188 +++++++++ config/recipes/beats/packetbeat_dns_http.yaml | 63 +++ test/e2e/beat/config_test.go | 6 +- test/e2e/beat/recipes_test.go | 197 +++++++++ test/e2e/beat/setup_test.go | 3 + test/e2e/samples_test.go | 15 +- test/e2e/test/beat/builder.go | 24 +- test/e2e/test/beat/checks.go | 30 +- test/e2e/test/helper/yaml.go | 239 ++++++++++- test/e2e/test/run.go | 37 ++ 22 files changed, 1362 insertions(+), 622 deletions(-) delete mode 100644 config/recipes/beats/0_ns.yaml delete mode 100644 config/recipes/beats/1_monitor.yaml delete mode 100644 config/recipes/beats/2_filebeat-kubernetes.yaml delete mode 100644 config/recipes/beats/3_metricbeat-kubernetes.yaml create mode 100644 config/recipes/beats/README.md create mode 100644 config/recipes/beats/auditbeat_hosts.yaml create mode 100644 config/recipes/beats/filebeat_autodiscover.yaml create mode 100644 config/recipes/beats/filebeat_autodiscover_by_metadata.yaml create mode 100644 config/recipes/beats/filebeat_no_autodiscover.yaml create mode 100644 config/recipes/beats/heartbeat_es_kb_health.yaml create mode 100644 config/recipes/beats/journalbeat_hosts.yaml create mode 100644 config/recipes/beats/metricbeat_hosts.yaml create mode 100644 config/recipes/beats/packetbeat_dns_http.yaml create mode 100644 test/e2e/beat/recipes_test.go diff --git a/config/dev/elastic-psp.yaml b/config/dev/elastic-psp.yaml index 85c5e4cfd19..6a923b8de59 100644 --- a/config/dev/elastic-psp.yaml +++ b/config/dev/elastic-psp.yaml @@ -227,7 +227,7 @@ - SETPCAP - AUDIT_WRITE - NET_BIND_SERVICE - hostNetwork: false + hostNetwork: true hostIPC: false hostPID: false runAsUser: diff --git a/config/recipes/beats/0_ns.yaml b/config/recipes/beats/0_ns.yaml deleted file mode 100644 index 350ad314cff..00000000000 --- a/config/recipes/beats/0_ns.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: beats \ No newline at end of file diff --git a/config/recipes/beats/1_monitor.yaml b/config/recipes/beats/1_monitor.yaml deleted file mode 100644 index 5daaec54f08..00000000000 --- a/config/recipes/beats/1_monitor.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# This sample sets up an Elasticsearch cluster and a Kibana instance preconfigured for that cluster. -# The hints based autodiscover annotations for Filebeat are managed by ECK. -apiVersion: elasticsearch.k8s.elastic.co/v1 -kind: Elasticsearch -metadata: - name: monitor - namespace: beats -spec: - version: 7.8.0 - nodeSets: - - name: mdi - count: 3 - config: - node.store.allow_mmap: false - volumeClaimTemplates: - - metadata: - name: elasticsearch-data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Gi ---- -apiVersion: kibana.k8s.elastic.co/v1 -kind: Kibana -metadata: - name: monitor - namespace: beats -spec: - version: 7.8.0 - count: 1 - elasticsearchRef: - name: "monitor" - http: - service: - spec: - type: LoadBalancer \ No newline at end of file diff --git a/config/recipes/beats/2_filebeat-kubernetes.yaml b/config/recipes/beats/2_filebeat-kubernetes.yaml deleted file mode 100644 index e0ac2198f22..00000000000 --- a/config/recipes/beats/2_filebeat-kubernetes.yaml +++ /dev/null @@ -1,158 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: filebeat-config - namespace: beats - labels: - k8s-app: filebeat -data: - filebeat.yml: |- - filebeat.autodiscover: - providers: - - type: kubernetes - host: ${NODE_NAME} - hints.enabled: true - hints.default_config: - type: container - paths: - - /var/log/containers/*${data.kubernetes.container.id}.log - - processors: - - add_cloud_metadata: - - add_host_metadata: - - output.elasticsearch: - hosts: ['https://${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] - username: ${ELASTICSEARCH_USERNAME} - password: ${ELASTICSEARCH_PASSWORD} - ssl.certificate_authorities: - - /mnt/elastic/tls.crt ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: filebeat - namespace: beats - labels: - k8s-app: filebeat -spec: - selector: - matchLabels: - k8s-app: filebeat - template: - metadata: - labels: - k8s-app: filebeat - spec: - serviceAccountName: filebeat - terminationGracePeriodSeconds: 30 - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: filebeat - image: docker.elastic.co/beats/filebeat:7.8.0 - args: [ - "-c", "/etc/filebeat.yml", - "-e", - ] - env: - - name: ELASTICSEARCH_HOST - value: monitor-es-http - - name: ELASTICSEARCH_PORT - value: "9200" - - name: ELASTICSEARCH_USERNAME - value: elastic - - name: ELASTICSEARCH_PASSWORD - valueFrom: - secretKeyRef: - key: elastic - name: monitor-es-elastic-user - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - runAsUser: 0 - # If using Red Hat OpenShift uncomment this: - #privileged: true - resources: - limits: - memory: 200Mi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - name: config - mountPath: /etc/filebeat.yml - readOnly: true - subPath: filebeat.yml - - name: data - mountPath: /usr/share/filebeat/data - - name: varlibdockercontainers - mountPath: /var/lib/docker/containers - readOnly: true - - name: varlog - mountPath: /var/log - readOnly: true - - name: es-certs - mountPath: /mnt/elastic/tls.crt - readOnly: true - subPath: tls.crt - volumes: - - name: config - configMap: - defaultMode: 0600 - name: filebeat-config - - name: varlibdockercontainers - hostPath: - path: /var/lib/docker/containers - - name: varlog - hostPath: - path: /var/log - # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart - - name: data - hostPath: - path: /var/lib/filebeat-data - type: DirectoryOrCreate - - name: es-certs - secret: - secretName: monitor-es-http-certs-public ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: filebeat -subjects: -- kind: ServiceAccount - name: filebeat - namespace: beats -roleRef: - kind: ClusterRole - name: filebeat - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: filebeat - labels: - k8s-app: filebeat -rules: -- apiGroups: [""] # "" indicates the core API group - resources: - - namespaces - - pods - verbs: - - get - - watch - - list ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: filebeat - namespace: beats - labels: - k8s-app: filebeat ---- diff --git a/config/recipes/beats/3_metricbeat-kubernetes.yaml b/config/recipes/beats/3_metricbeat-kubernetes.yaml deleted file mode 100644 index 52399e912a5..00000000000 --- a/config/recipes/beats/3_metricbeat-kubernetes.yaml +++ /dev/null @@ -1,385 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: metricbeat-daemonset-config - namespace: beats - labels: - k8s-app: metricbeat -data: - metricbeat.yml: |- - metricbeat.config.modules: - # Mounted `metricbeat-daemonset-modules` configmap: - path: ${path.config}/modules.d/*.yml - # Reload module configs as they change: - reload.enabled: false - - # To enable hints based autodiscover uncomment this: - metricbeat.autodiscover: - providers: - - type: kubernetes - host: ${NODE_NAME} - hints.enabled: true - - processors: - - add_cloud_metadata: - - output.elasticsearch: - hosts: ['https://${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] - username: ${ELASTICSEARCH_USERNAME} - password: ${ELASTICSEARCH_PASSWORD} - ssl.certificate_authorities: - - /mnt/elastic/tls.crt ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: metricbeat-daemonset-modules - namespace: beats - labels: - k8s-app: metricbeat -data: - system.yml: |- - - module: system - period: 10s - metricsets: - - cpu - - load - - memory - - network - - process - - process_summary - #- core - #- diskio - #- socket - processes: ['.*'] - process.include_top_n: - by_cpu: 5 # include top 5 processes by CPU - by_memory: 5 # include top 5 processes by memory - - - module: system - period: 1m - metricsets: - - filesystem - - fsstat - processors: - - drop_event.when.regexp: - system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' - kubernetes.yml: |- - - module: kubernetes - metricsets: - - node - - system - - pod - - container - - volume - period: 10s - host: ${NODE_NAME} - hosts: ["https://${HOSTNAME}:10250"] - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - ssl.verification_mode: "none" - # If using Red Hat OpenShift remove ssl.verification_mode entry and - # uncomment these settings: - #ssl.certificate_authorities: - #- /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - - module: kubernetes - metricsets: - - proxy - period: 10s - host: ${NODE_NAME} - hosts: ["localhost:10249"] ---- -# Deploy a Metricbeat instance per node for node metrics retrieval -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: metricbeat - namespace: beats - labels: - k8s-app: metricbeat -spec: - selector: - matchLabels: - k8s-app: metricbeat - template: - metadata: - labels: - k8s-app: metricbeat - spec: - serviceAccountName: metricbeat - terminationGracePeriodSeconds: 30 - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: metricbeat - image: docker.elastic.co/beats/metricbeat:7.8.0 - args: [ - "-c", "/etc/metricbeat.yml", - "-e", - "-system.hostfs=/hostfs", - "-d", "autodiscover", - "-d", "kubernetes", - ] - env: - - name: ELASTICSEARCH_HOST - value: monitor-es-http - - name: ELASTICSEARCH_PORT - value: "9200" - - name: ELASTICSEARCH_USERNAME - value: elastic - - name: ELASTICSEARCH_PASSWORD - valueFrom: - secretKeyRef: - key: elastic - name: monitor-es-elastic-user - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - runAsUser: 0 - resources: - limits: - memory: 200Mi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - name: config - mountPath: /etc/metricbeat.yml - readOnly: true - subPath: metricbeat.yml - - name: modules - mountPath: /usr/share/metricbeat/modules.d - readOnly: true - - name: dockersock - mountPath: /var/run/docker.sock - - name: proc - mountPath: /hostfs/proc - readOnly: true - - name: cgroup - mountPath: /hostfs/sys/fs/cgroup - readOnly: true - - name: es-certs - mountPath: /mnt/elastic/tls.crt - readOnly: true - subPath: tls.crt - volumes: - - name: proc - hostPath: - path: /proc - - name: cgroup - hostPath: - path: /sys/fs/cgroup - - name: dockersock - hostPath: - path: /var/run/docker.sock - - name: config - configMap: - defaultMode: 0600 - name: metricbeat-daemonset-config - - name: modules - configMap: - defaultMode: 0600 - name: metricbeat-daemonset-modules - - name: es-certs - secret: - secretName: monitor-es-http-certs-public ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: metricbeat-deployment-config - namespace: beats - labels: - k8s-app: metricbeat -data: - metricbeat.yml: |- - metricbeat.config.modules: - # Mounted `metricbeat-daemonset-modules` configmap: - path: ${path.config}/modules.d/*.yml - # Reload module configs as they change: - reload.enabled: false - - processors: - - add_cloud_metadata: - - setup.dashboards.enabled: true - - setup.kibana: - host: "https://${KIBANA_HOST:kibana}:${KIBANA_PORT:5601}" - ssl.enabled: true - ssl.certificate_authorities: - - /mnt/kibana/ca.crt - - output.elasticsearch: - hosts: ['https://${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] - username: ${ELASTICSEARCH_USERNAME} - password: ${ELASTICSEARCH_PASSWORD} - ssl.certificate_authorities: - - /mnt/elastic/tls.crt - ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: metricbeat-deployment-modules - namespace: beats - labels: - k8s-app: metricbeat -data: - # This module requires `kube-state-metrics` up and running under `kube-system` namespace - kubernetes.yml: |- - - module: kubernetes - metricsets: - - state_node - - state_deployment - - state_replicaset - - state_pod - - state_container - # Uncomment this to get k8s events: - #- event - period: 10s - host: ${NODE_NAME} - hosts: ["kube-state-metrics.kube-system:8080"] ---- -# Deploy singleton instance in the whole cluster for some unique data sources, like kube-state-metrics -apiVersion: apps/v1 -kind: Deployment -metadata: - name: metricbeat - namespace: beats - labels: - k8s-app: metricbeat -spec: - selector: - matchLabels: - k8s-app: metricbeat - template: - metadata: - labels: - k8s-app: metricbeat - spec: - serviceAccountName: metricbeat - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: metricbeat - image: docker.elastic.co/beats/metricbeat:7.8.0 - args: [ - "-c", "/etc/metricbeat.yml", - "-e", - "-d", "autodiscover", - ] - env: - - name: ELASTICSEARCH_HOST - value: monitor-es-http - - name: ELASTICSEARCH_PORT - value: "9200" - - name: ELASTICSEARCH_USERNAME - value: elastic - - name: ELASTICSEARCH_PASSWORD - valueFrom: - secretKeyRef: - key: elastic - name: monitor-es-elastic-user - - name: KIBANA_HOST - value: monitor-kb-http - - name: KIBANA_PORT - value: "5601" - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - runAsUser: 0 - resources: - limits: - memory: 200Mi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - name: config - mountPath: /etc/metricbeat.yml - readOnly: true - subPath: metricbeat.yml - - name: modules - mountPath: /usr/share/metricbeat/modules.d - readOnly: true - - name: es-certs - mountPath: /mnt/elastic/tls.crt - readOnly: true - subPath: tls.crt - - name: kb-certs - mountPath: /mnt/kibana/ca.crt - readOnly: true - subPath: ca.crt - volumes: - - name: config - configMap: - defaultMode: 0600 - name: metricbeat-deployment-config - - name: modules - configMap: - defaultMode: 0600 - name: metricbeat-deployment-modules - - name: es-certs - secret: - secretName: monitor-es-http-certs-public - - name: kb-certs - secret: - secretName: monitor-kb-http-certs-public ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: metricbeat -subjects: -- kind: ServiceAccount - name: metricbeat - namespace: beats -roleRef: - kind: ClusterRole - name: metricbeat - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: metricbeat - labels: - k8s-app: metricbeat -rules: -- apiGroups: [""] - resources: - - nodes - - namespaces - - events - - pods - verbs: ["get", "list", "watch"] -- apiGroups: ["extensions"] - resources: - - replicasets - verbs: ["get", "list", "watch"] -- apiGroups: ["apps"] - resources: - - statefulsets - - deployments - - replicasets - verbs: ["get", "list", "watch"] -- apiGroups: - - "" - resources: - - nodes/stats - verbs: - - get ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: metricbeat - namespace: beats - labels: - k8s-app: metricbeat ---- diff --git a/config/recipes/beats/README.md b/config/recipes/beats/README.md new file mode 100644 index 00000000000..8d35ecf76e1 --- /dev/null +++ b/config/recipes/beats/README.md @@ -0,0 +1,39 @@ +# Beats Configuration Examples + +This directory contains yaml manifests with example configurations for Beats. These manifests are self-contained and work out-of-the-box on any non-secured Kubernetes cluster. All of them contain three-node Elasticsearch cluster and single Kibana instance. All Beat configurations set up Kibana dashboards if they are available for a given Beat and all required RBAC resources. + + +#### Metricbeat for Kubernetes monitoring - `metricbeat_hosts.yaml` + +Deploys Metricbeat as a DaemonSet that monitors the host resource usage (cpu, memory, network, filesystem) and Kubernetes resources (Nodes, Pods, Containers, Volumes). + +#### Filebeat with autodiscover - `filebeat_autodiscover.yaml` + +Deploys Filebeat as DaemonSet with autodiscover feature enabled. All pods in all namespaces will have logs shipped to an Elasticsearch cluster. + +#### Filebeat with autodiscover for metadata - `filebeat_autodiscover_by_metadata.yaml` + +Deploys Filebeat as a DaemonSet with autodiscover feature enabled. Fullfilling any of the two conditions below will cause a given Pod's logs to be shipped to an Elasticsearch cluster: + +- Pod is in `log-namespace` namespace +- Pod has `log-label: "true"` label + +#### Filebeat without autodiscover - `filebeat_no_autodiscover.yaml` + +Deploys Filebeat as a DaemonSet with autodiscover feature disabled. Uses entire logs directory on the host as the input. Doesn't require any RBAC resources as no Kubernetes APIs are used. + +#### Heartbeat monitoring Elasticsearch and Kibana health - `heartbeat_es_kb_heatlh.yaml` + +Deploys Heartbeat as a single Pod deployment that monitors the health of Elasticsearch and Kibana by TCP probing their Service endpoints. Note that Heartbeat expects that Elasticsearch and Kibana are deployed in the `default` namespace. + +#### Auditbeat - `auditbeat_hosts.yaml` + +Deploys Auditbeat as a DaemonSet that checks file integrity and audits file operations on the host system. + +#### Journalbeat - `journalbeat_hosts.yaml` + +Deploys Journalbeat as a DaemonSet that ships data from systemd journals. + +#### Packetbeat monitoring DNS and HTTP traffic - `packetbeat_dns_http.yaml` + +Deploys Packetbeat as a DaemonSet that monitors DNS on port `53` and HTTP(S) traffic on ports `80`, `8000`, `8080` and `9200`. diff --git a/config/recipes/beats/auditbeat_hosts.yaml b/config/recipes/beats/auditbeat_hosts.yaml new file mode 100644 index 00000000000..32bfcf32139 --- /dev/null +++ b/config/recipes/beats/auditbeat_hosts.yaml @@ -0,0 +1,137 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: auditbeat +spec: + type: auditbeat + version: 7.8.0 + elasticsearchRef: + name: elasticsearch + kibanaRef: + name: kibana + config: + auditbeat.modules: + - module: file_integrity + paths: + - /hostfs/bin + - /hostfs/usr/bin + - /hostfs/sbin + - /hostfs/usr/sbin + - /hostfs/etc + exclude_files: + - '(?i)\.sw[nop]$' + - '~$' + - '/\.git($|/)' + scan_at_start: true + scan_rate_per_sec: 50 MiB + max_file_size: 100 MiB + hash_types: [sha1] + recursive: true + - module: auditd + audit_rules: | + # Executions + -a always,exit -F arch=b64 -S execve,execveat -k exec + + # Unauthorized access attempts + -a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EACCES -k access + -a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access + + processors: + - add_cloud_metadata: {} + - add_host_metadata: {} + - add_process_metadata: + match_pids: ['process.pid'] + daemonSet: + podTemplate: + spec: + hostPID: true # Required by auditd module + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true # Allows to provide richer host metadata + automountServiceAccountToken: true # some older Beat versions are depending on this settings presence in k8s context + securityContext: + runAsUser: 0 + volumes: + - name: bin + hostPath: + path: /bin + - name: usrbin + hostPath: + path: /usr/bin + - name: sbin + hostPath: + path: /sbin + - name: usrsbin + hostPath: + path: /usr/sbin + - name: etc + hostPath: + path: /etc + - name: run-containerd + hostPath: + path: /run/containerd + type: DirectoryOrCreate + # Uncomment the below when running on GKE. See https://github.com/elastic/beats/issues/8523 for more context. + #- name: run + # hostPath: + # path: /run + #initContainers: + #- name: cos-init + # image: docker.elastic.co/beats/auditbeat:7.8.0 + # volumeMounts: + # - name: run + # mountPath: /run + # command: ['sh', '-c', 'export SYSTEMD_IGNORE_CHROOT=1 && systemctl stop systemd-journald-audit.socket && systemctl mask systemd-journald-audit.socket && systemctl restart systemd-journald'] + containers: + - name: auditbeat + securityContext: + capabilities: + add: + # Capabilities needed for auditd module + - 'AUDIT_READ' + - 'AUDIT_WRITE' + - 'AUDIT_CONTROL' + volumeMounts: + - name: bin + mountPath: /hostfs/bin + readOnly: true + - name: sbin + mountPath: /hostfs/sbin + readOnly: true + - name: usrbin + mountPath: /hostfs/usr/bin + readOnly: true + - name: usrsbin + mountPath: /hostfs/usr/sbin + readOnly: true + - name: etc + mountPath: /hostfs/etc + readOnly: true + # Directory with root filesystems of containers executed with containerd, this can be + # different with other runtimes. This volume is needed to monitor the file integrity + # of files in containers. + - name: run-containerd + mountPath: /run/containerd + readOnly: true +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: elasticsearch +spec: + version: 7.8.0 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana +spec: + version: 7.8.0 + count: 1 + elasticsearchRef: + name: elasticsearch +... diff --git a/config/recipes/beats/filebeat_autodiscover.yaml b/config/recipes/beats/filebeat_autodiscover.yaml new file mode 100644 index 00000000000..86ec4cc92d3 --- /dev/null +++ b/config/recipes/beats/filebeat_autodiscover.yaml @@ -0,0 +1,113 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: filebeat +spec: + type: filebeat + version: 7.8.0 + elasticsearchRef: + name: elasticsearch + kibanaRef: + name: kibana + config: + filebeat: + autodiscover: + providers: + - type: kubernetes + host: ${HOSTNAME} + hints: + enabled: true + default_config: + type: container + paths: + - /var/log/containers/*${data.kubernetes.container.id}.log + processors: + - add_cloud_metadata: {} + - add_host_metadata: {} + daemonSet: + podTemplate: + spec: + serviceAccountName: filebeat + automountServiceAccountToken: true + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true # Allows to provide richer host metadata + securityContext: + runAsUser: 0 + # If using Red Hat OpenShift uncomment this: + #privileged: true + containers: + - name: filebeat + volumeMounts: + - name: varlogcontainers + mountPath: /var/log/containers + - name: varlogpods + mountPath: /var/log/pods + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + volumes: + - name: varlogcontainers + hostPath: + path: /var/log/containers + - name: varlogpods + hostPath: + path: /var/log/pods + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + verbs: + - get + - watch + - list +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: filebeat +subjects: +- kind: ServiceAccount + name: filebeat + namespace: default +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: elasticsearch +spec: + version: 7.8.0 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana +spec: + version: 7.8.0 + count: 1 + elasticsearchRef: + name: elasticsearch +... diff --git a/config/recipes/beats/filebeat_autodiscover_by_metadata.yaml b/config/recipes/beats/filebeat_autodiscover_by_metadata.yaml new file mode 100644 index 00000000000..8709c001693 --- /dev/null +++ b/config/recipes/beats/filebeat_autodiscover_by_metadata.yaml @@ -0,0 +1,115 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: filebeat +spec: + type: filebeat + version: 7.8.0 + elasticsearchRef: + name: elasticsearch + kibanaRef: + name: kibana + config: + filebeat.autodiscover.providers: + - node: ${HOSTNAME} + type: kubernetes + hints.default_config.enabled: "false" + templates: + - condition.equals.kubernetes.namespace: log-namespace + config: + - paths: ["/var/log/containers/*${data.kubernetes.container.id}.log"] + type: container + - condition.equals.kubernetes.labels.log-label: "true" + config: + - paths: ["/var/log/containers/*${data.kubernetes.container.id}.log"] + type: container + processors: + - add_cloud_metadata: {} + - add_host_metadata: {} + daemonSet: + podTemplate: + spec: + serviceAccountName: filebeat + automountServiceAccountToken: true + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true # Allows to provide richer host metadata + securityContext: + runAsUser: 0 + # If using Red Hat OpenShift uncomment this: + #privileged: true + containers: + - name: filebeat + volumeMounts: + - name: varlogcontainers + mountPath: /var/log/containers + - name: varlogpods + mountPath: /var/log/pods + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + volumes: + - name: varlogcontainers + hostPath: + path: /var/log/containers + - name: varlogpods + hostPath: + path: /var/log/pods + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + verbs: + - get + - watch + - list +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: filebeat +subjects: +- kind: ServiceAccount + name: filebeat + namespace: default +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: elasticsearch +spec: + version: 7.8.0 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana +spec: + version: 7.8.0 + count: 1 + elasticsearchRef: + name: elasticsearch +... diff --git a/config/recipes/beats/filebeat_no_autodiscover.yaml b/config/recipes/beats/filebeat_no_autodiscover.yaml new file mode 100644 index 00000000000..736925613b1 --- /dev/null +++ b/config/recipes/beats/filebeat_no_autodiscover.yaml @@ -0,0 +1,73 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: filebeat +spec: + type: filebeat + version: 7.8.0 + elasticsearchRef: + name: elasticsearch + kibanaRef: + name: kibana + config: + filebeat.inputs: + - type: container + paths: + - /var/log/containers/*.log + processors: + - add_host_metadata: {} + - add_cloud_metadata: {} + daemonSet: + podTemplate: + spec: + automountServiceAccountToken: true + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true # Allows to provide richer host metadata + securityContext: + runAsUser: 0 + containers: + - name: filebeat + # If using Red Hat OpenShift uncomment this: + #securityContext: + #privileged: true + volumeMounts: + - name: varlogcontainers + mountPath: /var/log/containers + - name: varlogpods + mountPath: /var/log/pods + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + volumes: + - name: varlogcontainers + hostPath: + path: /var/log/containers + - name: varlogpods + hostPath: + path: /var/log/pods + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: elasticsearch +spec: + version: 7.8.0 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana +spec: + version: 7.8.0 + count: 1 + elasticsearchRef: + name: elasticsearch +... diff --git a/config/recipes/beats/heartbeat_es_kb_health.yaml b/config/recipes/beats/heartbeat_es_kb_health.yaml new file mode 100644 index 00000000000..98d236f8fe4 --- /dev/null +++ b/config/recipes/beats/heartbeat_es_kb_health.yaml @@ -0,0 +1,48 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: heartbeat +spec: + type: heartbeat + version: 7.8.0 + elasticsearchRef: + name: elasticsearch + kibanaRef: + name: kibana + config: + heartbeat.monitors: + - type: tcp + schedule: '@every 5s' + hosts: ["elasticsearch-es-http.default.svc:9200"] + - type: tcp + schedule: '@every 5s' + hosts: ["kibana-kb-http.default.svc:5601"] + deployment: + replicas: 1 + podTemplate: + spec: + securityContext: + runAsUser: 0 +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: elasticsearch +spec: + version: 7.8.0 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana +spec: + version: 7.8.0 + count: 1 + elasticsearchRef: + name: elasticsearch +... diff --git a/config/recipes/beats/journalbeat_hosts.yaml b/config/recipes/beats/journalbeat_hosts.yaml new file mode 100644 index 00000000000..b0e646f4d4c --- /dev/null +++ b/config/recipes/beats/journalbeat_hosts.yaml @@ -0,0 +1,70 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: journalbeat +spec: + type: journalbeat + version: 7.8.0 + elasticsearchRef: + name: elasticsearch + kibanaRef: + name: kibana + config: + journalbeat.inputs: + - paths: [] + seek: cursor + cursor_seek_fallback: tail + processors: + - add_cloud_metadata: {} + - add_host_metadata: {} + daemonSet: + podTemplate: + spec: + automountServiceAccountToken: true # some older Beat versions are depending on this settings presence in k8s context + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: journalbeat + volumeMounts: + - mountPath: /var/log/journal + name: var-journal + - mountPath: /run/log/journal + name: run-journal + - mountPath: /etc/machine-id + name: machine-id + hostNetwork: true # Allows to provide richer host metadata + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 30 + volumes: + - hostPath: + path: /var/log/journal + name: var-journal + - hostPath: + path: /run/log/journal + name: run-journal + - hostPath: + path: /etc/machine-id + name: machine-id +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: elasticsearch +spec: + version: 7.8.0 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana +spec: + version: 7.8.0 + count: 1 + elasticsearchRef: + name: elasticsearch +... diff --git a/config/recipes/beats/metricbeat_hosts.yaml b/config/recipes/beats/metricbeat_hosts.yaml new file mode 100644 index 00000000000..06bc0ad5d36 --- /dev/null +++ b/config/recipes/beats/metricbeat_hosts.yaml @@ -0,0 +1,188 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: metricbeat +spec: + type: metricbeat + version: 7.8.0 + elasticsearchRef: + name: elasticsearch + kibanaRef: + name: kibana + config: + metricbeat: + autodiscover: + providers: + - hints: + default_config: {} + enabled: "true" + host: ${HOSTNAME} + type: kubernetes + modules: + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + process: + include_top_n: + by_cpu: 5 + by_memory: 5 + processes: + - .* + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event: + when: + regexp: + system: + filesystem: + mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib)($|/) + - module: kubernetes + period: 10s + host: ${HOSTNAME} + hosts: + - https://${HOSTNAME}:10250 + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl: + verification_mode: none + metricsets: + - node + - system + - pod + - container + - volume + processors: + - add_cloud_metadata: {} + - add_host_metadata: {} + daemonSet: + podTemplate: + spec: + serviceAccountName: metricbeat + automountServiceAccountToken: true # some older Beat versions are depending on this settings presence in k8s context + containers: + - args: + - -e + - -c + - /etc/beat.yml + - -system.hostfs=/hostfs + name: metricbeat + volumeMounts: + - mountPath: /hostfs/sys/fs/cgroup + name: cgroup + - mountPath: /var/run/docker.sock + name: dockersock + - mountPath: /hostfs/proc + name: proc + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true # Allows to provide richer host metadata + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 30 + volumes: + - hostPath: + path: /sys/fs/cgroup + name: cgroup + - hostPath: + path: /var/run/docker.sock + name: dockersock + - hostPath: + path: /proc + name: proc +--- +# permissions needed for metricbeat +# source: https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-module-kubernetes.html +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metricbeat +rules: +- apiGroups: + - "" + resources: + - nodes + - namespaces + - events + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "extensions" + resources: + - replicasets + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - statefulsets + - deployments + - replicasets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metricbeat + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metricbeat +subjects: +- kind: ServiceAccount + name: metricbeat + namespace: default +roleRef: + kind: ClusterRole + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: elasticsearch +spec: + version: 7.8.0 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana +spec: + version: 7.8.0 + count: 1 + elasticsearchRef: + name: elasticsearch +... diff --git a/config/recipes/beats/packetbeat_dns_http.yaml b/config/recipes/beats/packetbeat_dns_http.yaml new file mode 100644 index 00000000000..171352ab305 --- /dev/null +++ b/config/recipes/beats/packetbeat_dns_http.yaml @@ -0,0 +1,63 @@ +apiVersion: beat.k8s.elastic.co/v1beta1 +kind: Beat +metadata: + name: packetbeat +spec: + type: packetbeat + version: 7.8.0 + elasticsearchRef: + name: elasticsearch + kibanaRef: + name: kibana + config: + packetbeat.interfaces.device: any + packetbeat.protocols: + - type: dns + ports: [53] + include_authorities: true + include_additionals: true + - type: http + ports: [80, 8000, 8080, 9200] + packetbeat.flows: + timeout: 30s + period: 10s + processors: + - add_cloud_metadata: {} + - add_host_metadata: {} + daemonSet: + podTemplate: + spec: + terminationGracePeriodSeconds: 30 + hostNetwork: true + automountServiceAccountToken: true # some older Beat versions are depending on this settings presence in k8s context + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: packetbeat + securityContext: + runAsUser: 0 + capabilities: + add: + - NET_ADMIN +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: elasticsearch +spec: + version: 7.8.0 + nodeSets: + - name: default + count: 3 + config: + node.store.allow_mmap: false +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: kibana +spec: + version: 7.8.0 + count: 1 + elasticsearchRef: + name: elasticsearch +... diff --git a/test/e2e/beat/config_test.go b/test/e2e/beat/config_test.go index c10b76fbfe8..263f76cc634 100644 --- a/test/e2e/beat/config_test.go +++ b/test/e2e/beat/config_test.go @@ -37,6 +37,7 @@ func TestFilebeatDefaultConfig(t *testing.T) { testPodBuilder := beat.NewPodBuilder(name) fbBuilder := beat.NewBuilder(name). + WithRoles(beat.PSPClusterRoleName, beat.AutodiscoverClusterRoleName). WithType(filebeat.Type). WithElasticsearchRef(esBuilder.Ref()). WithESValidations( @@ -59,7 +60,7 @@ func TestMetricbeatDefaultConfig(t *testing.T) { mbBuilder := beat.NewBuilder(name). WithType(metricbeat.Type). - WithRoles(beat.MetricbeatClusterRoleName). + WithRoles(beat.MetricbeatClusterRoleName, beat.PSPClusterRoleName, beat.AutodiscoverClusterRoleName). WithElasticsearchRef(esBuilder.Ref()). WithESValidations( beat.HasEventFromBeat(metricbeat.Type), @@ -85,6 +86,7 @@ func TestHeartbeatConfig(t *testing.T) { hbBuilder := beat.NewBuilder(name). WithType(heartbeat.Type). + WithRoles(beat.PSPClusterRoleName). WithDeployment(). WithElasticsearchRef(esBuilder.Ref()). WithESValidations( @@ -120,6 +122,7 @@ func TestBeatSecureSettings(t *testing.T) { fbBuilder := beat.NewBuilder(name). WithType(filebeat.Type). + WithRoles(beat.PSPClusterRoleName, beat.AutodiscoverClusterRoleName). WithElasticsearchRef(esBuilder.Ref()). WithSecureSettings(secretName). WithObjects(secret). @@ -191,6 +194,7 @@ processors: fbBuilder := beat.NewBuilder(name). WithType(filebeat.Type). + WithRoles(beat.PSPClusterRoleName, beat.AutodiscoverClusterRoleName). WithElasticsearchRef(esBuilder.Ref()). WithConfigRef(secretName). WithObjects(secret). diff --git a/test/e2e/beat/recipes_test.go b/test/e2e/beat/recipes_test.go new file mode 100644 index 00000000000..cc40a6a0fe4 --- /dev/null +++ b/test/e2e/beat/recipes_test.go @@ -0,0 +1,197 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package beat + +import ( + "fmt" + "path" + "strings" + "testing" + + commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" + esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" + beatcommon "github.com/elastic/cloud-on-k8s/pkg/controller/beat/common" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/settings" + "github.com/elastic/cloud-on-k8s/pkg/controller/kibana" + "github.com/elastic/cloud-on-k8s/test/e2e/test" + "github.com/elastic/cloud-on-k8s/test/e2e/test/beat" + "github.com/elastic/cloud-on-k8s/test/e2e/test/helper" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/rand" +) + +func TestFilebeatNoAutodiscoverRecipe(t *testing.T) { + name := "fb-no-autodiscover" + pod, loggedString := loggingTestPod(name) + customize := func(builder beat.Builder) beat.Builder { + return builder. + WithRoles(beat.PSPClusterRoleName). + WithESValidations( + beat.HasMessageContaining(loggedString), + ) + } + + runBeatRecipe(t, "filebeat_no_autodiscover.yaml", customize, pod) +} + +func TestFilebeatAutodiscoverRecipe(t *testing.T) { + name := "fb-autodiscover" + pod, loggedString := loggingTestPod(name) + customize := func(builder beat.Builder) beat.Builder { + return builder. + WithRoles(beat.PSPClusterRoleName). + WithESValidations( + beat.HasEventFromPod(pod.Name), + beat.HasMessageContaining(loggedString), + ) + } + + runBeatRecipe(t, "filebeat_autodiscover.yaml", customize, pod) +} + +func TestFilebeatAutodiscoverByMetadataRecipe(t *testing.T) { + name := "fb-autodiscover-meta" + podBad, badLog := loggingTestPod(name + "-bad") + podLabel, goodLog := loggingTestPod(name + "-label") + podLabel.Labels["log-label"] = "true" + + customize := func(builder beat.Builder) beat.Builder { + return builder. + WithRoles(beat.PSPClusterRoleName, beat.AutodiscoverClusterRoleName). + WithESValidations( + beat.HasEventFromPod(podLabel.Name), + beat.HasMessageContaining(goodLog), + beat.NoMessageContaining(badLog), + ) + } + + runBeatRecipe(t, "filebeat_autodiscover_by_metadata.yaml", customize, podLabel, podBad) +} + +func TestMetricbeatHostsRecipe(t *testing.T) { + customize := func(builder beat.Builder) beat.Builder { + return builder. + WithRoles(beat.PSPClusterRoleName). + WithESValidations( + beat.HasEvent("event.dataset:system.cpu"), + beat.HasEvent("event.dataset:system.load"), + beat.HasEvent("event.dataset:system.memory"), + beat.HasEvent("event.dataset:system.network"), + beat.HasEvent("event.dataset:system.process"), + beat.HasEvent("event.dataset:system.process.summary"), + beat.HasEvent("event.dataset:system.fsstat"), + ) + } + + runBeatRecipe(t, "metricbeat_hosts.yaml", customize) +} + +func TestHeartbeatEsKbHealthRecipe(t *testing.T) { + customize := func(builder beat.Builder) beat.Builder { + cfg := settings.MustCanonicalConfig(builder.Beat.Spec.Config.Data) + yamlBytes, err := cfg.Render() + require.NoError(t, err) + + spec := builder.Beat.Spec + newEsHost := fmt.Sprintf("%s.%s.svc", esv1.HTTPService(spec.ElasticsearchRef.Name), builder.Beat.Namespace) + newKbHost := fmt.Sprintf("%s.%s.svc", kibana.HTTPService(spec.KibanaRef.Name), builder.Beat.Namespace) + + yaml := string(yamlBytes) + yaml = strings.ReplaceAll(yaml, "elasticsearch-es-http.default.svc", newEsHost) + yaml = strings.ReplaceAll(yaml, "kibana-kb-http.default.svc", newKbHost) + + builder.Beat.Spec.Config = &commonv1.Config{} + err = settings.MustParseConfig([]byte(yaml)).Unpack(&builder.Beat.Spec.Config.Data) + require.NoError(t, err) + + return builder. + WithRoles(beat.PSPClusterRoleName). + WithESValidations( + beat.HasEvent("monitor.status:up"), + ) + } + + runBeatRecipe(t, "heartbeat_es_kb_health.yaml", customize) +} + +func TestAuditbeatHostsRecipe(t *testing.T) { + if test.Ctx().Provider == "kind" { + // kind doesn't support configuring required settings + // see https://github.com/elastic/cloud-on-k8s/issues/3328 for more context + t.SkipNow() + } + + customize := func(builder beat.Builder) beat.Builder { + return builder. + WithRoles(beat.AuditbeatPSPClusterRoleName). + WithESValidations( + beat.HasEvent("event.dataset:file"), + beat.HasEvent("event.module:file_integrity"), + ) + } + + runBeatRecipe(t, "auditbeat_hosts.yaml", customize) +} + +func TestPacketbeatDnsHttpRecipe(t *testing.T) { + customize := func(builder beat.Builder) beat.Builder { + if !(test.Ctx().Provider == "kind" && test.Ctx().KubernetesVersion == "1.12") { + // there are some issues with kind 1.12 and tracking http traffic + builder = builder.WithESValidations(beat.HasEvent("event.dataset:http")) + } + + return builder. + WithRoles(beat.PacketbeatPSPClusterRoleName). + WithESValidations( + beat.HasEvent("event.dataset:flow"), + beat.HasEvent("event.dataset:dns"), + ) + } + + runBeatRecipe(t, "packetbeat_dns_http.yaml", customize) +} + +func TestJournalbeatHostsRecipe(t *testing.T) { + customize := func(builder beat.Builder) beat.Builder { + return builder. + WithRoles(beat.JournalbeatPSPClusterRoleName) + } + + runBeatRecipe(t, "journalbeat_hosts.yaml", customize) +} + +func runBeatRecipe( + t *testing.T, + fileName string, + customize func(builder beat.Builder) beat.Builder, + additionalObjects ...runtime.Object, +) { + filePath := path.Join("../../../config/recipes/beats", fileName) + namespace := test.Ctx().ManagedNamespace(0) + suffix := rand.String(4) + + transformationsWrapped := func(builder test.Builder) test.Builder { + beatBuilder, ok := builder.(beat.Builder) + if !ok { + return builder + } + + if customize != nil { + beatBuilder = customize(beatBuilder) + } + + return beatBuilder. + WithESValidations(beat.HasEventFromBeat(beatcommon.Type(beatBuilder.Beat.Spec.Type))) + } + + helper.RunFile(t, filePath, namespace, suffix, additionalObjects, transformationsWrapped) +} + +func loggingTestPod(name string) (*corev1.Pod, string) { + podBuilder := beat.NewPodBuilder(name) + return &podBuilder.Pod, podBuilder.Logged +} diff --git a/test/e2e/beat/setup_test.go b/test/e2e/beat/setup_test.go index 1d40274ac25..c6a41399950 100644 --- a/test/e2e/beat/setup_test.go +++ b/test/e2e/beat/setup_test.go @@ -50,6 +50,7 @@ func TestBeatKibanaRef(t *testing.T) { fbBuilder := beat.NewBuilder(name). WithType(filebeat.Type). + WithRoles(beat.PSPClusterRoleName, beat.AutodiscoverClusterRoleName). WithElasticsearchRef(esBuilder.Ref()). WithKibanaRef(kbBuilder.Ref()) @@ -57,6 +58,7 @@ func TestBeatKibanaRef(t *testing.T) { mbBuilder := beat.NewBuilder(name). WithType(metricbeat.Type). + WithRoles(beat.PSPClusterRoleName, beat.AutodiscoverClusterRoleName). WithElasticsearchRef(esBuilder.Ref()). WithKibanaRef(kbBuilder.Ref()). WithRoles(beat.MetricbeatClusterRoleName) @@ -65,6 +67,7 @@ func TestBeatKibanaRef(t *testing.T) { hbBuilder := beat.NewBuilder(name). WithType(heartbeat.Type). + WithRoles(beat.PSPClusterRoleName). WithDeployment(). WithElasticsearchRef(esBuilder.Ref()) diff --git a/test/e2e/samples_test.go b/test/e2e/samples_test.go index a7549e4b4c5..cdce2cf5a72 100644 --- a/test/e2e/samples_test.go +++ b/test/e2e/samples_test.go @@ -8,7 +8,6 @@ import ( "bufio" "os" "path/filepath" - "strings" "testing" commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" @@ -31,7 +30,7 @@ func TestSamples(t *testing.T) { decoder := helper.NewYAMLDecoder() for _, sample := range sampleFiles { - testName := mkTestName(t, sample) + testName := helper.MkTestName(t, sample) builders := createBuilders(t, decoder, sample, testName) t.Run(testName, func(t *testing.T) { test.Sequence(nil, test.EmptySteps, builders...).RunSequential(t) @@ -39,18 +38,6 @@ func TestSamples(t *testing.T) { } } -func mkTestName(t *testing.T, path string) string { - t.Helper() - - baseName := filepath.Base(path) - baseName = strings.TrimSuffix(baseName, ".yaml") - parentDir := filepath.Base(filepath.Dir(path)) - testName := filepath.Join(parentDir, baseName) - - // testName will be used as label, so avoid using illegal chars - return strings.ReplaceAll(testName, "/", "-") -} - func createBuilders(t *testing.T, decoder *helper.YAMLDecoder, sampleFile, testName string) []test.Builder { t.Helper() diff --git a/test/e2e/test/beat/builder.go b/test/e2e/test/beat/builder.go index f4436cd893c..002aa4ccecc 100644 --- a/test/e2e/test/beat/builder.go +++ b/test/e2e/test/beat/builder.go @@ -48,8 +48,20 @@ func (b Builder) SkipTest() bool { } -func NewBuilderWithoutSuffix(name string) Builder { - return newBuilder(name, "") +// NewBuilderFromBeat creates a Beat builder from an existing Beat config. Sets all additional Builder fields +// appropriately. +func NewBuilderFromBeat(beat *beatv1beta1.Beat) Builder { + var podTemplate *corev1.PodTemplateSpec + if beat.Spec.DaemonSet != nil { + podTemplate = &beat.Spec.DaemonSet.PodTemplate + } else if beat.Spec.Deployment != nil { + podTemplate = &beat.Spec.Deployment.PodTemplate + } + + return Builder{ + Beat: *beat, + PodTemplate: podTemplate, + } } func NewBuilder(name string) Builder { @@ -73,8 +85,7 @@ func newBuilder(name string, suffix string) Builder { }. WithSuffix(suffix). WithLabel(run.TestNameLabel, name). - WithDaemonSet(). - WithRoles(AutodiscoverClusterRoleName, PSPClusterRoleName) + WithDaemonSet() } type ValidationFunc func(client.Client) error @@ -196,9 +207,10 @@ func (b Builder) WithRoles(clusterRoleNames ...string) Builder { } func bind(b Builder, clusterRoleName string) Builder { - saName := fmt.Sprintf("%s-sa", b.Beat.Name) + saName := b.PodTemplate.Spec.ServiceAccountName - if b.PodTemplate.Spec.ServiceAccountName != saName { + if saName == "" { + saName = fmt.Sprintf("%s-sa", b.Beat.Name) b = b.WithPodTemplateServiceAccount(saName) sa := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/e2e/test/beat/checks.go b/test/e2e/test/beat/checks.go index 9ec1ed06234..cda35e9a455 100644 --- a/test/e2e/test/beat/checks.go +++ b/test/e2e/test/beat/checks.go @@ -27,11 +27,37 @@ func HasMessageContaining(message string) ValidationFunc { return HasEvent(fmt.Sprintf("message:%s", message)) } +func NoMessageContaining(message string) ValidationFunc { + return NoEvent(fmt.Sprintf("message:%s", message)) +} + func HasEvent(query string) ValidationFunc { return hasEvent(fmt.Sprintf("/*beat*/_search?q=%s", query)) } +func NoEvent(query string) ValidationFunc { + return noEvent(fmt.Sprintf("/*beat*/_search?q=%s", query)) +} + func hasEvent(url string) ValidationFunc { + return checkEvent(url, func(hitsCount int) error { + if hitsCount == 0 { + return fmt.Errorf("hit count should be more than 0 for %s", url) + } + return nil + }) +} + +func noEvent(url string) ValidationFunc { + return checkEvent(url, func(hitsCount int) error { + if hitsCount != 0 { + return fmt.Errorf("hit count should be 0 for %s", url) + } + return nil + }) +} + +func checkEvent(url string, check func(int) error) ValidationFunc { return func(esClient client.Client) error { req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { @@ -52,8 +78,8 @@ func hasEvent(url string) ValidationFunc { if err != nil { return err } - if len(results.Hits.Hits) == 0 { - return fmt.Errorf("hit count should be more than 0 for %s", url) + if err := check(len(results.Hits.Hits)); err != nil { + return err } return nil diff --git a/test/e2e/test/helper/yaml.go b/test/e2e/test/helper/yaml.go index 7d2ad0933a1..a4b9cd54dfb 100644 --- a/test/e2e/test/helper/yaml.go +++ b/test/e2e/test/helper/yaml.go @@ -8,18 +8,30 @@ import ( "bufio" "fmt" "io" + "os" + "path/filepath" + "strings" + "testing" apmv1 "github.com/elastic/cloud-on-k8s/pkg/apis/apm/v1" beatv1beta1 "github.com/elastic/cloud-on-k8s/pkg/apis/beat/v1beta1" + commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" entv1beta1 "github.com/elastic/cloud-on-k8s/pkg/apis/enterprisesearch/v1beta1" kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1" + beatcommon "github.com/elastic/cloud-on-k8s/pkg/controller/beat/common" + "github.com/elastic/cloud-on-k8s/test/e2e/cmd/run" "github.com/elastic/cloud-on-k8s/test/e2e/test" "github.com/elastic/cloud-on-k8s/test/e2e/test/apmserver" "github.com/elastic/cloud-on-k8s/test/e2e/test/beat" "github.com/elastic/cloud-on-k8s/test/e2e/test/elasticsearch" "github.com/elastic/cloud-on-k8s/test/e2e/test/enterprisesearch" "github.com/elastic/cloud-on-k8s/test/e2e/test/kibana" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + meta2 "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/yaml" @@ -39,6 +51,10 @@ func NewYAMLDecoder() *YAMLDecoder { scheme.AddKnownTypes(apmv1.GroupVersion, &apmv1.ApmServer{}, &apmv1.ApmServerList{}) scheme.AddKnownTypes(beatv1beta1.GroupVersion, &beatv1beta1.Beat{}, &beatv1beta1.BeatList{}) scheme.AddKnownTypes(entv1beta1.GroupVersion, &entv1beta1.EnterpriseSearch{}, &entv1beta1.EnterpriseSearchList{}) + + scheme.AddKnownTypes(rbacv1.SchemeGroupVersion, &rbacv1.ClusterRoleBinding{}, &rbacv1.ClusterRoleBindingList{}) + scheme.AddKnownTypes(rbacv1.SchemeGroupVersion, &rbacv1.ClusterRole{}, &rbacv1.ClusterRoleList{}) + scheme.AddKnownTypes(corev1.SchemeGroupVersion, &corev1.ServiceAccount{}, &corev1.ServiceAccountList{}) decoder := serializer.NewCodecFactory(scheme).UniversalDeserializer() return &YAMLDecoder{decoder: decoder} @@ -77,19 +93,7 @@ func (yd *YAMLDecoder) ToBuilders(reader *bufio.Reader, transform BuilderTransfo b.ApmServer = *decodedObj builder = transform(b) case *beatv1beta1.Beat: - b := beat.NewBuilderWithoutSuffix(decodedObj.Name) - b.Beat = *decodedObj - // adjust builder to compensate for overwriting Beat struct - // RBAC objects were populated using a name that doesn't have proper test suffix, - // hence clearing them here so the next calls can populate them properly - b.AdditionalObjects = nil - - // Since b.Beat was overwritten, b.PodTemplate is pointing to wrong struct, fixing it here - if b.Beat.Spec.DaemonSet != nil { - b.PodTemplate = &b.Beat.Spec.DaemonSet.PodTemplate - } else if b.Beat.Spec.Deployment != nil { - b.PodTemplate = &b.Beat.Spec.Deployment.PodTemplate - } + b := beat.NewBuilderFromBeat(decodedObj) builder = transform(b) case *entv1beta1.EnterpriseSearch: b := enterprisesearch.NewBuilderWithoutSuffix(decodedObj.Name) @@ -104,3 +108,212 @@ func (yd *YAMLDecoder) ToBuilders(reader *bufio.Reader, transform BuilderTransfo return builders, nil } + +func (yd *YAMLDecoder) ToObjects(reader *bufio.Reader) ([]runtime.Object, error) { + var objects []runtime.Object + + yamlReader := yaml.NewYAMLReader(reader) + for { + yamlBytes, err := yamlReader.Read() + if err != nil { + if err == io.EOF { + break + } + return nil, fmt.Errorf("failed to read YAML: %w", err) + } + obj, _, err := yd.decoder.Decode(yamlBytes, nil, nil) + if err != nil { + return nil, fmt.Errorf("failed to decode YAML: %w", err) + } + + objects = append(objects, obj) + } + + return objects, nil +} + +// RunFile runs the builder workflow for all known resources in a yaml file, all other objects are created before and deleted +// after. Resources will be created in a given namespace and with a given suffix. Additional objects to be created and deleted +// can be passed as well as set of optional transformations to apply to all Builders. +func RunFile( + t *testing.T, + filePath, namespace, suffix string, + additionalObjects []runtime.Object, + transformations ...BuilderTransform) { + builders, objects, err := extractFromFile(t, filePath, namespace, suffix, MkTestName(t, filePath), transformations...) + if err != nil { + panic(err) + } + + objects = append(objects, additionalObjects...) + + creates, deletes := makeObjectSteps(t, objects) + + test.BeforeAfterSequence(creates, deletes, builders...).RunSequential(t) +} + +func extractFromFile( + t *testing.T, + filePath, namespace, suffix, fullTestName string, + transformations ...BuilderTransform, +) ([]test.Builder, []runtime.Object, error) { + f, err := os.Open(filePath) + require.NoError(t, err, "Failed to open file %s", filePath) + defer f.Close() + + decoder := NewYAMLDecoder() + objects, err := decoder.ToObjects(bufio.NewReader(f)) + if err != nil { + return nil, nil, err + } + + builders, objects := transformToE2E(namespace, fullTestName, suffix, transformations, objects) + return builders, objects, nil +} + +func makeObjectSteps( + t *testing.T, + objects []runtime.Object, +) (func(k *test.K8sClient) test.StepList, func(k *test.K8sClient) test.StepList) { + return func(k *test.K8sClient) test.StepList { + steps := test.StepList{} + for i := range objects { + ii := i + meta, err := meta2.Accessor(objects[ii]) + require.NoError(t, err) + steps = steps.WithStep(test.Step{ + Name: fmt.Sprintf("Create %s/%s", meta.GetNamespace(), meta.GetNamespace()), + Test: func(t *testing.T) { + err := k.Client.Create(objects[ii]) + if !k8serrors.IsAlreadyExists(err) { + require.NoError(t, err) + } + }, + }) + } + return steps + }, func(k *test.K8sClient) test.StepList { + steps := test.StepList{} + for i := range objects { + ii := i + meta, err := meta2.Accessor(objects[ii]) + require.NoError(t, err) + steps = steps.WithStep(test.Step{ + Name: fmt.Sprintf("Delete %s/%s", meta.GetNamespace(), meta.GetNamespace()), + Test: func(t *testing.T) { + err := k.Client.Delete(objects[ii]) + if !k8serrors.IsNotFound(err) { + require.NoError(t, err) + } + }, + }) + } + return steps + } +} + +func transformToE2E(namespace, fullTestName, suffix string, transformers []BuilderTransform, objects []runtime.Object) ([]test.Builder, []runtime.Object) { + var builders []test.Builder + var otherObjects []runtime.Object + for _, object := range objects { + var builder test.Builder + switch decodedObj := object.(type) { + case *esv1.Elasticsearch: + b := elasticsearch.NewBuilderWithoutSuffix(decodedObj.Name) + b.Elasticsearch = *decodedObj + builder = b.WithNamespace(namespace). + WithSuffix(suffix). + WithRestrictedSecurityContext(). + WithLabel(run.TestNameLabel, fullTestName). + WithPodLabel(run.TestNameLabel, fullTestName) + case *kbv1.Kibana: + b := kibana.NewBuilderWithoutSuffix(decodedObj.Name) + b.Kibana = *decodedObj + builder = b.WithNamespace(namespace). + WithSuffix(suffix). + WithElasticsearchRef(tweakServiceRef(b.Kibana.Spec.ElasticsearchRef, suffix)). + WithRestrictedSecurityContext(). + WithLabel(run.TestNameLabel, fullTestName). + WithPodLabel(run.TestNameLabel, fullTestName) + case *apmv1.ApmServer: + b := apmserver.NewBuilderWithoutSuffix(decodedObj.Name) + b.ApmServer = *decodedObj + builder = b.WithNamespace(namespace). + WithSuffix(suffix). + WithElasticsearchRef(tweakServiceRef(b.ApmServer.Spec.ElasticsearchRef, suffix)). + WithKibanaRef(tweakServiceRef(b.ApmServer.Spec.KibanaRef, suffix)). + WithConfig(map[string]interface{}{"apm-server.ilm.enabled": false}). + WithRestrictedSecurityContext(). + WithLabel(run.TestNameLabel, fullTestName). + WithPodLabel(run.TestNameLabel, fullTestName) + case *beatv1beta1.Beat: + b := beat.NewBuilderFromBeat(decodedObj) + + builder = b.WithNamespace(namespace). + WithSuffix(suffix). + WithElasticsearchRef(tweakServiceRef(b.Beat.Spec.ElasticsearchRef, suffix)). + WithLabel(run.TestNameLabel, fullTestName). + WithPodLabel(run.TestNameLabel, fullTestName). + WithESValidations(beat.HasEventFromBeat(beatcommon.Type(b.Beat.Spec.Type))). + WithKibanaRef(tweakServiceRef(b.Beat.Spec.KibanaRef, suffix)) + + if b.PodTemplate.Spec.ServiceAccountName != "" { + b = b.WithPodTemplateServiceAccount(b.PodTemplate.Spec.ServiceAccountName + "-" + suffix) + } + case *entv1beta1.EnterpriseSearch: + b := enterprisesearch.NewBuilderWithoutSuffix(decodedObj.Name) + b.EnterpriseSearch = *decodedObj + builder = b.WithNamespace(namespace). + WithSuffix(suffix). + WithElasticsearchRef(tweakServiceRef(b.EnterpriseSearch.Spec.ElasticsearchRef, suffix)). + WithRestrictedSecurityContext(). + WithLabel(run.TestNameLabel, fullTestName). + WithPodLabel(run.TestNameLabel, fullTestName) + case *corev1.ServiceAccount: + decodedObj.Namespace = namespace + decodedObj.Name = decodedObj.Name + "-" + suffix + case *rbacv1.ClusterRoleBinding: + decodedObj.Subjects[0].Namespace = namespace + decodedObj.Subjects[0].Name = decodedObj.Subjects[0].Name + "-" + suffix + decodedObj.RoleRef.Name = decodedObj.RoleRef.Name + "-" + suffix + decodedObj.Name = decodedObj.Name + "-" + suffix + case *rbacv1.ClusterRole: + decodedObj.Name = decodedObj.Name + "-" + suffix + } + + if builder != nil { + // ECK driven resources can be further transformed + for _, transformer := range transformers { + builder = transformer(builder) + } + builders = append(builders, builder) + } else { + // built-in in resources are separated as they are treated differently + otherObjects = append(otherObjects, object) + } + } + + return builders, otherObjects +} + +func tweakServiceRef(ref commonv1.ObjectSelector, suffix string) commonv1.ObjectSelector { + // All the objects defined in the YAML file will have a random test suffix added to prevent clashes with previous runs. + // This necessitates changing the Elasticsearch reference to match the suffixed name. + if ref.Name != "" { + ref.Name = ref.Name + "-" + suffix + } + + return ref +} + +func MkTestName(t *testing.T, path string) string { + t.Helper() + + baseName := filepath.Base(path) + baseName = strings.TrimSuffix(baseName, ".yaml") + parentDir := filepath.Base(filepath.Dir(path)) + testName := filepath.Join(parentDir, baseName) + + // testName will be used as label, so avoid using illegal chars + return strings.ReplaceAll(testName, "/", "-") +} diff --git a/test/e2e/test/run.go b/test/e2e/test/run.go index cc168c8124a..70ccba0f7c0 100644 --- a/test/e2e/test/run.go +++ b/test/e2e/test/run.go @@ -40,3 +40,40 @@ func Sequence(before StepsFunc, f StepsFunc, builders ...Builder) StepList { return steps } + +// BeforeAfterSequence returns a list of steps corresponding to a workflow that allows defining a list of steps to execute +// before and after builder workflow (before steps, init, create, checks, deletes, after steps) +func BeforeAfterSequence(before StepsFunc, after StepsFunc, builders ...Builder) StepList { + steps := StepList{} + for _, b := range builders { + // ignore the test if some builders cannot be tested + if b.SkipTest() { + return steps + } + } + + k := NewK8sClientOrFatal() + + if before != nil { + steps = steps.WithSteps(before(k)) + } + + for _, b := range builders { + steps = steps.WithSteps(b.InitTestSteps(k)) + } + for _, b := range builders { + steps = steps.WithSteps(b.CreationTestSteps(k)) + } + for _, b := range builders { + steps = steps.WithSteps(CheckTestSteps(b, k)) + } + for _, b := range builders { + steps = steps.WithSteps(b.DeletionTestSteps(k)) + } + + if after != nil { + steps = steps.WithSteps(after(k)) + } + + return steps +}