Add elasticsearch

Makefile

@@ -47,10 +47,10 @@ install-tools-and-app: install-tools install-app
 install-tools-and-app-local: install-tools-local install-app-local
 
 .PHONY: install-tools-local
-install-tools-local: create-namespace-observability install-cert-manager install-jaeger install-tempo install-kube-prometheus-stack install-opentelemetry-operator install-opentelemetry-collector-local
+install-tools-local: create-namespace-observability install-cert-manager install-jaeger install-tempo install-kube-prometheus-stack install-elasticsearch install-opentelemetry-operator install-opentelemetry-collector-local
 
 .PHONY: install-tools
-install-tools: create-namespace-observability install-cert-manager install-jaeger install-tempo install-kube-prometheus-stack install-opentelemetry-operator install-opentelemetry-collector
+install-tools: create-namespace-observability install-cert-manager install-jaeger install-tempo install-kube-prometheus-stack install-elasticsearch install-opentelemetry-operator install-opentelemetry-collector
 
 .PHONY: create-namespace-observability
 create-namespace-observability:
@@ -78,17 +78,25 @@ install-opentelemetry-operator:
 
 .PHONY: install-opentelemetry-collector-local
 install-opentelemetry-collector-local:
-	helm upgrade --install -n observability otel ./collector/opentelemetry
+	helm upgrade --install \
+		-n observability \
+		--set es.password=$(shell kubectl get secret -n observability elasticsearch-es-elastic-user -o jsonpath='{.data.elastic}' | base64 -d) \
+		otel ./collector/opentelemetry
 
 .PHONY: install-opentelemetry-collector
 install-opentelemetry-collector:
 	helm upgrade \
 		-n observability \
 		--install \
 		--set collector.image.repository=ghcr.io/trstringer/otel-shopping-cart-collector \
+		--set es.password=$(shell kubectl get secret -n observability elasticsearch-es-elastic-user -o jsonpath='{.data.elastic}' | base64 -d) \
 		otel \
 		./collector/opentelemetry
 
+.PHONY: install-elasticsearch
+install-elasticsearch:
+	./scripts/elasticsearch_install.sh
+
 .PHONY: create-namespace-app
 create-namespace-app:
 	if ! kubectl get ns app; then kubectl create ns app; fi
@@ -148,6 +156,13 @@ port-forward-grafana:
 port-forward-prometheus:
 	kubectl port-forward -n observability svc/prometheus-kube-prometheus-prometheus 9090
 
+.PHONY: port-forward-kibana
+port-forward-kibana:
+	@echo "Kibana elastic user password:"
+	@kubectl get secret -n observability elasticsearch-es-elastic-user -o jsonpath='{.data.elastic}' | base64 -d
+	@echo
+	kubectl port-forward -n observability svc/kibana-kb-http 5601
+
 .PHONY: e2e
 e2e:
 	./scripts/e2e.sh

collector/manifest.yaml

@@ -11,13 +11,16 @@ exporters:
     gomod: go.opentelemetry.io/collector v0.94.1
   - import: go.opentelemetry.io/collector/exporter/otlpexporter
     gomod: go.opentelemetry.io/collector v0.94.1
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusexporter v0.94.0
-
-processors:
-  - import: go.opentelemetry.io/collector/processor/batchprocessor
-    gomod: go.opentelemetry.io/collector v0.94.1
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusexporter v0.96.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter v0.96.0
 
 receivers:
   - import: go.opentelemetry.io/collector/receiver/otlpreceiver
     gomod: go.opentelemetry.io/collector v0.94.1
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.94.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.96.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver v0.96.0
+
+processors:
+  - import: go.opentelemetry.io/collector/processor/batchprocessor
+    gomod: go.opentelemetry.io/collector v0.94.1
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.96.0

collector/opentelemetry/templates/opentelemetry-collector.yaml

@@ -4,12 +4,27 @@ metadata:
   name: otel-collector
 spec:
   image: "{{ .Values.collector.image.repository }}:{{ .Values.collector.image.tag }}"
-  mode: statefulset
+  mode: daemonset
   serviceAccount: otelcol
   ports:
     - name: prom
       port: 9090
+  volumes:
+    - name: varlog
+      hostPath:
+        path: /var/log
+    - name: es
+      secret:
+        secretName: elasticsearch-es-default-es-transport-certs
+  volumeMounts:
+    - name: varlog
+      mountPath: /var/log
+      readOnly: true
+    - name: es
+      mountPath: /var/es
+      readOnly: true
   targetAllocator:
+    allocationStrategy: "per-node"
     enabled: true
     serviceAccount: otelcol
     prometheusCR:
@@ -33,9 +48,111 @@ spec:
           endpoint: http://otel-collector-targetallocator
           interval: 30s
           collector_id: "${POD_NAME}"
+      # Pulled directly from https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/80ebb76abe215144c0c3314c19e1a767629433cd/examples/kubernetes/otel-collector.yaml
+      filelog:
+        include:
+          - /var/log/pods/*/*/*.log
+        start_at: end
+        include_file_path: true
+        include_file_name: false
+        operators:
+          # Find out which format is used by kubernetes
+          - type: router
+            id: get-format
+            routes:
+              - output: parser-docker
+                expr: 'body matches "^\\{"'
+              - output: parser-crio
+                expr: 'body matches "^[^ Z]+ "'
+              - output: parser-containerd
+                expr: 'body matches "^[^ Z]+Z"'
+          # Parse CRI-O format
+          - type: regex_parser
+            id: parser-crio
+            regex: '^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$'
+            output: extract_metadata_from_filepath
+            timestamp:
+              parse_from: attributes.time
+              layout_type: gotime
+              layout: '2006-01-02T15:04:05.999999999Z07:00'
+          # Parse CRI-Containerd format
+          - type: regex_parser
+            id: parser-containerd
+            regex: '^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$'
+            output: extract_metadata_from_filepath
+            timestamp:
+              parse_from: attributes.time
+              layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+          # Parse Docker format
+          - type: json_parser
+            id: parser-docker
+            output: extract_metadata_from_filepath
+            timestamp:
+              parse_from: attributes.time
+              layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+          # Extract metadata from file path
+          - type: regex_parser
+            id: extract_metadata_from_filepath
+            regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]{36})\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log$'
+            parse_from: attributes["log.file.path"]
+            cache:
+              size: 128  # default maximum amount of Pods per Node is 110
+          # Update body field after finishing all parsing
+          - type: move
+            from: attributes.log
+            to: body
+          # Rename attributes
+          - type: move
+            from: attributes.stream
+            to: attributes["log.iostream"]
+          - type: move
+            from: attributes.container_name
+            to: resource["k8s.container.name"]
+          - type: move
+            from: attributes.namespace
+            to: resource["k8s.namespace.name"]
+          - type: move
+            from: attributes.pod_name
+            to: resource["k8s.pod.name"]
+          - type: move
+            from: attributes.restart_count
+            to: resource["k8s.container.restart_count"]
+          - type: move
+            from: attributes.uid
+            to: resource["k8s.pod.uid"]
 
     processors:
       batch:
+      k8sattributes:
+        auth_type: "serviceAccount"
+        passthrough: false
+        extract:
+          metadata:
+            - k8s.pod.name
+            - k8s.pod.uid
+            - k8s.deployment.name
+            - k8s.namespace.name
+            - k8s.node.name
+            - k8s.pod.start_time
+            - k8s.cluster.uid
+          # Pod labels which can be fetched via K8sattributeprocessor
+          labels:
+            - tag_name: key1
+              key: label1
+              from: pod
+            - tag_name: key2
+              key: label2
+              from: pod
+        # Pod association using resource attributes and connection
+        pod_association:
+          - sources:
+            - from: resource_attribute
+              name: k8s.pod.uid
+          - sources:
+            - from: resource_attribute
+              name: k8s.pod.ip
+          - sources:
+            - from: connection
 
     exporters:
       logging:
@@ -50,6 +167,15 @@ spec:
           insecure: true
       prometheus:
         endpoint: "0.0.0.0:9090"
+      elasticsearch:
+        endpoints: ["https://elasticsearch-es-default:9200"]
+        tls:
+          ca_file: /var/es/ca.crt
+          cert_file: /var/es/elasticsearch-es-default-0.tls.crt
+          key_file: /var/es/elasticsearch-es-default-0.tls.key
+          insecure_skip_verify: true
+        user: elastic
+        password: {{ .Values.es.password }}
 
     service:
       pipelines:
@@ -61,3 +187,7 @@ spec:
           receivers: [prometheus]
           processors: []
           exporters: [prometheus]
+        logs:
+          receivers: [otlp,filelog]
+          processors: [k8sattributes]
+          exporters: [elasticsearch]

scripts/elasticsearch_install.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+kubectl create -f https://download.elastic.co/downloads/eck/2.11.1/crds.yaml
+kubectl apply -f https://download.elastic.co/downloads/eck/2.11.1/operator.yaml
+
+cat <<EOF | kubectl apply -f -
+apiVersion: elasticsearch.k8s.elastic.co/v1
+kind: Elasticsearch
+metadata:
+  name: elasticsearch
+  namespace: observability
+spec:
+  version: 8.12.2
+  nodeSets:
+  - name: default
+    count: 1
+    config:
+      node.store.allow_mmap: false
+EOF
+
+cat <<EOF | kubectl apply -f -
+apiVersion: kibana.k8s.elastic.co/v1
+kind: Kibana
+metadata:
+  name: kibana
+  namespace: observability
+spec:
+  version: 8.12.2
+  count: 1
+  elasticsearchRef:
+    name: elasticsearch
+EOF

version

@@ -1 +1 @@
-1.1
+1.2