signalfx · jvoravong · Sep 6, 2023 · Apr 18, 2023 · May 5, 2023 · May 9, 2023
@@ -86,6 +86,7 @@ This Splunk OpenTelemetry Collector for Kubernetes release adopts the [Splunk Op
 ### Added
 
 - Option to use lightprometheus receiver through a feature gate for metrics collection from discovered Prometheus endpoints [757](https://github.com/signalfx/splunk-otel-collector-chart/pull/757)
+- Configuration of persistent buffering for agent [753](https://github.com/signalfx/splunk-otel-collector-chart/pull/753)
 
 ### Changed
 

diff --git a/helm-charts/splunk-otel-collector/templates/_helpers.tpl b/helm-charts/splunk-otel-collector/templates/_helpers.tpl
@@ -441,6 +441,26 @@ Whether clusterReceiver should be enabled
 {{- end -}}
 
 
+{{/*
+Whether persistentQueue should be enabled
+*/}}
+{{- define "splunk-otel-collector.persistentQueueEnabledLogs" -}}
+{{ $gatewayDisabled := ne (include "splunk-otel-collector.gatewayEnabled" .) "true" }}
+{{- and $gatewayDisabled (.Values.splunkPlatform.sendingQueue.persistentQueueEnabled.logs) (and (ne (include "splunk-otel-collector.distribution" .) "eks/fargate") (ne (include "splunk-otel-collector.distribution" .) "gke/autopilot")) -}}
+{{- end -}}
+
+
+{{- define "splunk-otel-collector.persistentQueueEnabledMetrics" -}}
+{{ $gatewayDisabled := ne (include "splunk-otel-collector.gatewayEnabled" .) "true" }}
+{{- and $gatewayDisabled (.Values.splunkPlatform.sendingQueue.persistentQueueEnabled.metrics) (and (ne (include "splunk-otel-collector.distribution" .) "eks/fargate") (ne (include "splunk-otel-collector.distribution" .) "gke/autopilot")) -}}
+{{- end -}}
+
+
+{{- define "splunk-otel-collector.persistentQueueEnabled" -}}
+{{- or (eq (include "splunk-otel-collector.persistentQueueEnabledLogs" .) "true") (eq (include "splunk-otel-collector.persistentQueueEnabledMetrics" .) "true") }}
+{{- end -}}
+
+
 {{/*
 Build the securityContext for Linux and Windows
 */}}

@@ -235,6 +235,26 @@ filter/logs:
           value: "true"
 {{- end }}
 
+{{- define "splunk-otel-collector.persistentQueueLogs" -}}
+file_storage/persistent_queue_logs:
+  {{- if .forAgent }}
+  directory: {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/agent/logs
+  {{- else }}
+  directory: {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/clusterReceiver/logs
+  {{- end }}
+  timeout: 0
+{{- end }}
+
+{{- define "splunk-otel-collector.persistentQueueMetrics" -}}
+file_storage/persistent_queue_metrics:
+  {{- if .forAgent }}
+  directory: {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/agent/metrics
+  {{- else }}
+  directory: {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/clusterReceiver/metrics
+  {{- end }}
+  timeout: 0
+{{- end }}
+
 {{/*
 Splunk Platform Logs exporter
 */}}
@@ -271,6 +291,9 @@ splunk_hec/platform_logs:
     enabled:  {{ .Values.splunkPlatform.sendingQueue.enabled }}
     num_consumers: {{ .Values.splunkPlatform.sendingQueue.numConsumers }}
     queue_size: {{ .Values.splunkPlatform.sendingQueue.queueSize }}
+    {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledLogs" .) "true") }}
+    storage: file_storage/persistent_queue_logs
+    {{- end }}
 {{- end }}
 
 {{/*
@@ -308,6 +331,9 @@ splunk_hec/platform_metrics:
     enabled:  {{ .Values.splunkPlatform.sendingQueue.enabled }}
     num_consumers: {{ .Values.splunkPlatform.sendingQueue.numConsumers }}
     queue_size: {{ .Values.splunkPlatform.sendingQueue.queueSize }}
+    {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledMetrics" .) "true") }}
+    storage: file_storage/persistent_queue_metrics
+    {{- end }}
 {{- end }}
 
 {{/*

@@ -11,6 +11,11 @@ extensions:
     directory: {{ .Values.logsCollection.checkpointPath }}
   {{- end }}
 
+  {{- if (eq (include "splunk-otel-collector.persistentQueueEnabled" .) "true") }}
+  {{- include "splunk-otel-collector.persistentQueueLogs" (dict "Values" .Values "forAgent" true) | nindent 2 }}
+  {{- include "splunk-otel-collector.persistentQueueMetrics" (dict "Values" .Values "forAgent" true) | nindent 2 }}
+  {{- end }}
+
   memory_ballast:
     size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
 
@@ -648,6 +653,10 @@ service:
     {{- if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") (eq .Values.logsEngine "otel") }}
     - file_storage
     {{- end }}
+    {{- if (eq (include "splunk-otel-collector.persistentQueueEnabled" .) "true") }}
+    - file_storage/persistent_queue_logs
+    - file_storage/persistent_queue_metrics
+    {{- end }}
     - health_check
     - k8s_observer
     - memory_ballast

diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl
@@ -5,6 +5,11 @@ The values can be overridden in .Values.clusterReceiver.config
 {{- define "splunk-otel-collector.clusterReceiverConfig" -}}
 {{ $clusterReceiver := fromYaml (include "splunk-otel-collector.clusterReceiver" .) -}}
 extensions:
+  {{- if (eq (include "splunk-otel-collector.persistentQueueEnabled" .) "true") }}
+  {{- include "splunk-otel-collector.persistentQueueLogs" (dict "Values" .Values "forAgent" false) | nindent 2 }}
+  {{- include "splunk-otel-collector.persistentQueueMetrics" (dict "Values" .Values "forAgent" false) | nindent 2 }}
+  {{- end }}
+
   health_check:
 
   memory_ballast:
@@ -198,11 +203,16 @@ service:
   telemetry:
     metrics:
       address: 0.0.0.0:8889
-  {{- if eq (include "splunk-otel-collector.distribution" .) "eks/fargate" }}
-  extensions: [health_check, memory_ballast, k8s_observer]
-  {{- else }}
-  extensions: [health_check, memory_ballast]
-  {{- end }}
+  extensions:
+    - health_check
+    - memory_ballast
+    {{- if eq (include "splunk-otel-collector.distribution" .) "eks/fargate" }}
+    - k8s_observer
+    {{- end }}
+    {{- if (eq (include "splunk-otel-collector.persistentQueueEnabled" .) "true") }}
+    - file_storage/persistent_queue_metrics
+    - file_storage/persistent_queue_logs
+    {{- end }}
   pipelines:
     {{- if or (eq (include "splunk-otel-collector.o11yMetricsEnabled" $) "true") (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }}
     # k8s metrics pipeline

@@ -160,6 +160,18 @@ spec:
           then
               setfacl -n -Rm d:m::rx,m::rx,d:g:{{ $agent.securityContext.runAsGroup | default 999 }}:rx,g:{{ $agent.securityContext.runAsGroup | default 999 }}:rx {{ .Values.logsCollection.journald.directory }};
           fi;
+          {{- end }}
+          {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledLogs" .) "true") }}
+          mkdir -p {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/agent/logs;
+          chown -Rv {{ $agent.securityContext.runAsUser | default 999 }}:{{ $agent.securityContext.runAsGroup | default 999 }} {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/agent/logs;
+          chmod -v g+rwxs {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/agent/logs;
+          setfacl -n -Rm d:m::rx,m::rx,d:g:{{ $agent.securityContext.runAsGroup | default 999 }}:rx,g:{{ $agent.securityContext.runAsGroup | default 999 }}:rx {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/agent/logs;
+          {{- end }}
+          {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledMetrics" .) "true") }}
+          mkdir -p {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/agent/metrics;
+          chown -Rv {{ $agent.securityContext.runAsUser | default 999 }}:{{ $agent.securityContext.runAsGroup | default 999 }} {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/agent/metrics;
+          chmod -v g+rwxs {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/agent/metrics;
+          setfacl -n -Rm d:m::rx,m::rx,d:g:{{ $agent.securityContext.runAsGroup | default 999 }}:rx,g:{{ $agent.securityContext.runAsGroup | default 999 }}:rx {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/agent/metrics;
           {{- end }}']
           securityContext:
             runAsUser: 0
@@ -178,6 +190,14 @@ spec:
             - name: journaldlogs
               mountPath: {{ .Values.logsCollection.journald.directory }}
             {{- end }}
+            {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledLogs" .) "true") }}
+            - name: persistent-queue-logs
+              mountPath: {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/agent/logs
+            {{- end }}
+            {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledMetrics" .) "true") }}
+            - name: persistent-queue-metrics
+              mountPath: {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/agent/metrics
+            {{- end }}
         {{- end }}
         {{- end }}
       {{- end }}
@@ -398,6 +418,14 @@ spec:
         {{- end }}
         - name: checkpoint
           mountPath: {{ .Values.logsCollection.checkpointPath }}
+        {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledLogs" .) "true") }}
+        - name: persistent-queue-logs
+          mountPath: {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/agent/logs
+        {{- end }}
+        {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledMetrics" .) "true") }}
+        - name: persistent-queue-metrics
+          mountPath: {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/agent/metrics
+        {{- end }}
         {{- if .Values.logsCollection.journald.enabled }}
         - mountPath: {{.Values.logsCollection.journald.directory}}
           name: journaldlogs
@@ -464,6 +492,18 @@ spec:
         hostPath:
           path: {{ .Values.logsCollection.checkpointPath }}
           type: DirectoryOrCreate
+      {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledLogs" .) "true") }}
+      - name: persistent-queue-logs
+        hostPath:
+          path: {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/agent/logs
+          type: DirectoryOrCreate
+      {{- end }}
+      {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledMetrics" .) "true") }}
+      - name: persistent-queue-metrics
+        hostPath:
+          path: {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/agent/metrics
+          type: DirectoryOrCreate
+      {{- end }}
       {{- if .Values.logsCollection.journald.enabled }}
       - name: journaldlogs
         hostPath:

diff --git a/helm-charts/splunk-otel-collector/templates/deployment-cluster-receiver.yaml b/helm-charts/splunk-otel-collector/templates/deployment-cluster-receiver.yaml
@@ -33,6 +33,11 @@ spec:
       app: {{ template "splunk-otel-collector.name" . }}
       component: otel-k8s-cluster-receiver
       release: {{ .Release.Name }}
+  {{- if (eq (include "splunk-otel-collector.persistentQueueEnabled" .) "true") }}
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+  {{- end }}
   template:
     metadata:
       labels:
@@ -81,8 +86,9 @@ spec:
       securityContext:
         {{- include "splunk-otel-collector.securityContext" (dict "isWindows" .Values.isWindows "securityContext" $clusterReceiver.securityContext) | nindent 8 }}
       {{- end }}
-      {{- if eq (include "splunk-otel-collector.distribution" .) "eks/fargate" }}
+      {{- if or (eq (include "splunk-otel-collector.distribution" .) "eks/fargate") (.Values.splunkPlatform.sendingQueue.persistentQueueEnabled.logs) (.Values.splunkPlatform.sendingQueue.persistentQueueEnabled.metrics) }}
       initContainers:
+        {{- if eq (include "splunk-otel-collector.distribution" .) "eks/fargate" }}
         - name: cluster-receiver-node-discoverer
           image: public.ecr.aws/amazonlinux/amazonlinux:latest
           imagePullPolicy: IfNotPresent
@@ -103,6 +109,36 @@ spec:
               mountPath: /splunk-messages
             - mountPath: /conf
               name: collector-configmap
+        {{- end }}
+        {{- if (eq (include "splunk-otel-collector.persistentQueueEnabled" .) "true") }}
+        - name: patch-log-dirs
+          image: {{ template "splunk-otel-collector.image.initPatchLogDirs" . }}
+          imagePullPolicy: {{ .Values.image.initPatchLogDirs.pullPolicy }}
+          command: ['sh', '-c', '
+          {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledLogs" .) "true") }}
+          mkdir -p {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/clusterReceiver/logs;
+          chown -Rv {{ $clusterReceiver.securityContext.runAsUser | default 999 }}:{{ $clusterReceiver.securityContext.runAsGroup | default 999 }} {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/clusterReceiver/logs;
+          chmod -v g+rwxs {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/clusterReceiver/logs;
+          setfacl -n -Rm d:m::rx,m::rx,d:g:{{ $clusterReceiver.securityContext.runAsGroup | default 999 }}:rx,g:{{ $clusterReceiver.securityContext.runAsGroup | default 999 }}:rx {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/clusterReceiver/logs;
+          {{- end }}
+          {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledMetrics" .) "true") }}
+          mkdir -p {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/clusterReceiver/metrics;
+          chown -Rv {{ $clusterReceiver.securityContext.runAsUser | default 999 }}:{{ $clusterReceiver.securityContext.runAsGroup | default 999 }} {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/clusterReceiver/metrics;
+          chmod -v g+rwxs {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/clusterReceiver/metrics;
+          setfacl -n -Rm d:m::rx,m::rx,d:g:{{ $clusterReceiver.securityContext.runAsGroup | default 999 }}:rx,g:{{ $clusterReceiver.securityContext.runAsGroup | default 999 }}:rx {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/clusterReceiver/metrics;
+          {{- end }}']
+          securityContext:
+            runAsUser: 0
+          volumeMounts:
+            {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledMetrics" .) "true") }}
+            - name: persistent-queue-metrics
+              mountPath: {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/clusterReceiver/metrics
+            {{- end }}
+            {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledLogs" .) "true") }}
+            - name: persistent-queue-logs
+              mountPath: {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/clusterReceiver/logs
+            {{- end }}
+        {{- end }}
       {{- end }}
       containers:
       - name: otel-collector
@@ -194,6 +230,14 @@ spec:
         - mountPath: /splunk-messages
           name: messages
         {{- end }}
+        {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledMetrics" .) "true") }}
+        - mountPath: {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/clusterReceiver/metrics
+          name: persistent-queue-metrics
+        {{- end }}
+        {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledLogs" .) "true") }}
+        - mountPath: {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/clusterReceiver/logs
+          name: persistent-queue-logs
+        {{- end }}
         - mountPath: /usr/lib/splunk-otel-collector/agent-bundle/run/collectd
           name: run-collectd
           readOnly: false
@@ -227,6 +271,18 @@ spec:
       - name: messages
         emptyDir: {}
       {{- end }}
+      {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledMetrics" .) "true") }}
+      - name: persistent-queue-metrics
+        hostPath:
+          path: {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/clusterReceiver/metrics
+          type: DirectoryOrCreate
+      {{- end }}
+      {{- if (eq (include "splunk-otel-collector.persistentQueueEnabledLogs" .) "true") }}
+      - name: persistent-queue-logs
+        hostPath:
+          path: {{ .Values.splunkPlatform.sendingQueue.persistentQueueEnabled.storagePath }}/clusterReceiver/logs
+          type: DirectoryOrCreate
+      {{- end }}
       {{- if $clusterReceiver.extraVolumes }}
       {{- toYaml $clusterReceiver.extraVolumes | nindent 6 }}
       {{- end }}

@@ -135,6 +135,20 @@
             },
             "queueSize": {
               "type": "integer"
+            },
+            "persistentQueueEnabled": {
+              "type": "object",
+              "properties": {
+                "logs": {
+                  "type": "boolean"
+                },
+                "metrics": {
+                  "type": "boolean"
+                },
+                "storagePath": {
+                  "type": "string"
+                }
+              }
             }
           }
         }

@@ -106,6 +106,18 @@ splunkPlatform:
     #   requests_per_second is the average number of requests per seconds.
     queueSize: 5000
 
+    # When enabled, it uses file_storage extension to persist the queue data.
+    # This can be used as an alternative redundancy mechanism for data being exported.
+    # NOTE: The File Storage extension will persist state to the node's local file system.
+    # While using the persistent queue it is advised to increase memory limit for agent (agent.resources.limits.memory)
+    # to 1Gi. Persistent queue is currently supported only for logs and metrics of the agent.
+    persistentQueueEnabled:
+      # Specifies whether to persist log data.
+      logs: false
+      # Specifies whether to persist metric data.
+      metrics: false
+      storagePath: "/var/addon/splunk/persist"
+
 ################################################################################
 # Splunk Observability configuration
 ################################################################################