Simplify configuration for switching to native OTel logs collection

CHANGELOG.md

@@ -4,6 +4,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+[Upgrade
+guidelines](https://github.com/signalfx/splunk-otel-collector-chart#0362-to-0370)
+
 ### Added
 
 - Add recommended Kubernetes labels (#217)
@@ -20,15 +23,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
   to the end of the list. Applying this change in an EC2 or Azure environment
   may change the `host.name` dimension and the resource ID dimension
   on some MTSes, possibly causing detectors to fire.
-- BREAKING CHANGE: Reduce scope of host mounted volumes on linux systems. See
-  [the upgrade
-  guideline](https://github.com/signalfx/splunk-otel-collector-chart#0362-to-0370)
-  (#232)
+- BREAKING CHANGE: Reduce scope of host mounted volumes on linux systems (#232)
 - Change `run_id` log resource attribute to `k8s.container.restart_count` (#226)
 - Use only `splunkPlatform.endpoint` and `splunkObservability.realm` parameters
   to identify which destination is enabled, remove default value for
   `splunkObservability.realm` (#230, #233)
 - Upgrade splunk-otel-collector image to 0.37.0 (#237)
+- Simplify configuration for switching to native OTel logs collection (#245)
 
 ### Fixed
 

README.md

@@ -254,10 +254,7 @@ You already have an option to use OpenTelemetry logs collection instead of fluen
 The following configuration can be used to achieve that:
 
 ```yaml
-fluentd:
-  enabled: false
-logsCollection:
-  enabled: true
+logsEngine: otel
 ```
 
 There are following known limitations of native OTel logs collection:
@@ -333,6 +330,27 @@ You need to mount the docker socket to your container as follows:
         path: /var/run/docker.sock
 ```
 
+[#245 Simplify configuration for switching to native OTel logs
+collection](https://github.com/signalfx/splunk-otel-collector-chart/pull/232)
+
+The config to enable native OTel logs collection was changed from
+
+```yaml
+fluentd:
+  enabled: false
+logsCollection:
+  enabled: true
+```
+
+to
+
+```yaml
+logsEngine: otel
+```
+
+Enabling both engines is not supported anymore. If you need that, you can
+install fluentd separately.
+
 ### 0.35.3 to 0.36.0
 
 [#209 Configuration interface changed to support both Splunk Enterprise/Cloud and Splunk Observability destinations](https://github.com/signalfx/splunk-otel-collector-chart/pull/209)

helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl

@@ -4,7 +4,7 @@ The values can be overridden in .Values.otelAgent.config
 */}}
 {{- define "splunk-otel-collector.otelAgentConfig" -}}
 extensions:
-  {{- if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") .Values.logsCollection.enabled }}
+  {{- if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") (eq .Values.logsEngine "otel") }}
   file_storage:
     directory: {{ .Values.logsCollection.checkpointPath }}
   {{- end }}
@@ -102,7 +102,7 @@ receivers:
     listenAddress: 0.0.0.0:9080
   {{- end }}
 
-  {{- if and .Values.logsCollection.enabled .Values.logsCollection.containers.enabled }}
+  {{- if and (eq .Values.logsEngine "otel") .Values.logsCollection.containers.enabled }}
   filelog:
     include: ["/var/log/pods/*/*/*.log"]
     # Exclude logs. The file format is
@@ -283,7 +283,7 @@ processors:
         {{- end }}
       {{- end }}
 
-  {{- if .Values.fluentd.enabled }}
+  {{- if eq .Values.logsEngine "fluentd" }}
   # Move flat fluentd logs attributes to resource attributes
   groupbyattrs/logs:
     keys:
@@ -400,7 +400,7 @@ exporters:
 
 service:
   extensions:
-    {{- if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") .Values.logsCollection.enabled }}
+    {{- if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") (eq .Values.logsEngine "otel") }}
     - file_storage
     {{- end }}
     - health_check
@@ -416,14 +416,14 @@ service:
     {{- if (eq (include "splunk-otel-collector.logsEnabled" .) "true") }}
     logs:
       receivers:
-        {{- if and .Values.logsCollection.enabled .Values.logsCollection.containers.enabled }}
+        {{- if and (eq .Values.logsEngine "otel") .Values.logsCollection.containers.enabled }}
         - filelog
         {{- end }}
         - fluentforward
         - otlp
       processors:
         - memory_limiter
-        {{- if .Values.fluentd.enabled }}
+        {{- if eq .Values.logsEngine "fluentd" }}
         - groupbyattrs/logs
         {{- end }}
         - k8sattributes

helm-charts/splunk-otel-collector/templates/configmap-fluentd-cri.yaml

@@ -1,4 +1,4 @@
-{{ if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") .Values.otelAgent.enabled .Values.fluentd.enabled }}
+{{ if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") .Values.otelAgent.enabled (eq .Values.logsEngine "fluentd") }}
 {{/*
 Fluentd config parts applied only to clusters with containerd/cri-o runtime.
 */}}

helm-charts/splunk-otel-collector/templates/configmap-fluentd-json.yaml

@@ -1,4 +1,4 @@
-{{ if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") .Values.otelAgent.enabled .Values.fluentd.enabled }}
+{{ if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") .Values.otelAgent.enabled (eq .Values.logsEngine "fluentd") }}
 {{/*
 Fluentd config parts applied only to clusters with docker runtime.
 */}}

helm-charts/splunk-otel-collector/templates/configmap-fluentd.yaml

@@ -1,4 +1,4 @@
-{{ if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") .Values.otelAgent.enabled .Values.fluentd.enabled }}
+{{ if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") .Values.otelAgent.enabled (eq .Values.logsEngine "fluentd") }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

helm-charts/splunk-otel-collector/templates/daemonset.yaml

@@ -9,7 +9,7 @@ metadata:
     chart: {{ template "splunk-otel-collector.chart" . }}
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
-    {{- if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") .Values.fluentd.enabled }}
+    {{- if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") (eq .Values.logsEngine "fluentd") }}
     engine: fluentd
     {{- end }}
   {{- if .Values.otelAgent.annotations }}
@@ -60,7 +60,7 @@ spec:
       tolerations:
         {{ toYaml . | nindent 8 }}
       {{- end }}
-      {{- if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") .Values.fluentd.enabled }}
+      {{- if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") (eq .Values.logsEngine "fluentd") }}
       initContainers:
         - name: prepare-fluentd-config
           image: {{ .Values.image.fluentd.initContainer.image }}
@@ -92,7 +92,7 @@ spec:
               mountPath: /fluentd/etc/cri
       {{- end }}
       containers:
-      {{- if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") .Values.fluentd.enabled }}
+      {{- if and (eq (include "splunk-otel-collector.logsEnabled" .) "true") (eq .Values.logsEngine "fluentd") }}
       - name: fluentd
         image: {{ template "splunk-otel-collector.image.fluentd" . }}
         imagePullPolicy: {{ .Values.image.fluentd.pullPolicy }}
@@ -157,9 +157,9 @@ spec:
         {{- end }}
         image: {{ template "splunk-otel-collector.image.otelcol" . }}
         imagePullPolicy: {{ .Values.image.otelcol.pullPolicy }}
-        {{- if or .Values.otelAgent.securityContext (and (eq (include "splunk-otel-collector.logsEnabled" $) "true") .Values.logsCollection.enabled) }}
+        {{- if or .Values.otelAgent.securityContext (and (eq (include "splunk-otel-collector.logsEnabled" $) "true") (eq .Values.logsEngine "otel")) }}
         securityContext:
-          {{- if and (eq (include "splunk-otel-collector.logsEnabled" $) "true") .Values.logsCollection.enabled }}
+          {{- if and (eq (include "splunk-otel-collector.logsEnabled" $) "true") (eq .Values.logsEngine "otel") }}
           runAsUser: 0
           {{- end }}
           {{- if .Values.otelAgent.securityContext }}
@@ -273,7 +273,7 @@ spec:
           readOnly: true
         {{- end }}
         {{- end }}
-        {{- if and (eq (include "splunk-otel-collector.logsEnabled" $) "true") .Values.logsCollection.enabled }}
+        {{- if and (eq (include "splunk-otel-collector.logsEnabled" $) "true") (eq .Values.logsEngine "otel") }}
         - name: varlog
           mountPath: /var/log
           readOnly: true
@@ -289,7 +289,7 @@ spec:
       terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
       volumes:
       {{- if (eq (include "splunk-otel-collector.logsEnabled" .) "true") }}
-      {{- if .Values.fluentd.enabled }}
+      {{- if eq .Values.logsEngine "fluentd" }}
       - name: varlog
         hostPath:
           path: {{ .Values.fluentd.config.containers.path }}
@@ -314,7 +314,7 @@ spec:
         configMap:
           name: {{ template "splunk-otel-collector.fullname" . }}-fluentd-json
       {{- end}}
-      {{- if .Values.logsCollection.enabled }}
+      {{- if eq .Values.logsEngine "otel" }}
       - name: varlog
         hostPath:
           path: /var/log

helm-charts/splunk-otel-collector/values.schema.json

@@ -152,6 +152,14 @@
         }
       ]
     },
+    "logsEngine": {
+      "description": "Logs collection engine",
+      "type": "string",
+      "enum": [
+        "fluentd",
+        "otel"
+      ]
+    },
     "provider": {
       "description": "Cloud provider where the collector is running",
       "type": "string",
@@ -436,9 +444,6 @@
       "type": "object",
       "additionalProperties": false,
       "properties": {
-        "enabled": {
-          "type": "boolean"
-        },
         "containers": {
           "type": "object",
           "additionalProperties": false,
@@ -482,9 +487,6 @@
       "type": "object",
       "additionalProperties": false,
       "properties": {
-        "enabled": {
-          "type": "boolean"
-        },
         "resources": {
           "type": "object",
           "additionalProperties": false,

helm-charts/splunk-otel-collector/values.yaml

@@ -94,6 +94,18 @@ splunkObservability:
   metricsEnabled: true
   tracesEnabled: true
 
+################################################################################
+# Logs collection engine:
+# - `fluentd`: deploy a fluentd sidecar that will collect logs and send them to
+#   otel-collector agent for further processing.
+# - `otel`: utilize native OpenTelemetry log collection (experimental).
+#
+# Change it to `otel` to get higher throughput performance and avoid installing
+# an extra container for fluentd.
+################################################################################
+
+logsEngine: fluentd
+
 ################################################################################
 # Cloud provider, if any, the collector is running on. Leave empty for none/other.
 # - "aws" (Amazon Web Services)
@@ -304,9 +316,6 @@ otelK8sClusterReceiver:
 #################################################################
 
 logsCollection:
-  # Otel native logs collection is disabled by default.
-  # If you want to enabled it, make sure to disable fluentd to avoid duplication.
-  enabled: false
 
   # Container logs collection
   containers:
@@ -365,8 +374,6 @@ logsCollection:
 ################################################################################
 
 fluentd:
-  enabled: true
-
   resources:
     limits:
       cpu: 500m