add combineWith to multiline options (#993)

* add combineWith to multiline options

* Add multiline config to example

* support empty strings and default value properly

* code review

.chloggen/recombine_with_field.yaml

@@ -0,0 +1,12 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+# The name of the component, or a single word describing the area of concern, (e.g. agent, clusterReceiver, gateway, networkExplorer, operator, chart, other)
+component: agent
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add combineWith field to multiline configuration
+# One or more tracking issues related to the change
+issues: [756]
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

docs/advanced-configuration.md

@@ -474,6 +474,7 @@ logsCollection:
         containerName:
           value: server
         firstEntryRegex: ^[^\s].*
+        combineWith: ""
 ```
 
 Use https://regex101.com/ to find a golang regex that works for your format and specify it in the config file for the config option `firstEntryRegex`.

examples/only-logs-otel/only-logs-otel-values.yaml

@@ -7,3 +7,33 @@ splunkObservability:
   tracesEnabled: false
 
 logsEngine: otel
+logsCollection:
+  containers:
+   multilineConfigs:
+     - namespaceName:
+         value: default
+       podName:
+         value: buttercup-app-.*
+         useRegexp: true
+       containerName:
+         value: server
+       firstEntryRegex: ^[^\s].*
+       combineWith: "\t" # Recombine entries together with a tab character.
+     - namespaceName:
+         value: k8s-nm2
+       podName:
+         value: foo-app-.*
+         useRegexp: true
+       containerName:
+         value: client
+       firstEntryRegex: ^[^\s].*
+       combineWith: "" # Recombine entries without a separator.
+     - namespaceName:
+         value: k8s-nm3
+       podName:
+         value: foo-app-.*
+         useRegexp: true
+       containerName:
+         value: server
+       firstEntryRegex: ^[^\s].*
+       # Recombine entries with the default separator, "\n".

examples/only-logs-otel/rendered_manifests/configmap-agent.yaml

@@ -227,6 +227,41 @@ data:
         - from: attributes["log.file.path"]
           to: resource["com.splunk.source"]
           type: move
+        - default: clean-up-log-record
+          routes:
+          - expr: (resource["k8s.namespace.name"]) == "default" && (resource["k8s.pod.name"])
+              matches "buttercup-app-.*" && (resource["k8s.container.name"]) == "server"
+            output: default_buttercup-app-.*_server
+          - expr: (resource["k8s.namespace.name"]) == "k8s-nm2" && (resource["k8s.pod.name"])
+              matches "foo-app-.*" && (resource["k8s.container.name"]) == "client"
+            output: k8s-nm2_foo-app-.*_client
+          - expr: (resource["k8s.namespace.name"]) == "k8s-nm3" && (resource["k8s.pod.name"])
+              matches "foo-app-.*" && (resource["k8s.container.name"]) == "server"
+            output: k8s-nm3_foo-app-.*_server
+          type: router
+        - combine_field: attributes.log
+          combine_with: "\t"
+          id: default_buttercup-app-.*_server
+          is_first_entry: (attributes.log) matches "^[^\\s].*"
+          max_log_size: 1048576
+          output: clean-up-log-record
+          source_identifier: resource["com.splunk.source"]
+          type: recombine
+        - combine_field: attributes.log
+          combine_with: ""
+          id: k8s-nm2_foo-app-.*_client
+          is_first_entry: (attributes.log) matches "^[^\\s].*"
+          max_log_size: 1048576
+          output: clean-up-log-record
+          source_identifier: resource["com.splunk.source"]
+          type: recombine
+        - combine_field: attributes.log
+          id: k8s-nm3_foo-app-.*_server
+          is_first_entry: (attributes.log) matches "^[^\\s].*"
+          max_log_size: 1048576
+          output: clean-up-log-record
+          source_identifier: resource["com.splunk.source"]
+          type: recombine
         - from: attributes.log
           id: clean-up-log-record
           to: body

examples/only-logs-otel/rendered_manifests/daemonset.yaml

@@ -31,7 +31,7 @@ spec:
         component: otel-collector-agent
         release: default
       annotations:
-        checksum/config: 82c6445c679f194d080fa8bd2763321f93e84c16fa3cb7217750960f4348e1ee
+        checksum/config: 4f78d6630e152af9891dda63bcdfc713718524850c2de9794b88ba005c4875ec
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl

@@ -409,6 +409,9 @@ receivers:
         combine_field: attributes.log
         is_first_entry: '(attributes.log) matches {{ .firstEntryRegex | quote }}'
         max_log_size: {{ $.Values.logsCollection.containers.maxRecombineLogSize }}
+        {{- if hasKey . "combineWith" }}
+        combine_with: {{ .combineWith | quote }}
+        {{- end }}
       {{- end }}
       {{- end }}
       # Clean up log record

helm-charts/splunk-otel-collector/values.yaml

@@ -587,6 +587,7 @@ logsCollection:
     #     containerName:
     #       value: server
     #     firstEntryRegex: ^[^\s].*
+    #     combineWith: ""
     multilineConfigs: []
     # Set useSplunkIncludeAnnotation flag to `true` to collect logs from pods with `splunk.com/include: true` annotation and ignore others.
     # All other logs will be ignored.