Merge pull request #1963 from bosch-io/feature/make-logstash-buffer-s…

…izes-configurable make logstash buffer sizes configurable
eclipse-ditto · Jun 12, 2024 · 6a1bbc5 · 6a1bbc5
2 parents 44bd11b + 256b87f
commit 6a1bbc5
Show file tree

Hide file tree

Showing 12 changed files with 35 additions and 1 deletion.
diff --git a/connectivity/service/src/main/resources/logback.xml b/connectivity/service/src/main/resources/logback.xml
@@ -53,6 +53,8 @@
             <appender name="stash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
                 <destination>${DITTO_LOGGING_LOGSTASH_SERVER}</destination>
                 <reconnectionDelay>1 second</reconnectionDelay>
+                <writeBufferSize>${LOGSTASH_WRITE_BUFFER_SIZE:-8192}</writeBufferSize>
+                <ringBufferSize>${LOGSTASH_RING_BUFFER_SIZE:-8192}</ringBufferSize>
 
                 <!-- Encoder is required -->
                 <encoder class="net.logstash.logback.encoder.LogstashEncoder">

diff --git a/deployment/helm/ditto/Chart.yaml b/deployment/helm/ditto/Chart.yaml
@@ -16,7 +16,7 @@ description: |
   A digital twin is a virtual, cloud based, representation of his real world counterpart
   (real world “Things”, e.g. devices like sensors, smart heating, connected cars, smart grids, EV charging stations etc).
 type: application
-version: 3.5.8  # chart version is effectively set by release-job
+version: 3.5.9  # chart version is effectively set by release-job
 appVersion: 3.5.8
 keywords:
   - iot-chart

diff --git a/deployment/helm/ditto/templates/connectivity-deployment.yaml b/deployment/helm/ditto/templates/connectivity-deployment.yaml
@@ -105,6 +105,10 @@ spec:
             {{- if .Values.global.logging.logstash.enabled }}
             - name: DITTO_LOGGING_LOGSTASH_SERVER
               value: "{{ .Values.global.logging.logstash.endpoint }}"
+            - name: LOGSTASH_WRITE_BUFFER_SIZE
+              value: "{{ .Values.global.logging.logstash.writeBufferSize }}"
+            - name: LOGSTASH_RING_BUFFER_SIZE
+              value: "{{ .Values.global.logging.logstash.ringBufferSize }}"
             {{- end }}
             - name: POD_LABEL_SELECTOR
               value: "app.kubernetes.io/name=%s"

diff --git a/deployment/helm/ditto/templates/gateway-deployment.yaml b/deployment/helm/ditto/templates/gateway-deployment.yaml
@@ -106,6 +106,10 @@ spec:
             {{- if .Values.global.logging.logstash.enabled }}
             - name: DITTO_LOGGING_LOGSTASH_SERVER
               value: "{{ .Values.global.logging.logstash.endpoint }}"
+            - name: LOGSTASH_WRITE_BUFFER_SIZE
+              value: "{{ .Values.global.logging.logstash.writeBufferSize }}"
+            - name: LOGSTASH_RING_BUFFER_SIZE
+              value: "{{ .Values.global.logging.logstash.ringBufferSize }}"
             {{- end }}
             - name: POD_LABEL_SELECTOR
               value: "app.kubernetes.io/name=%s"

diff --git a/deployment/helm/ditto/templates/policies-deployment.yaml b/deployment/helm/ditto/templates/policies-deployment.yaml
@@ -105,6 +105,10 @@ spec:
             {{- if .Values.global.logging.logstash.enabled }}
             - name: DITTO_LOGGING_LOGSTASH_SERVER
               value: "{{ .Values.global.logging.logstash.endpoint }}"
+            - name: LOGSTASH_WRITE_BUFFER_SIZE
+              value: "{{ .Values.global.logging.logstash.writeBufferSize }}"
+            - name: LOGSTASH_RING_BUFFER_SIZE
+              value: "{{ .Values.global.logging.logstash.ringBufferSize }}"
             {{- end }}
             - name: POD_LABEL_SELECTOR
               value: "app.kubernetes.io/name=%s"

diff --git a/deployment/helm/ditto/templates/things-deployment.yaml b/deployment/helm/ditto/templates/things-deployment.yaml
@@ -105,6 +105,10 @@ spec:
             {{- if .Values.global.logging.logstash.enabled }}
             - name: DITTO_LOGGING_LOGSTASH_SERVER
               value: "{{ .Values.global.logging.logstash.endpoint }}"
+            - name: LOGSTASH_WRITE_BUFFER_SIZE
+              value: "{{ .Values.global.logging.logstash.writeBufferSize }}"
+            - name: LOGSTASH_RING_BUFFER_SIZE
+              value: "{{ .Values.global.logging.logstash.ringBufferSize }}"
             {{- end }}
             - name: POD_LABEL_SELECTOR
               value: "app.kubernetes.io/name=%s"

diff --git a/deployment/helm/ditto/templates/thingssearch-deployment.yaml b/deployment/helm/ditto/templates/thingssearch-deployment.yaml
@@ -105,6 +105,10 @@ spec:
             {{- if .Values.global.logging.logstash.enabled }}
             - name: DITTO_LOGGING_LOGSTASH_SERVER
               value: "{{ .Values.global.logging.logstash.endpoint }}"
+            - name: LOGSTASH_WRITE_BUFFER_SIZE
+              value: "{{ .Values.global.logging.logstash.writeBufferSize }}"
+            - name: LOGSTASH_RING_BUFFER_SIZE
+              value: "{{ .Values.global.logging.logstash.ringBufferSize }}"
             {{- end }}
             - name: POD_LABEL_SELECTOR
               value: "app.kubernetes.io/name=%s"

diff --git a/deployment/helm/ditto/values.yaml b/deployment/helm/ditto/values.yaml
@@ -131,6 +131,10 @@ global:
       enabled: false
       # endpoint configures the logstash endpoint to send logs to
       endpoint: ""
+      # logstash write buffer size
+      writeBufferSize: 8192
+      # logstash ring buffer size
+      ringBufferSize: 8192
     # logFiles defines logging to log files config
     logFiles:
       # enabled whether to write logs to log files

diff --git a/gateway/service/src/main/resources/logback.xml b/gateway/service/src/main/resources/logback.xml
@@ -53,6 +53,8 @@
             <appender name="stash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
                 <destination>${DITTO_LOGGING_LOGSTASH_SERVER}</destination>
                 <reconnectionDelay>1 second</reconnectionDelay>
+                <writeBufferSize>${LOGSTASH_WRITE_BUFFER_SIZE:-8192}</writeBufferSize>
+                <ringBufferSize>${LOGSTASH_RING_BUFFER_SIZE:-8192}</ringBufferSize>
 
                 <!-- Encoder is required -->
                 <encoder class="net.logstash.logback.encoder.LogstashEncoder">

diff --git a/policies/service/src/main/resources/logback.xml b/policies/service/src/main/resources/logback.xml
@@ -53,6 +53,8 @@
             <appender name="stash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
                 <destination>${DITTO_LOGGING_LOGSTASH_SERVER}</destination>
                 <reconnectionDelay>1 second</reconnectionDelay>
+                <writeBufferSize>${LOGSTASH_WRITE_BUFFER_SIZE:-8192}</writeBufferSize>
+                <ringBufferSize>${LOGSTASH_RING_BUFFER_SIZE:-8192}</ringBufferSize>
 
                 <!-- Encoder is required -->
                 <encoder class="net.logstash.logback.encoder.LogstashEncoder">

diff --git a/things/service/src/main/resources/logback.xml b/things/service/src/main/resources/logback.xml
@@ -53,6 +53,8 @@
             <appender name="stash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
                 <destination>${DITTO_LOGGING_LOGSTASH_SERVER}</destination>
                 <reconnectionDelay>1 second</reconnectionDelay>
+                <writeBufferSize>${LOGSTASH_WRITE_BUFFER_SIZE:-8192}</writeBufferSize>
+                <ringBufferSize>${LOGSTASH_RING_BUFFER_SIZE:-8192}</ringBufferSize>
 
                 <!-- Encoder is required -->
                 <encoder class="net.logstash.logback.encoder.LogstashEncoder">

diff --git a/thingsearch/service/src/main/resources/logback.xml b/thingsearch/service/src/main/resources/logback.xml
@@ -53,6 +53,8 @@
             <appender name="stash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
                 <destination>${DITTO_LOGGING_LOGSTASH_SERVER}</destination>
                 <reconnectionDelay>1 second</reconnectionDelay>
+                <writeBufferSize>${LOGSTASH_WRITE_BUFFER_SIZE:-8192}</writeBufferSize>
+                <ringBufferSize>${LOGSTASH_RING_BUFFER_SIZE:-8192}</ringBufferSize>
 
                 <!-- Encoder is required -->
                 <encoder class="net.logstash.logback.encoder.LogstashEncoder">