Merge branch 'master' into bugfix/remove-unused-configuration

bom/pom.xml

@@ -55,7 +55,7 @@
         <metrics4-scala.version>4.3.2</metrics4-scala.version>
 
         <!-- Keep these version consistent with pekko-persistence-mongodb.version's build.sbt -->
-        <mongo-java-driver.version>5.1.3</mongo-java-driver.version>
+        <mongo-java-driver.version>5.1.4</mongo-java-driver.version>
 
         <!-- AWS SDK version needed for MongoDB AWS IAM authentication -->
         <awssdk.version>2.26.21</awssdk.version>

deployment/helm/ditto/templates/connectivity-deployment.yaml

@@ -63,7 +63,7 @@ spec:
           image: busybox
           securityContext:
             runAsUser: 0
-          command: [ "sh", "-c", "chown -R 1000:1000 /var/log/ditto && echo 'changed ownership of /var/log/ditto to 1000:1000'" ]
+          command: [ "sh", "-c", "chown -R 1005:1005 /var/log/ditto && echo 'changed ownership of /var/log/ditto to 1005:1005'" ]
           volumeMounts:
             - name: ditto-log-files-directory
               mountPath: /var/log/ditto
@@ -221,6 +221,10 @@ spec:
               value: "{{ .Values.connectivity.config.cleanup.timerThreshold }}"
             - name: CLEANUP_CREDITS_PER_BATCH
               value: "{{ .Values.connectivity.config.cleanup.creditsPerBatch }}"
+            - name: CLEANUP_READS_PER_QUERY
+              value: "{{ .Values.connectivity.config.cleanup.readsPerQuery }}"
+            - name: CLEANUP_WRITES_PER_CREDIT
+              value: "{{ .Values.connectivity.config.cleanup.writesPerCredit }}"
             - name: CLEANUP_DELETE_FINAL_DELETED_SNAPSHOT
               value: "{{ .Values.connectivity.config.cleanup.deleteFinalDeletedSnapshot }}"
             - name: MONGODB_READ_JOURNAL_SHOULD_CREATE_ADDITIONAL_SNAPSHOT_AGGREGATION_INDEX_PID_ID
@@ -339,6 +343,8 @@ spec:
             {{- if .Values.connectivity.extraVolumeMounts }}
               {{- toYaml .Values.connectivity.extraVolumeMounts | nindent 12 }}
             {{- end }}
+            - name: ditto-heap-dumps
+              mountPath: /opt/ditto/dumps
           resources:
             requests:
               cpu: {{ mulf .Values.connectivity.resources.cpu 1000 }}m
@@ -388,4 +394,6 @@ spec:
         {{- if .Values.connectivity.extraVolumes }}
           {{- toYaml .Values.connectivity.extraVolumes | nindent 8 }}
         {{- end}}
+        - name: ditto-heap-dumps
+          emptyDir: {}
 {{- end }}

deployment/helm/ditto/templates/gateway-deployment.yaml

@@ -64,7 +64,7 @@ spec:
           image: busybox
           securityContext:
             runAsUser: 0
-          command: [ "sh", "-c", "chown -R 1000:1000 /var/log/ditto && echo 'changed ownership of /var/log/ditto to 1000:1000'" ]
+          command: [ "sh", "-c", "chown -R 1005:1005 /var/log/ditto && echo 'changed ownership of /var/log/ditto to 1005:1005'" ]
           volumeMounts:
             - name: ditto-log-files-directory
               mountPath: /var/log/ditto
@@ -279,6 +279,8 @@ spec:
             {{- if .Values.gateway.extraVolumeMounts }}
               {{- toYaml .Values.gateway.extraVolumeMounts | nindent 12 }}
             {{- end }}
+            - name: ditto-heap-dumps
+              mountPath: /opt/ditto/dumps
           resources:
             requests:
               cpu: {{ mulf .Values.gateway.resources.cpu 1000 }}m
@@ -328,4 +330,6 @@ spec:
         {{- if .Values.gateway.extraVolumes }}
           {{- toYaml .Values.gateway.extraVolumes | nindent 8 }}
         {{- end}}
+        - name: ditto-heap-dumps
+          emptyDir: {}
 {{- end }}

deployment/helm/ditto/templates/policies-deployment.yaml

@@ -63,7 +63,7 @@ spec:
           image: busybox
           securityContext:
             runAsUser: 0
-          command: [ "sh", "-c", "chown -R 1000:1000 /var/log/ditto && echo 'changed ownership of /var/log/ditto to 1000:1000'" ]
+          command: [ "sh", "-c", "chown -R 1005:1005 /var/log/ditto && echo 'changed ownership of /var/log/ditto to 1005:1005'" ]
           volumeMounts:
             - name: ditto-log-files-directory
               mountPath: /var/log/ditto
@@ -239,6 +239,10 @@ spec:
               value: "{{ .Values.policies.config.cleanup.timerThreshold }}"
             - name: CLEANUP_CREDITS_PER_BATCH
               value: "{{ .Values.policies.config.cleanup.creditsPerBatch }}"
+            - name: CLEANUP_READS_PER_QUERY
+              value: "{{ .Values.policies.config.cleanup.readsPerQuery }}"
+            - name: CLEANUP_WRITES_PER_CREDIT
+              value: "{{ .Values.policies.config.cleanup.writesPerCredit }}"
             - name: CLEANUP_DELETE_FINAL_DELETED_SNAPSHOT
               value: "{{ .Values.policies.config.cleanup.deleteFinalDeletedSnapshot }}"
             - name: MONGODB_READ_JOURNAL_SHOULD_CREATE_ADDITIONAL_SNAPSHOT_AGGREGATION_INDEX_PID_ID
@@ -265,6 +269,14 @@ spec:
               value: "{{ .Values.policies.config.persistence.snapshots.interval }}"
             - name: POLICY_SNAPSHOT_THRESHOLD
               value: "{{ .Values.policies.config.persistence.snapshots.threshold }}"
+            - name: DITTO_POLICIES_ENFORCER_CACHE_ENABLED
+              value: "{{ .Values.policies.config.policiesEnforcer.cache.enabled }}"
+            - name: DITTO_POLICIES_ENFORCER_CACHE_MAX_SIZE
+              value: "{{ .Values.policies.config.policiesEnforcer.cache.maxSize }}"
+            - name: DITTO_POLICIES_ENFORCER_CACHE_EXPIRE_AFTER_WRITE
+              value: "{{ .Values.policies.config.policiesEnforcer.cache.expireAfterWrite }}"
+            - name: DITTO_POLICIES_ENFORCER_CACHE_EXPIRE_AFTER_ACCESS
+              value: "{{ .Values.policies.config.policiesEnforcer.cache.expireAfterAccess }}"
             {{- if .Values.policies.extraEnv }}
               {{- toYaml .Values.policies.extraEnv | nindent 12 }}
             {{- end }}
@@ -326,6 +338,8 @@ spec:
             {{- if .Values.policies.extraVolumeMounts }}
               {{- toYaml .Values.policies.extraVolumeMounts | nindent 12 }}
             {{- end }}
+            - name: ditto-heap-dumps
+              mountPath: /opt/ditto/dumps
           resources:
             requests:
               cpu: {{ mulf .Values.policies.resources.cpu 1000 }}m
@@ -375,4 +389,6 @@ spec:
         {{- if .Values.policies.extraVolumes }}
           {{- toYaml .Values.policies.extraVolumes | nindent 8 }}
         {{- end}}
+        - name: ditto-heap-dumps
+          emptyDir: {}
 {{- end }}

deployment/helm/ditto/templates/things-deployment.yaml

@@ -63,7 +63,7 @@ spec:
           image: busybox
           securityContext:
             runAsUser: 0
-          command: [ "sh", "-c", "chown -R 1000:1000 /var/log/ditto && echo 'changed ownership of /var/log/ditto to 1000:1000'" ]
+          command: [ "sh", "-c", "chown -R 1005:1005 /var/log/ditto && echo 'changed ownership of /var/log/ditto to 1005:1005'" ]
           volumeMounts:
             - name: ditto-log-files-directory
               mountPath: /var/log/ditto
@@ -270,6 +270,10 @@ spec:
               value: "{{ .Values.things.config.cleanup.timerThreshold }}"
             - name: CLEANUP_CREDITS_PER_BATCH
               value: "{{ .Values.things.config.cleanup.creditsPerBatch }}"
+            - name: CLEANUP_READS_PER_QUERY
+              value: "{{ .Values.things.config.cleanup.readsPerQuery }}"
+            - name: CLEANUP_WRITES_PER_CREDIT
+              value: "{{ .Values.things.config.cleanup.writesPerCredit }}"
             - name: CLEANUP_DELETE_FINAL_DELETED_SNAPSHOT
               value: "{{ .Values.things.config.cleanup.deleteFinalDeletedSnapshot }}"
             - name: MONGODB_READ_JOURNAL_SHOULD_CREATE_ADDITIONAL_SNAPSHOT_AGGREGATION_INDEX_PID_ID
@@ -406,6 +410,8 @@ spec:
             {{- if .Values.things.extraVolumeMounts }}
               {{- toYaml .Values.things.extraVolumeMounts | nindent 12 }}
             {{- end }}
+            - name: ditto-heap-dumps
+              mountPath: /opt/ditto/dumps
           resources:
             requests:
               cpu: {{ mulf .Values.things.resources.cpu 1000 }}m
@@ -455,4 +461,6 @@ spec:
         {{- if .Values.things.extraVolumes }}
           {{- toYaml .Values.things.extraVolumes | nindent 8 }}
         {{- end}}
+        - name: ditto-heap-dumps
+          emptyDir: {}
 {{- end }}

deployment/helm/ditto/templates/thingssearch-deployment.yaml

@@ -63,7 +63,7 @@ spec:
           image: busybox
           securityContext:
             runAsUser: 0
-          command: [ "sh", "-c", "chown -R 1000:1000 /var/log/ditto && echo 'changed ownership of /var/log/ditto to 1000:1000'" ]
+          command: [ "sh", "-c", "chown -R 1005:1005 /var/log/ditto && echo 'changed ownership of /var/log/ditto to 1005:1005'" ]
           volumeMounts:
             - name: ditto-log-files-directory
               mountPath: /var/log/ditto
@@ -312,6 +312,8 @@ spec:
             {{- if .Values.thingsSearch.extraVolumeMounts }}
               {{- toYaml .Values.thingsSearch.extraVolumeMounts | nindent 12 }}
             {{- end }}
+            - name: ditto-heap-dumps
+              mountPath: /opt/ditto/dumps
           resources:
             requests:
               cpu: {{ mulf .Values.thingsSearch.resources.cpu 1000 }}m
@@ -361,4 +363,6 @@ spec:
         {{- if .Values.thingsSearch.extraVolumes }}
           {{- toYaml .Values.thingsSearch.extraVolumes | nindent 8 }}
         {{- end}}
+        - name: ditto-heap-dumps
+          emptyDir: {}
 {{- end }}

deployment/helm/ditto/values.yaml

@@ -104,6 +104,8 @@ global:
   # jvmOptions defines the JVM options applied to all Ditto services running in the JVM, it is put in JAVA_TOOL_OPTIONS
   jvmOptions: >
     -XX:+ExitOnOutOfMemoryError
+    -XX:+HeapDumpOnOutOfMemoryError
+    -XX:HeapDumpPath=/opt/ditto/dumps/oom.bin
     -XX:+UseContainerSupport
     -XX:+UseStringDeduplication
     -Xss512k
@@ -652,7 +654,7 @@ policies:
     cleanup:
       # enabled configures whether background cleanup is enabled or not
       #  if enabled, stale "snapshot" and "journal" entries will be cleaned up from the MongoDB by a background process:
-      enabled: false
+      enabled: true
       # quietPeriod defines how long to stay in a state where the background cleanup is not yet started
       quietPeriod: 5m
       # history contains configuration regarding the event history
@@ -671,7 +673,18 @@ policies:
       # timerThreshold configures the maximum database latency to give out credit for cleanup actions
       timerThreshold: 100ms
       # creditsPerBatch configures how many "cleanup credits" should be generated per "interval" as long as the
+      #  write operations to the MongoDB are less than the configured `timerThreshold`.
+      #  Limits the rate of cleanup actions to this many per credit decision interval.
+      #  One credit means that the "journal" and "snapshot" entries of one entity are cleaned up each `interval`.
       creditsPerBatch: 5
+      # readsPerQuery configures the number of snapshots to scan per MongoDB query.
+      #  Configuring this to high values will reduce the need to query MongoDB too often - it should however be aligned
+      #  with the amount of `creditsPerBatch` issued per `interval` - in order to avoid long running queries.
+      readsPerQuery: 100
+      # readsPerQuery configures the number of documents to delete for each credit.
+      #  If for example one entity would have 1000 journal entries to cleanup, a `writes-per-credit` of 100 would lead
+      #  to 10 delete operations performed against MongoDB.
+      writesPerCredit: 100
       # deleteFinalDeletedSnapshot configures whether for a deleted entity, the final snapshot (containing the
       #  "deleted" information) should be deleted or not.
       #  If the final snapshot is not deleted, re-creating the entity will cause that the recreated entity starts with
@@ -741,6 +754,18 @@ policies:
       revokes: []
       # - namespaces: []
       #   authSubjects: []
+    # policiesEnforcer contains configuration for Ditto "Policy Enforcers", e.g. regarding caching
+    policiesEnforcer:
+      # cache holds the configuration of policy enforcer caching
+      cache:
+        # enabled whether caching of policy enforcers should be enabled
+        enabled: true
+        # maxSize the maximum size of policy enforcers to keep in the cache
+        maxSize: 1000
+        # expireAfterWrite the maximum duration of inconsistency after losing a cache invalidation
+        expireAfterWrite: 4h
+        # expireAfterAccess prolonged on each cache access by that duration
+        expireAfterAccess: 2h
 
 ## ----------------------------------------------------------------------------
 ## things configuration
@@ -912,7 +937,18 @@ things:
       # timerThreshold configures the maximum database latency to give out credit for cleanup actions
       timerThreshold: 100ms
       # creditsPerBatch configures how many "cleanup credits" should be generated per "interval" as long as the
+      #  write operations to the MongoDB are less than the configured `timerThreshold`.
+      #  Limits the rate of cleanup actions to this many per credit decision interval.
+      #  One credit means that the "journal" and "snapshot" entries of one entity are cleaned up each `interval`.
       creditsPerBatch: 5
+      # readsPerQuery configures the number of snapshots to scan per MongoDB query.
+      #  Configuring this to high values will reduce the need to query MongoDB too often - it should however be aligned
+      #  with the amount of `creditsPerBatch` issued per `interval` - in order to avoid long running queries.
+      readsPerQuery: 100
+      # readsPerQuery configures the number of documents to delete for each credit.
+      #  If for example one entity would have 1000 journal entries to cleanup, a `writes-per-credit` of 100 would lead
+      #  to 10 delete operations performed against MongoDB.
+      writesPerCredit: 100
       # deleteFinalDeletedSnapshot configures whether for a deleted entity, the final snapshot (containing the
       #  "deleted" information) should be deleted or not.
       #  If the final snapshot is not deleted, re-creating the entity will cause that the recreated entity starts with
@@ -983,11 +1019,11 @@ things:
         # enabled whether caching of policy enforcers should be enabled
         enabled: true
         # maxSize the maximum size of policy enforcers to keep in the cache
-        maxSize: 50000
+        maxSize: 2000
         # expireAfterWrite the maximum duration of inconsistency after losing a cache invalidation
-        expireAfterWrite: 8h
+        expireAfterWrite: 4h
         # expireAfterAccess prolonged on each cache access by that duration
-        expireAfterAccess: 4h
+        expireAfterAccess: 2h
     # wot contains Web of Things (WoT) specific configuration
     wot:
       # tdBasePrefix is the base to use where the Ditto endpoint is located in order to be injected into TDs:
@@ -1258,7 +1294,7 @@ thingsSearch:
         # thingCache configures the cache configuration for caching of things in things-search
         thingCache:
           # maxSize defines how many things to cache
-          maxSize: 30000
+          maxSize: 3000
           # expireAfterWrite defines how long at most to keep things in the cache after loading them into the cache
           expireAfterWrite: 12h
           # expireAfterWrite defines how long at most to keep things in the cache after last accessing them from the cache
@@ -1470,14 +1506,14 @@ connectivity:
         # maxSize the maximum size of policy enforcers to keep in the cache
         maxSize: 1000
         # expireAfterWrite the maximum duration of inconsistency after losing a cache invalidation
-        expireAfterWrite: 8h
+        expireAfterWrite: 4h
         # expireAfterAccess prolonged on each cache access by that duration
-        expireAfterAccess: 4h
+        expireAfterAccess: 2h
     # cleanup contains the configuration for the background cleanup of stale snapshots and events
     cleanup:
       # enabled configures whether background cleanup is enabled or not
       #  if enabled, stale "snapshot" and "journal" entries will be cleaned up from the MongoDB by a background process:
-      enabled: false
+      enabled: true
       # quietPeriod defines how long to stay in a state where the background cleanup is not yet started
       quietPeriod: 5m
       # history contains configuration regarding the event history
@@ -1496,7 +1532,18 @@ connectivity:
       # timerThreshold configures the maximum database latency to give out credit for cleanup actions
       timerThreshold: 100ms
       # creditsPerBatch configures how many "cleanup credits" should be generated per "interval" as long as the
-      creditsPerBatch: 5
+      #  write operations to the MongoDB are less than the configured `timerThreshold`.
+      #  Limits the rate of cleanup actions to this many per credit decision interval.
+      #  One credit means that the "journal" and "snapshot" entries of one entity are cleaned up each `interval`.
+      creditsPerBatch: 1
+      # readsPerQuery configures the number of snapshots to scan per MongoDB query.
+      #  Configuring this to high values will reduce the need to query MongoDB too often - it should however be aligned
+      #  with the amount of `creditsPerBatch` issued per `interval` - in order to avoid long running queries.
+      readsPerQuery: 100
+      # readsPerQuery configures the number of documents to delete for each credit.
+      #  If for example one entity would have 1000 journal entries to cleanup, a `writes-per-credit` of 100 would lead
+      #  to 10 delete operations performed against MongoDB.
+      writesPerCredit: 100
       # deleteFinalDeletedSnapshot configures whether for a deleted entity, the final snapshot (containing the
       #  "deleted" information) should be deleted or not.
       #  If the final snapshot is not deleted, re-creating the entity will cause that the recreated entity starts with

documentation/src/main/resources/Gemfile.lock

@@ -11,7 +11,6 @@ GEM
     eventmachine (1.2.7)
     ffi (1.15.5)
     forwardable-extended (2.6.0)
-    google-protobuf (3.22.0)
     google-protobuf (3.22.0-arm64-darwin)
     http_parser.rb (0.8.0)
     i18n (1.12.0)
@@ -53,15 +52,11 @@ GEM
     rb-fsevent (0.11.2)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
-    rexml (3.3.6)
-      strscan
+    rexml (3.2.5)
     rouge (4.1.0)
     safe_yaml (1.0.5)
     sass-embedded (1.58.3-arm64-darwin)
       google-protobuf (~> 3.21)
-    sass-embedded (1.58.3-x86_64-linux-gnu)
-      google-protobuf (~> 3.21)
-    strscan (3.1.0)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
     unicode-display_width (2.4.2)