Merge pull request #1611 from eclipse-ditto/1593-merge-patch-key-func…

…tions

#1593 support removing existing fields from a JSON object in a merge …

documentation/src/main/resources/pages/ditto/httpapi-concepts.md

@@ -393,6 +393,82 @@ interpreted as delete in contrast to `PUT` requests where `null` values have no
 Like `PUT` requests, `PATCH` requests can be applied at any level of the JSON structure of a thing, e.g. patching a
 complete thing at root level or patching a single property value at property level.
 
+### Removing fields in a merge update with a regex
+
+{% include note.html content="This is an addition to the JSON merge patch (RFC-7396), enhancing using `null` values
+    for deleting certain parts of JSON objects specified with a regular expression before applying new fields to it." %}
+
+The merge patch functionality in Ditto solves a common problem to the JSON merge patch (RFC-7396): whenever a JSON object
+shall be patched, all the old json fields are merged with all the new json fields, unless the exact field names are 
+specified in the patch with `"<field>": null`.  
+This would however require to know all existing fields upfront, which for a merge patch can not be assumed.
+
+The solution is a little enhancement to Ditto's merge patch functionality: The ability to delete arbitrary parts from
+JSON objects using a regular expression **before** applying all other patch values.
+
+The syntax for this function is rather specific (so that no "normally" occurring JSON keys match the same syntax):
+```json
+{
+  "{%raw%}{{ /.*/ }}{%endraw%}": null
+}
+```
+
+When such a `{%raw%}{{ /<regex>/ }}{%endraw%}` with the value `null` is detected in the merge patch, the content between the 2 `/` is 
+interpreted as regular expression to apply for finding keys to delete from the target object.  
+As a result, using `"{%raw%}{{ /.*/ }}{%endraw%}": null` would delete all the values inside a JSON object before applying the new 
+values provided in the patch.
+
+Example:  
+Assuming that inside a JSON object every month some aggregated data is stored with the year and month:  
+```json
+{
+  "thingId": "{thingId}",
+  "policyId": "{policyId}",
+  "features": {
+    "aggregated-history": {
+      "properties": {
+        "2022-11": 42.3,
+        "2022-12": 54.3,
+        "2023-01": 80.2,
+        "2023-02": 99.9
+      }
+    }
+  }
+}
+```
+
+Then the data from "last year" could be purged with the following patch, while adding a new value to the existing ones
+of this year:
+```json
+{
+  "features": {
+    "aggregated-history": {
+      "properties": {
+        "{%raw%}{{ /2022-.*/ }}{%endraw%}": null,
+        "2023-03": 105.21
+      }
+    }
+  }
+}
+```
+
+The resulting Thing JSON after applying the patch would then look like:
+```json
+{
+  "thingId": "{thingId}",
+  "policyId": "{policyId}",
+  "features": {
+    "aggregated-history": {
+      "properties": {
+        "2023-01": 80.2,
+        "2023-02": 99.9,
+        "2023-03": 105.21
+      }
+    }
+  }
+}
+```
+
 ### Permissions required for merge update
 
 To successfully execute merge update the authorized subject needs to have *WRITE* permission on *all* resources 

json/src/main/java/org/eclipse/ditto/json/JsonMergePatch.java

@@ -12,9 +12,11 @@
  */
 package org.eclipse.ditto.json;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import javax.annotation.Nullable;
@@ -165,16 +167,43 @@ private static JsonObject mergeJsonObjects(final JsonObject jsonObject1, final J
             }
         });
 
+        final List<JsonKey> toBeNulledKeysByRegex = determineToBeNulledKeysByRegex(jsonObject1, jsonObject2);
+
         // add fields of jsonObject2 not present in jsonObject1
         jsonObject2.forEach(jsonField -> {
-            if (!jsonObject1.contains(jsonField.getKey())) {
+            if (!jsonObject1.contains(jsonField.getKey()) && !toBeNulledKeysByRegex.contains(jsonField.getKey())) {
                 builder.set(jsonField);
             }
         });
 
         return builder.build();
     }
 
+    private static List<JsonKey> determineToBeNulledKeysByRegex(
+            final JsonObject jsonObject1,
+            final JsonObject jsonObject2) {
+
+        final List<JsonKey> toBeNulledKeysByRegex = new ArrayList<>();
+        final List<JsonKey> keyRegexes = jsonObject1.getKeys().stream()
+                .filter(key -> key.toString().startsWith("{{") && key.toString().endsWith("}}"))
+                .collect(Collectors.toList());
+        keyRegexes.forEach(keyRegex -> {
+            final String keyRegexWithoutCurly = keyRegex.toString().substring(2, keyRegex.length() - 2).trim();
+            if (keyRegexWithoutCurly.startsWith("/") && keyRegexWithoutCurly.endsWith("/")) {
+                final String regexStr = keyRegexWithoutCurly.substring(1, keyRegexWithoutCurly.length() - 1);
+                final Pattern pattern = Pattern.compile(regexStr);
+                jsonObject1.getValue(keyRegex)
+                        .filter(JsonValue::isNull) // only support deletion via regex, so only support "null" values
+                        .ifPresent(keyRegexValue ->
+                                jsonObject2.getKeys().stream()
+                                        .filter(key -> pattern.matcher(key).matches())
+                                        .forEach(toBeNulledKeysByRegex::add)
+                );
+            }
+        });
+        return toBeNulledKeysByRegex;
+    }
+
     /**
      * Applies this merge patch on the given json value.
      *

json/src/test/java/org/eclipse/ditto/json/JsonMergePatchTest.java

@@ -447,4 +447,81 @@ public void mergeFieldsFromBothObjectsRFC7396_TestCase13() {
         Assertions.assertThat(mergedObject).isEqualTo(expectedObject);
     }
 
+    @Test
+    public void removeFieldsUsingRegexWithNullValue() {
+        final JsonObject originalObject = JsonFactory.newObjectBuilder()
+                .set("a", JsonFactory.newObjectBuilder()
+                        .set("2023-04-01", JsonValue.of(true))
+                        .set("2023-04-02", JsonValue.of("hello"))
+                        .set("2023-04-03", JsonValue.of("darkness"))
+                        .set("2023-04-04", JsonValue.of("my"))
+                        .set("2023-04-05", JsonValue.of("old"))
+                        .set("2023-04-06", JsonValue.of("friend"))
+                        .build())
+                .set("b", JsonFactory.newObjectBuilder()
+                        .set("2023-04-01", JsonValue.of(true))
+                        .set("2023-04-02", JsonValue.of("hello"))
+                        .set("2023-04-03", JsonValue.of("darkness"))
+                        .set("2023-04-04", JsonValue.of("my"))
+                        .set("2023-04-05", JsonValue.of("old"))
+                        .set("2023-04-06", JsonValue.of("friend"))
+                        .build())
+                .set("c", JsonFactory.newObjectBuilder()
+                        .set("some", JsonValue.of(true))
+                        .set("2023-04-02", JsonValue.of("hello"))
+                        .set("totally-other", JsonValue.of("darkness"))
+                        .set("foo", JsonValue.of("my"))
+                        .build())
+                .build();
+
+        final JsonObject objectToPatch = JsonFactory.newObjectBuilder()
+                .set("a", JsonFactory.newObjectBuilder()
+                        .set("{{ /2023-04-.*/ }}", JsonValue.nullLiteral())
+                        .set("2023-05-01", JsonValue.of("new"))
+                        .set("2023-05-02", JsonValue.of("catch"))
+                        .set("2023-05-03", JsonValue.of("phrase"))
+                        .build())
+                .set("b", JsonFactory.newObjectBuilder()
+                        .set("{{ /2023-04-01/ }}", JsonValue.nullLiteral())
+                        .set("{{ /^2023-04-03$/ }}", JsonValue.nullLiteral())
+                        .set("{{ /[0-9]{4}-04-.+4/ }}", JsonValue.nullLiteral())
+                        .set("2023-05-01", JsonValue.of("new"))
+                        .set("2023-05-02", JsonValue.of("catch"))
+                        .set("2023-05-03", JsonValue.of("phrase"))
+                        .build())
+                .set("c", JsonFactory.newObjectBuilder()
+                        .set("{{ /.*/ }}", JsonValue.nullLiteral())
+                        .set("2023-05-01", JsonValue.of("new"))
+                        .set("2023-05-02", JsonValue.of("catch"))
+                        .set("2023-05-03", JsonValue.of("phrase"))
+                        .build())
+                .build();
+
+        final JsonValue expectedObject = JsonFactory.newObjectBuilder()
+                .set("a", JsonFactory.newObjectBuilder()
+                        .set("2023-05-01", JsonValue.of("new"))
+                        .set("2023-05-02", JsonValue.of("catch"))
+                        .set("2023-05-03", JsonValue.of("phrase"))
+                        .build())
+                .set("b", JsonFactory.newObjectBuilder()
+                        .set("2023-04-02", JsonValue.of("hello"))
+                        .set("2023-04-05", JsonValue.of("old"))
+                        .set("2023-04-06", JsonValue.of("friend"))
+                        .set("2023-05-01", JsonValue.of("new"))
+                        .set("2023-05-02", JsonValue.of("catch"))
+                        .set("2023-05-03", JsonValue.of("phrase"))
+                        .build())
+                .set("c", JsonFactory.newObjectBuilder()
+                        .set("2023-05-01", JsonValue.of("new"))
+                        .set("2023-05-02", JsonValue.of("catch"))
+                        .set("2023-05-03", JsonValue.of("phrase"))
+                        .build())
+                .build();
+
+        final JsonValue mergedObject = JsonMergePatch.of(objectToPatch).applyOn(originalObject);
+
+        Assertions.assertThat(mergedObject).isEqualTo(expectedObject);
+    }
+
+
 }

things/service/src/main/java/org/eclipse/ditto/things/service/enforcement/ThingCommandEnforcement.java

@@ -37,6 +37,7 @@
 import org.eclipse.ditto.internal.utils.cacheloaders.config.AskWithRetryConfig;
 import org.eclipse.ditto.json.JsonFactory;
 import org.eclipse.ditto.json.JsonFieldSelector;
+import org.eclipse.ditto.json.JsonKey;
 import org.eclipse.ditto.json.JsonObject;
 import org.eclipse.ditto.json.JsonPointer;
 import org.eclipse.ditto.json.JsonPointerInvalidException;
@@ -503,7 +504,15 @@ private static boolean enforceMergeThingCommand(final Enforcer enforcer,
     private static Set<ResourceKey> calculateLeaves(final JsonPointer path, final JsonValue value) {
         if (value.isObject()) {
             return value.asObject().stream()
-                    .map(f -> calculateLeaves(path.append(f.getKey().asPointer()), f.getValue()))
+                    .map(f -> {
+                        final JsonKey key = f.getKey();
+                        if (isMergeWithNulledKeysByRegex(key, f.getValue())) {
+                            // if regex is contained, writing all below that path must be granted in the policy:
+                            return Set.of(PoliciesResourceType.thingResource(path));
+                        } else {
+                            return calculateLeaves(path.append(key.asPointer()), f.getValue());
+                        }
+                    })
                     .reduce(new HashSet<>(), ThingCommandEnforcement::addAll, ThingCommandEnforcement::addAll);
         } else {
             return Set.of(PoliciesResourceType.thingResource(path));
@@ -515,4 +524,15 @@ private static Set<ResourceKey> addAll(final Set<ResourceKey> result, final Set<
         return result;
     }
 
+    private static boolean isMergeWithNulledKeysByRegex(final JsonKey key, final JsonValue value) {
+        final String keyString = key.toString();
+        if (keyString.startsWith("{{") && keyString.endsWith("}}")) {
+            final String keyRegexWithoutCurly = keyString.substring(2, keyString.length() - 2).trim();
+            return keyRegexWithoutCurly.startsWith("/") && keyRegexWithoutCurly.endsWith("/") &&
+                    value.isNull();
+        } else {
+            return false;
+        }
+    }
+
 }

things/service/src/test/java/org/eclipse/ditto/things/service/enforcement/MergeThingCommandEnforcementTest.java

@@ -32,6 +32,7 @@
 import org.eclipse.ditto.json.JsonFactory;
 import org.eclipse.ditto.json.JsonObject;
 import org.eclipse.ditto.json.JsonPointer;
+import org.eclipse.ditto.json.JsonValue;
 import org.eclipse.ditto.policies.model.EffectedPermissions;
 import org.eclipse.ditto.policies.model.PoliciesModelFactory;
 import org.eclipse.ditto.policies.model.Policy;
@@ -41,6 +42,7 @@
 import org.eclipse.ditto.things.api.Permission;
 import org.eclipse.ditto.things.model.signals.commands.exceptions.ThingNotModifiableException;
 import org.eclipse.ditto.things.model.signals.commands.modify.MergeThing;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
@@ -119,6 +121,44 @@ void rejectByPolicy(final TestArgument arg) {
                 () -> ThingCommandEnforcement.authorizeByPolicyOrThrow(policyEnforcer, arg.getMergeThing()));
     }
 
+    @Test
+    void acceptUsingRegexInPolicy() {
+        final TestArgument testArgument = TestArgument.of("/",
+                Set.of("/", "/attributes/complex"),
+                Set.of("/attributes/simple")
+        );
+        final TrieBasedPolicyEnforcer policyEnforcer = TrieBasedPolicyEnforcer.newInstance(testArgument.getPolicy());
+        final JsonObject patch = JsonObject.newBuilder()
+                .set("attributes", JsonObject.newBuilder()
+                        .set("complex", JsonObject.newBuilder()
+                                .set("{{ /.*/ }}", JsonValue.nullLiteral())
+                                .build()
+                        )
+                        .set("another", 42)
+                        .build())
+                .build();
+        final MergeThing mergeThing = MergeThing.of(TestSetup.THING_ID, JsonPointer.empty(), patch, TestArgument.headers());
+        final MergeThing authorizedMergeThing =
+                ThingCommandEnforcement.authorizeByPolicyOrThrow(policyEnforcer, mergeThing);
+        assertThat(authorizedMergeThing.getDittoHeaders().getAuthorizationContext()).isNotNull();
+    }
+
+    @Test
+    void rejectUsingRegexWithRevokesInPolicy() {
+        final TestArgument testArgument = TestArgument.of("/", Set.of("/"), Set.of("/attributes/complex/nested/secret"));
+        final TrieBasedPolicyEnforcer policyEnforcer = TrieBasedPolicyEnforcer.newInstance(testArgument.getPolicy());
+        final JsonObject patch = JsonObject.newBuilder()
+                .set("attributes", JsonObject.newBuilder()
+                        .set("{{ /.*/ }}", JsonValue.nullLiteral())
+                        .set("simple", "value")
+                        .set("another", 42)
+                        .build())
+                .build();
+        final MergeThing mergeThing = MergeThing.of(TestSetup.THING_ID, JsonPointer.empty(), patch, TestArgument.headers());
+        assertThatExceptionOfType(ThingNotModifiableException.class).isThrownBy(
+                () -> ThingCommandEnforcement.authorizeByPolicyOrThrow(policyEnforcer, mergeThing));
+    }
+
     /**
      * Generates combinations of a Policy and MergeThing commands that should be <b>accepted</b> by command enforcement.
      */
@@ -244,7 +284,7 @@ private static MergeThing toMergeCommand(final JsonPointer path, final JsonObjec
             return MergeThing.of(TestSetup.THING_ID, path, patch.getValue(path).orElseThrow(), headers());
         }
 
-        private static DittoHeaders headers() {
+        static DittoHeaders headers() {
             return DittoHeaders.newBuilder()
                     .authorizationContext(
                             AuthorizationContext.newInstance(DittoAuthorizationContextType.UNSPECIFIED,