From 3b58257b4fab993b507a221b9f67dad9851805de Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Sat, 27 Aug 2022 21:02:34 -0700 Subject: [PATCH 01/31] Initial commit --- .../cosmos/models/ClientEncryptionPolicy.java | 67 +++++-- .../models/CosmosContainerProperties.java | 4 +- .../com/azure/cosmos/CosmosContainerTest.java | 170 ++++++++++++++++-- 3 files changed, 210 insertions(+), 31 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java index 44aefa16cf2e8..7dac6485b7f67 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java @@ -40,11 +40,27 @@ public final class ClientEncryptionPolicy { */ @Beta(value = Beta.SinceVersion.V4_14_0, warningText = Beta.PREVIEW_SUBJECT_TO_CHANGE_WARNING) public ClientEncryptionPolicy(List paths) { - this.validateIncludedPaths(paths); + this.validateIncludedPaths(paths, policyFormatVersion); this.includedPaths = paths; this.policyFormatVersion = 1; } + /** + * Constructor. + * + * @param paths list of path of the item that need encryption along with path-specific settings. + * @param policyFormatVersion version of the client encryption policy definition. Current supported versions are 1 and 2. Default version is 1. + */ + @Beta(value = Beta.SinceVersion.V4_14_0, warningText = Beta.PREVIEW_SUBJECT_TO_CHANGE_WARNING) + public ClientEncryptionPolicy(List paths, int policyFormatVersion) { + if (policyFormatVersion > 2 || policyFormatVersion < 1) { + throw new IllegalArgumentException("Supported versions of client encryption policy are 1 and 2."); + } + this.policyFormatVersion = policyFormatVersion; + this.validateIncludedPaths(paths, policyFormatVersion); + this.includedPaths = paths; + } + /** * Constructor. */ @@ -71,7 +87,7 @@ public ClientEncryptionPolicy() { } /** - * Gets the list of path of the item that need encryption along with path-specific settings. + * Gets the list of paths of the item that need encryption along with path-specific settings. * @return includedPaths */ @Beta(value = Beta.SinceVersion.V4_11_0, warningText = Beta.PREVIEW_SUBJECT_TO_CHANGE_WARNING) @@ -88,28 +104,44 @@ public int getPolicyFormatVersion() { return policyFormatVersion; } - void validatePartitionKeyPathsAreNotEncrypted(List> partitionKeyPathTokens) { + /** + * Ensures that partition key paths specified in the client encryption policy for encryption are encrypted using Deterministic encryption algorithm. + * @param partitionKeyPathTokens Tokens corresponding to validated partition key. + */ + void validatePartitionKeyPathsIfEncrypted(List> partitionKeyPathTokens) { checkNotNull(partitionKeyPathTokens, "partitionKeyPathTokens cannot be null"); - List propertiesToEncrypt = - this.includedPaths.stream().map(clientEncryptionIncludedPath -> clientEncryptionIncludedPath.getPath().substring(1)).collect(Collectors.toList()); + for (List tokensInPath : partitionKeyPathTokens) { checkNotNull(tokensInPath); if (tokensInPath.size() > 0) { String topLevelToken = tokensInPath.get(0); - if (propertiesToEncrypt.contains(topLevelToken)) { + + // paths in included paths start with "/". Get the ClientEncryptionIncludedPath and validate. + List encrypterPartitionKeyPath = + this.includedPaths.stream().filter(clientEncryptionIncludedPath -> clientEncryptionIncludedPath.getPath().substring(1).equals(topLevelToken)).collect(Collectors.toList()); + + if (encrypterPartitionKeyPath.size() >0) { + if (this.policyFormatVersion < 2) { + throw new IllegalArgumentException(String.format("Path %s which is part of the partition key " + + "cannot be encrypted" + + " with PolicyFormatVersion %s", topLevelToken, policyFormatVersion)); + } + + // for the ClientEncryptionIncludedPath found check the encryption type. + if (encrypterPartitionKeyPath.stream().map(encrypter -> encrypter.getEncryptionType()).findFirst().orElse(null) != "Deterministic") throw new IllegalArgumentException(String.format("Path %s which is part of the partition key " + - "cannot be included" + - " in the ClientEncryptionPolicy.", topLevelToken)); + "has to be encrypted" + + " with Deterministic type Encryption.", topLevelToken)); } } } } - private void validateIncludedPaths(List clientEncryptionIncludedPath) { + private void validateIncludedPaths(List clientEncryptionIncludedPath, int policyFormatVersion) { List includedPathsList = new ArrayList<>(); for (ClientEncryptionIncludedPath path : clientEncryptionIncludedPath) { - this.validateClientEncryptionIncludedPath(path); + this.validateClientEncryptionIncludedPath(path, policyFormatVersion); if (includedPathsList.contains(path.getPath())) { throw new IllegalArgumentException("Duplicate Path found in clientEncryptionIncludedPath."); } @@ -118,7 +150,7 @@ private void validateIncludedPaths(List clientEncr } } - private void validateClientEncryptionIncludedPath(ClientEncryptionIncludedPath clientEncryptionIncludedPath) { + private void validateClientEncryptionIncludedPath(ClientEncryptionIncludedPath clientEncryptionIncludedPath, int policyFormatVersion) { if (clientEncryptionIncludedPath == null) { throw new IllegalArgumentException("clientEncryptionIncludedPath is null"); } @@ -128,11 +160,20 @@ private void validateClientEncryptionIncludedPath(ClientEncryptionIncludedPath c } if (clientEncryptionIncludedPath.getPath().charAt(0) != '/' - || clientEncryptionIncludedPath.getPath().lastIndexOf('/') != 0 - || clientEncryptionIncludedPath.getPath().substring(1).equals("id")) { + || clientEncryptionIncludedPath.getPath().lastIndexOf('/') != 0) { throw new IllegalArgumentException("Invalid path " + clientEncryptionIncludedPath.getPath()); } + if (clientEncryptionIncludedPath.getPath().substring(1).equals("id")) { + if (policyFormatVersion < 2) { + throw new IllegalArgumentException(String.format("Path %s cannot be encrypted with policyFormatVersion %s.", clientEncryptionIncludedPath.getPath(), policyFormatVersion)); + } + + if (clientEncryptionIncludedPath.getEncryptionType() != "Deterministic") { + throw new IllegalArgumentException(String.format("Only deterministic encryption type is supported for path %s.", clientEncryptionIncludedPath.getPath())); + } + } + if (StringUtils.isEmpty(clientEncryptionIncludedPath.getClientEncryptionKeyId())) { throw new IllegalArgumentException("clientEncryptionKeyId in clientEncryptionIncludedPath is empty"); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/CosmosContainerProperties.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/CosmosContainerProperties.java index 03f85f9eee9ab..9e818d2c60f4a 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/CosmosContainerProperties.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/CosmosContainerProperties.java @@ -126,7 +126,7 @@ public PartitionKeyDefinition getPartitionKeyDefinition() { public CosmosContainerProperties setPartitionKeyDefinition(PartitionKeyDefinition partitionKeyDefinition) { this.documentCollection.setPartitionKey(partitionKeyDefinition); if (this.getClientEncryptionPolicy() != null) { - this.getClientEncryptionPolicy().validatePartitionKeyPathsAreNotEncrypted(this.getPartitionKeyPathTokensList()); + this.getClientEncryptionPolicy().validatePartitionKeyPathsIfEncrypted(this.getPartitionKeyPathTokensList()); } return this; @@ -317,7 +317,7 @@ public ClientEncryptionPolicy getClientEncryptionPolicy() { @Beta(value = Beta.SinceVersion.V4_14_0, warningText = Beta.PREVIEW_SUBJECT_TO_CHANGE_WARNING) public CosmosContainerProperties setClientEncryptionPolicy(ClientEncryptionPolicy value) { if (value != null) { - value.validatePartitionKeyPathsAreNotEncrypted(this.getPartitionKeyPathTokensList()); + value.validatePartitionKeyPathsIfEncrypted(this.getPartitionKeyPathTokensList()); } this.documentCollection.setClientEncryptionPolicy(value); diff --git a/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerTest.java b/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerTest.java index 00e5b8856a22d..924718774a06b 100644 --- a/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerTest.java +++ b/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerTest.java @@ -121,16 +121,23 @@ public void createContainer_withEncryption() { path1.setClientEncryptionKeyId("containerTestKey1"); ClientEncryptionIncludedPath path2 = new ClientEncryptionIncludedPath(); - path2.setPath("/path2"); + path2.setPath("/mypk"); path2.setEncryptionAlgorithm("AEAD_AES_256_CBC_HMAC_SHA256"); path2.setEncryptionType("Deterministic"); path2.setClientEncryptionKeyId("containerTestKey2"); + ClientEncryptionIncludedPath path3 = new ClientEncryptionIncludedPath(); + path3.setPath("/id"); + path3.setEncryptionAlgorithm("AEAD_AES_256_CBC_HMAC_SHA256"); + path3.setEncryptionType("Deterministic"); + path3.setClientEncryptionKeyId("containerTestKey2"); + List paths = new ArrayList<>(); paths.add(path1); paths.add(path2); + paths.add(path3); - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths,2); containerProperties.setClientEncryptionPolicy(clientEncryptionPolicy); CosmosContainerResponse containerResponse = createdDatabase.createContainer(containerProperties); @@ -139,14 +146,14 @@ public void createContainer_withEncryption() { } @Test(groups = {"emulator"}, timeOut = TIMEOUT) - public void createContainer_withPartitionKeyInEncryption() { + public void createContainer_withPartitionKeyWrongPolicyFormatVersion() { String collectionName = UUID.randomUUID().toString(); CosmosContainerProperties containerProperties = getCollectionDefinition(collectionName); ClientEncryptionIncludedPath path1 = new ClientEncryptionIncludedPath(); path1.setPath("/mypk"); path1.setEncryptionAlgorithm("AEAD_AES_256_CBC_HMAC_SHA256"); - path1.setEncryptionType("Randomized"); + path1.setEncryptionType("Deterministic"); path1.setClientEncryptionKeyId("containerTestKey1"); ClientEncryptionIncludedPath path2 = new ClientEncryptionIncludedPath(); @@ -166,11 +173,114 @@ public void createContainer_withPartitionKeyInEncryption() { try { containerProperties.setClientEncryptionPolicy(clientEncryptionPolicy); containerResponse = createdDatabase.createContainer(containerProperties); - fail("createContainer should fail as mypk which is part of the partition key cannot be included in the " + - "ClientEncryptionPolicy."); + fail("createContainer should fail as mypk which is part of the partition key cannot be encrypted with " + + "PolicyFormatVersion 1."); + } catch (IllegalArgumentException ex) { + assertThat(ex.getMessage()).isEqualTo("Path mypk which is part of the partition key cannot be encrypted with PolicyFormatVersion 1"); + } + + + //Verify for composite key + collectionName = UUID.randomUUID().toString(); + containerProperties = new CosmosContainerProperties(collectionName, "/mypk/mypk1"); + try { + containerProperties.setClientEncryptionPolicy(clientEncryptionPolicy); + containerResponse = createdDatabase.createContainer(containerProperties); + fail("createContainer should fail as mypk which is part of the partition key cannot be encrypted with " + + "PolicyFormatVersion 1."); + } catch (IllegalArgumentException ex) { + assertThat(ex.getMessage()).isEqualTo("Path mypk which is part of the partition key cannot be encrypted with PolicyFormatVersion 1"); + } + + + //Verify setPartitionKeyDefinition with encrypted field. + collectionName = UUID.randomUUID().toString(); + containerProperties = new CosmosContainerProperties(collectionName, "/differentKey"); + try { + containerProperties.setClientEncryptionPolicy(clientEncryptionPolicy); + PartitionKeyDefinition partitionKeyDefinition = new PartitionKeyDefinition(); + List keyPaths = new ArrayList<>(); + keyPaths.add("/mypk"); + partitionKeyDefinition.setPaths(keyPaths); + containerProperties.setPartitionKeyDefinition(partitionKeyDefinition); + containerResponse = createdDatabase.createContainer(containerProperties); + fail("createContainer should fail as mypk which is part of the partition key cannot be encrypted with " + + "PolicyFormatVersion 1."); + } catch (IllegalArgumentException ex) { + assertThat(ex.getMessage()).isEqualTo("Path mypk which is part of the partition key cannot be encrypted with PolicyFormatVersion 1"); + } + + //This should pass as we check only the first key of the composite key. + collectionName = UUID.randomUUID().toString(); + containerProperties = new CosmosContainerProperties(collectionName, "/mypk1/mypk"); + containerProperties.setClientEncryptionPolicy(clientEncryptionPolicy); + containerResponse = createdDatabase.createContainer(containerProperties); + assertThat(containerResponse.getRequestCharge()).isGreaterThan(0); + validateContainerResponseWithEncryption(containerProperties, containerResponse, clientEncryptionPolicy); + } + + @Test(groups = {"emulator"}, timeOut = TIMEOUT) + public void createEncryptionPolicy_withIdWrongPolicyFormatVersion() { + String collectionName = UUID.randomUUID().toString(); + CosmosContainerProperties containerProperties = getCollectionDefinition(collectionName); + + ClientEncryptionIncludedPath path1 = new ClientEncryptionIncludedPath(); + path1.setPath("/id"); + path1.setEncryptionAlgorithm("AEAD_AES_256_CBC_HMAC_SHA256"); + path1.setEncryptionType("Deterministic"); + path1.setClientEncryptionKeyId("containerTestKey1"); + + ClientEncryptionIncludedPath path2 = new ClientEncryptionIncludedPath(); + path2.setPath("/path2"); + path2.setEncryptionAlgorithm("AEAD_AES_256_CBC_HMAC_SHA256"); + path2.setEncryptionType("Deterministic"); + path2.setClientEncryptionKeyId("containerTestKey2"); + + List paths = new ArrayList<>(); + paths.add(path1); + paths.add(path2); + + try { + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths); + fail("clientEncryptionPolicy should fail as id which is part of the partition key cannot be encrypted with " + + "PolicyFormatVersion 0."); + } catch (IllegalArgumentException ex) { + assertThat(ex.getMessage()).isEqualTo("Path /id cannot be encrypted with policyFormatVersion 0."); + } + } + + @Test(groups = {"emulator"}, timeOut = TIMEOUT) + public void createContainer_withPartitionKeyWrongEncryptionType() { + String collectionName = UUID.randomUUID().toString(); + CosmosContainerProperties containerProperties = getCollectionDefinition(collectionName); + + ClientEncryptionIncludedPath path1 = new ClientEncryptionIncludedPath(); + path1.setPath("/mypk"); + path1.setEncryptionAlgorithm("AEAD_AES_256_CBC_HMAC_SHA256"); + path1.setEncryptionType("Randomized"); + path1.setClientEncryptionKeyId("containerTestKey1"); + + ClientEncryptionIncludedPath path2 = new ClientEncryptionIncludedPath(); + path2.setPath("/path2"); + path2.setEncryptionAlgorithm("AEAD_AES_256_CBC_HMAC_SHA256"); + path2.setEncryptionType("Deterministic"); + path2.setClientEncryptionKeyId("containerTestKey2"); + + List paths = new ArrayList<>(); + paths.add(path1); + paths.add(path2); + + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 2); + CosmosContainerResponse containerResponse = null; + + //Verify partition key in CosmosContainerProperties constructor with encrypted field. + try { + containerProperties.setClientEncryptionPolicy(clientEncryptionPolicy); + containerResponse = createdDatabase.createContainer(containerProperties); + fail("createContainer should fail as mypk which is part of the partition key has to be encrypted with " + + "Deterministic type Encryption."); } catch (IllegalArgumentException ex) { - assertThat(ex.getMessage()).isEqualTo("Path mypk which is part of the partition key cannot be included in" + - " the ClientEncryptionPolicy."); + assertThat(ex.getMessage()).isEqualTo("Path mypk which is part of the partition key has to be encrypted with Deterministic type Encryption."); } @@ -180,11 +290,10 @@ public void createContainer_withPartitionKeyInEncryption() { try { containerProperties.setClientEncryptionPolicy(clientEncryptionPolicy); containerResponse = createdDatabase.createContainer(containerProperties); - fail("createContainer should fail as mypk which is part of the partition key cannot be included in the " + - "ClientEncryptionPolicy."); + fail("createContainer should fail as mypk which is part of the partition key has to be encrypted with " + + "Deterministic type Encryption."); } catch (IllegalArgumentException ex) { - assertThat(ex.getMessage()).isEqualTo("Path mypk which is part of the partition key cannot be included in" + - " the ClientEncryptionPolicy."); + assertThat(ex.getMessage()).isEqualTo("Path mypk which is part of the partition key has to be encrypted with Deterministic type Encryption."); } @@ -199,11 +308,10 @@ public void createContainer_withPartitionKeyInEncryption() { partitionKeyDefinition.setPaths(keyPaths); containerProperties.setPartitionKeyDefinition(partitionKeyDefinition); containerResponse = createdDatabase.createContainer(containerProperties); - fail("createContainer should fail as mypk which is part of the partition key cannot be included in the " + - "ClientEncryptionPolicy."); + fail("createContainer should fail as mypk which is part of the partition key has to be encrypted with " + + "Deterministic type Encryption."); } catch (IllegalArgumentException ex) { - assertThat(ex.getMessage()).isEqualTo("Path mypk which is part of the partition key cannot be included in" + - " the ClientEncryptionPolicy."); + assertThat(ex.getMessage()).isEqualTo("Path mypk which is part of the partition key has to be encrypted with Deterministic type Encryption."); } //This should pass as we check only the first key of the composite key. @@ -215,6 +323,36 @@ public void createContainer_withPartitionKeyInEncryption() { validateContainerResponseWithEncryption(containerProperties, containerResponse, clientEncryptionPolicy); } + @Test(groups = {"emulator"}, timeOut = TIMEOUT) + public void createEncryptionPolicy_withIdWrongEncryptionType() { + String collectionName = UUID.randomUUID().toString(); + CosmosContainerProperties containerProperties = getCollectionDefinition(collectionName); + + ClientEncryptionIncludedPath path1 = new ClientEncryptionIncludedPath(); + path1.setPath("/id"); + path1.setEncryptionAlgorithm("AEAD_AES_256_CBC_HMAC_SHA256"); + path1.setEncryptionType("Randomized"); + path1.setClientEncryptionKeyId("containerTestKey1"); + + ClientEncryptionIncludedPath path2 = new ClientEncryptionIncludedPath(); + path2.setPath("/path2"); + path2.setEncryptionAlgorithm("AEAD_AES_256_CBC_HMAC_SHA256"); + path2.setEncryptionType("Deterministic"); + path2.setClientEncryptionKeyId("containerTestKey2"); + + List paths = new ArrayList<>(); + paths.add(path1); + paths.add(path2); + + try { + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 2); + fail("clientEncryptionPolicy should fail as id which is part of the partition key has to be encrypted with " + + "Deterministic type Encryption."); + } catch (IllegalArgumentException ex) { + assertThat(ex.getMessage()).isEqualTo("Only deterministic encryption type is supported for path /id."); + } + } + @DataProvider public static Object[][] analyticalTTLProvider() { return new Object[][]{ From 3cdc355e7fc7676eeedb0207f56324ddb41d7cf2 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Sat, 27 Aug 2022 21:08:09 -0700 Subject: [PATCH 02/31] Updated changelog --- sdk/cosmos/azure-cosmos-encryption/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/cosmos/azure-cosmos-encryption/CHANGELOG.md b/sdk/cosmos/azure-cosmos-encryption/CHANGELOG.md index d6020e0d7832b..1ec1485de90ee 100644 --- a/sdk/cosmos/azure-cosmos-encryption/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-encryption/CHANGELOG.md @@ -3,6 +3,8 @@ ### 1.6.0-beta.1 (Unreleased) #### Features Added +* Added support for allowing parition key path and id to be part of client encryption policy - See [PR 30678](https://github.com/Azure/azure-sdk-for-java/pull/30678) + #### Breaking Changes From a043812606d9b596aed024acceab00305c2b16be Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Sat, 27 Aug 2022 22:57:54 -0700 Subject: [PATCH 03/31] Fixing spot bugs --- .../com/azure/cosmos/models/ClientEncryptionPolicy.java | 6 +++--- .../src/test/java/com/azure/cosmos/CosmosContainerTest.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java index 7dac6485b7f67..a863c1682f225 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java @@ -40,9 +40,9 @@ public final class ClientEncryptionPolicy { */ @Beta(value = Beta.SinceVersion.V4_14_0, warningText = Beta.PREVIEW_SUBJECT_TO_CHANGE_WARNING) public ClientEncryptionPolicy(List paths) { + this.policyFormatVersion = 1; this.validateIncludedPaths(paths, policyFormatVersion); this.includedPaths = paths; - this.policyFormatVersion = 1; } /** @@ -129,7 +129,7 @@ void validatePartitionKeyPathsIfEncrypted(List> partitionKeyPathTok } // for the ClientEncryptionIncludedPath found check the encryption type. - if (encrypterPartitionKeyPath.stream().map(encrypter -> encrypter.getEncryptionType()).findFirst().orElse(null) != "Deterministic") + if (!encrypterPartitionKeyPath.stream().map(encrypter -> encrypter.getEncryptionType()).findFirst().orElse(null).equals("Deterministic")) throw new IllegalArgumentException(String.format("Path %s which is part of the partition key " + "has to be encrypted" + " with Deterministic type Encryption.", topLevelToken)); @@ -169,7 +169,7 @@ private void validateClientEncryptionIncludedPath(ClientEncryptionIncludedPath c throw new IllegalArgumentException(String.format("Path %s cannot be encrypted with policyFormatVersion %s.", clientEncryptionIncludedPath.getPath(), policyFormatVersion)); } - if (clientEncryptionIncludedPath.getEncryptionType() != "Deterministic") { + if (!clientEncryptionIncludedPath.getEncryptionType().equals("Deterministic")) { throw new IllegalArgumentException(String.format("Only deterministic encryption type is supported for path %s.", clientEncryptionIncludedPath.getPath())); } } diff --git a/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerTest.java b/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerTest.java index 924718774a06b..918328e643669 100644 --- a/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerTest.java +++ b/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerTest.java @@ -243,9 +243,9 @@ public void createEncryptionPolicy_withIdWrongPolicyFormatVersion() { try { ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths); fail("clientEncryptionPolicy should fail as id which is part of the partition key cannot be encrypted with " + - "PolicyFormatVersion 0."); + "PolicyFormatVersion 1."); } catch (IllegalArgumentException ex) { - assertThat(ex.getMessage()).isEqualTo("Path /id cannot be encrypted with policyFormatVersion 0."); + assertThat(ex.getMessage()).isEqualTo("Path /id cannot be encrypted with policyFormatVersion 1."); } } From 860885b47ff26ec4841b1f07a05d68ae23ba9f85 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Mon, 26 Sep 2022 12:30:28 -0400 Subject: [PATCH 04/31] Resolving comments --- .../cosmos/models/ClientEncryptionPolicy.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java index a863c1682f225..f4c28adaab4e2 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java @@ -129,19 +129,21 @@ void validatePartitionKeyPathsIfEncrypted(List> partitionKeyPathTok } // for the ClientEncryptionIncludedPath found check the encryption type. - if (!encrypterPartitionKeyPath.stream().map(encrypter -> encrypter.getEncryptionType()).findFirst().orElse(null).equals("Deterministic")) - throw new IllegalArgumentException(String.format("Path %s which is part of the partition key " + - "has to be encrypted" + - " with Deterministic type Encryption.", topLevelToken)); + if (!encrypterPartitionKeyPath.stream().map(encrypter -> encrypter.getEncryptionType()).findFirst().orElse(null).equals(Constants.Properties.DETERMINISTIC)) { + throw new IllegalArgumentException(String.format("Path %s which is part of the partition key " + + "has to be encrypted" + + " with Deterministic type Encryption.", topLevelToken)); + } + } } } } - private void validateIncludedPaths(List clientEncryptionIncludedPath, int policyFormatVersion) { + private static void validateIncludedPaths(List clientEncryptionIncludedPath, int policyFormatVersion) { List includedPathsList = new ArrayList<>(); for (ClientEncryptionIncludedPath path : clientEncryptionIncludedPath) { - this.validateClientEncryptionIncludedPath(path, policyFormatVersion); + validateClientEncryptionIncludedPath(path, policyFormatVersion); if (includedPathsList.contains(path.getPath())) { throw new IllegalArgumentException("Duplicate Path found in clientEncryptionIncludedPath."); } @@ -150,7 +152,7 @@ private void validateIncludedPaths(List clientEncr } } - private void validateClientEncryptionIncludedPath(ClientEncryptionIncludedPath clientEncryptionIncludedPath, int policyFormatVersion) { + private static void validateClientEncryptionIncludedPath(ClientEncryptionIncludedPath clientEncryptionIncludedPath, int policyFormatVersion) { if (clientEncryptionIncludedPath == null) { throw new IllegalArgumentException("clientEncryptionIncludedPath is null"); } @@ -169,7 +171,7 @@ private void validateClientEncryptionIncludedPath(ClientEncryptionIncludedPath c throw new IllegalArgumentException(String.format("Path %s cannot be encrypted with policyFormatVersion %s.", clientEncryptionIncludedPath.getPath(), policyFormatVersion)); } - if (!clientEncryptionIncludedPath.getEncryptionType().equals("Deterministic")) { + if (!clientEncryptionIncludedPath.getEncryptionType().equals(Constants.Properties.DETERMINISTIC)) { throw new IllegalArgumentException(String.format("Only deterministic encryption type is supported for path %s.", clientEncryptionIncludedPath.getPath())); } } From c24a480f816df281557a6a72c449895eb4276e70 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Mon, 31 Oct 2022 11:11:14 -0700 Subject: [PATCH 05/31] Fixing CI build --- .../java/com/azure/cosmos/models/ClientEncryptionPolicy.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java index d7b253827307e..642766fef2816 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java @@ -6,6 +6,7 @@ import com.azure.cosmos.implementation.Constants; import com.azure.cosmos.implementation.JsonSerializable; import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; +import com.azure.cosmos.util.Beta; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -35,6 +36,8 @@ public final class ClientEncryptionPolicy { * Constructor. * * @param paths list of path of the item that need encryption along with path-specific settings. + * the PolicyFormatVersion will be set to 1 which is the default value. + * Note: If you need to include partition key or id field paths as part of the ClientEncryptionPolicy, please set PolicyFormatVersion to 2. */ public ClientEncryptionPolicy(List paths) { this.policyFormatVersion = 1; @@ -47,8 +50,8 @@ public ClientEncryptionPolicy(List paths) { * * @param paths list of path of the item that need encryption along with path-specific settings. * @param policyFormatVersion version of the client encryption policy definition. Current supported versions are 1 and 2. Default version is 1. + * Note: If you need to include partition key or id field paths as part of the ClientEncryptionPolicy, please set PolicyFormatVersion to 2. */ - @Beta(value = Beta.SinceVersion.V4_14_0, warningText = Beta.PREVIEW_SUBJECT_TO_CHANGE_WARNING) public ClientEncryptionPolicy(List paths, int policyFormatVersion) { if (policyFormatVersion > 2 || policyFormatVersion < 1) { throw new IllegalArgumentException("Supported versions of client encryption policy are 1 and 2."); From 337b94c111130b5e6b8863f2d8a24f8ac3b99683 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Mon, 31 Oct 2022 14:37:00 -0700 Subject: [PATCH 06/31] Fixing CI build --- .../com/azure/cosmos/models/ClientEncryptionPolicy.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java index 642766fef2816..f430d5fad5bf7 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java @@ -41,7 +41,7 @@ public final class ClientEncryptionPolicy { */ public ClientEncryptionPolicy(List paths) { this.policyFormatVersion = 1; - this.validateIncludedPaths(paths, policyFormatVersion); + validateIncludedPaths(paths, policyFormatVersion); this.includedPaths = paths; } @@ -57,7 +57,7 @@ public ClientEncryptionPolicy(List paths, int poli throw new IllegalArgumentException("Supported versions of client encryption policy are 1 and 2."); } this.policyFormatVersion = policyFormatVersion; - this.validateIncludedPaths(paths, policyFormatVersion); + validateIncludedPaths(paths, policyFormatVersion); this.includedPaths = paths; } @@ -123,7 +123,7 @@ void validatePartitionKeyPathsIfEncrypted(List> partitionKeyPathTok if (this.policyFormatVersion < 2) { throw new IllegalArgumentException(String.format("Path %s which is part of the partition key " + "cannot be encrypted" + - " with PolicyFormatVersion %s", topLevelToken, policyFormatVersion)); + " with PolicyFormatVersion %s. Please use PolicyFormatVersion 2.", topLevelToken, policyFormatVersion)); } // for the ClientEncryptionIncludedPath found check the encryption type. From dd159b3537d32d420a7999ae18eb40bd773e3ecd Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Wed, 2 Nov 2022 08:54:41 -0700 Subject: [PATCH 07/31] Added code to allow encryption of partitionKey and id --- .../azure-cosmos-encryption/CHANGELOG.md | 6 +- .../CosmosEncryptionAsyncContainer.java | 70 +++++++++++++++++++ .../encryption/implementation/Constants.java | 2 + .../implementation/EncryptionProcessor.java | 55 ++++++++++++++- .../implementation/EncryptionSettings.java | 10 +++ .../implementation/EncryptionUtils.java | 1 + 6 files changed, 139 insertions(+), 5 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/CHANGELOG.md b/sdk/cosmos/azure-cosmos-encryption/CHANGELOG.md index ac6d52866d2af..8b06f1d446cbc 100644 --- a/sdk/cosmos/azure-cosmos-encryption/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-encryption/CHANGELOG.md @@ -1,10 +1,10 @@ ## Release History -### 1.9.0-beta.1 (Unreleased) +### 2.0.0-beta.1 (Unreleased) #### Features Added -* Added support for allowing parition key path and id to be part of client encryption policy - See [PR 30678](https://github.com/Azure/azure-sdk-for-java/pull/30678) - +* Added support for allowing partition key path and id to be part of client encryption policy - See [PR 30678](https://github.com/Azure/azure-sdk-for-java/pull/30678) +* Added support for allowing the encryption of partition key and id - See [PR 30678](https://github.com/Azure/azure-sdk-for-java/pull/30678) #### Breaking Changes diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java index 36675fb7c08d5..6d3d29123dd8d 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java @@ -10,7 +10,9 @@ import com.azure.cosmos.encryption.implementation.Constants; import com.azure.cosmos.encryption.implementation.CosmosResponseFactory; import com.azure.cosmos.encryption.implementation.EncryptionImplementationBridgeHelpers; +import com.azure.cosmos.encryption.implementation.EncryptionSettings; import com.azure.cosmos.encryption.implementation.EncryptionUtils; +import com.azure.cosmos.encryption.implementation.mdesrc.cryptography.MicrosoftDataEncryptionException; import com.azure.cosmos.implementation.CosmosPagedFluxOptions; import com.azure.cosmos.implementation.HttpConstants; import com.azure.cosmos.implementation.ImplementationBridgeHelpers; @@ -22,6 +24,7 @@ import com.azure.cosmos.implementation.patch.PatchOperationCore; import com.azure.cosmos.implementation.patch.PatchOperationType; import com.azure.cosmos.implementation.query.Transformer; +import com.azure.cosmos.models.ClientEncryptionPolicy; import com.azure.cosmos.models.CosmosBatch; import com.azure.cosmos.models.CosmosBatchOperationResult; import com.azure.cosmos.models.CosmosBatchRequestOptions; @@ -39,13 +42,16 @@ import com.azure.cosmos.models.FeedResponse; import com.azure.cosmos.models.ModelBridgeInternal; import com.azure.cosmos.models.PartitionKey; +import com.azure.cosmos.models.PartitionKeyBuilder; import com.azure.cosmos.models.SqlParameter; import com.azure.cosmos.models.SqlQuerySpec; import com.azure.cosmos.util.CosmosPagedFlux; import com.azure.cosmos.util.UtilBridgeInternal; import com.azure.cosmos.encryption.implementation.EncryptionProcessor; import com.azure.cosmos.encryption.models.SqlQuerySpecWithEncryption; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -59,7 +65,9 @@ import java.util.stream.Collectors; import static com.azure.cosmos.implementation.Utils.getEffectiveCosmosChangeFeedRequestOptions; +import static com.azure.cosmos.implementation.Utils.isEmpty; import static com.azure.cosmos.implementation.Utils.setContinuationTokenAndMaxItemCount; +import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkElementIndex; import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; /** @@ -193,9 +201,62 @@ public Mono> deleteItem(String itemId, PartitionKey partitionKey, CosmosItemRequestOptions requestOptions) { + this.encryptionProcessor.initEncryptionSettingsIfNotInitializedAsync(); + EncryptionSettings encryptionSettings = this.encryptionProcessor.getEncryptionSettings(); + + try { + itemId = checkAndGetEncryptedId(itemId, encryptionSettings); + partitionKey = checkAndGetEncryptedPartitionKey(partitionKey, encryptionSettings); + } catch (JsonProcessingException | MicrosoftDataEncryptionException e) { + return Mono.error(e); + } return container.deleteItem(itemId, partitionKey, requestOptions); } + private String checkAndGetEncryptedId(String itemId, EncryptionSettings encryptionSettings) throws MicrosoftDataEncryptionException { + if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream().filter(includedPath -> includedPath.getPath().substring(1).equals(Constants.PROPERTY_NAME_ID)).findFirst().isEmpty()) { + return itemId; + } + return this.encryptionProcessor.encryptAndSerializeValue(encryptionSettings, itemId, Constants.PROPERTY_NAME_ID); + } + + private PartitionKey checkAndGetEncryptedPartitionKey(PartitionKey partitionKey, EncryptionSettings encryptionSettings) throws JsonProcessingException, MicrosoftDataEncryptionException { + if (encryptionSettings.getPartitionKeyPaths().isEmpty() || partitionKey == null) { + return partitionKey; + } + JsonNode partitionKeyNode = EncryptionUtils.getSimpleObjectMapper().readTree(partitionKey.toString()); + if (partitionKeyNode.isArray()) { + ArrayNode arrayNode = (ArrayNode) partitionKeyNode; + PartitionKeyBuilder partitionKeyBuilder = new PartitionKeyBuilder(); + + for (String path : encryptionSettings.getPartitionKeyPaths()) { + // case: partition key path is /a/b/c and the client encryption policy has /a in path. + // hence encrypt the partition key value with using its top level path /a since /c would have been encrypted in the document using /a's policy. + String partitionKeyPath = path.split("/")[0]; + + String childPartitionKey = arrayNode.elements().next().toString(); + if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream().filter(includedPath -> includedPath.getPath().substring(1).equals(partitionKeyPath)).findFirst().isEmpty()) { + partitionKeyBuilder.add(childPartitionKey); + continue; + } + partitionKeyBuilder.add(this.encryptionProcessor.encryptAndSerializeValue(encryptionSettings, childPartitionKey, partitionKeyPath)); + } + return partitionKeyBuilder.build(); + } + + else { + if (encryptionSettings.getPartitionKeyPaths().size() > 1) { + throw new MicrosoftDataEncryptionException("There should only be 1 PartitionKeyPath."); + } + String partitionKeyPath = encryptionSettings.getPartitionKeyPaths().get(0); + if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream().filter(includedPath -> includedPath.getPath().substring(1).equals(partitionKeyPath)).findFirst().isEmpty()) { + return partitionKey; + } + return new PartitionKey(this.encryptionProcessor.encryptAndSerializeValue(encryptionSettings, partitionKey.toString(), partitionKeyPath)); + } + } + + /** * Deletes the item. *

@@ -228,6 +289,15 @@ Mono> deleteAllItemsByPartitionKey(PartitionKey parti if (requestOptions == null) { requestOptions = new CosmosItemRequestOptions(); } + + this.encryptionProcessor.initEncryptionSettingsIfNotInitializedAsync(); + EncryptionSettings encryptionSettings = this.encryptionProcessor.getEncryptionSettings(); + + try { + partitionKey = checkAndGetEncryptedPartitionKey(partitionKey, encryptionSettings); + } catch (JsonProcessingException | MicrosoftDataEncryptionException e) { + return Mono.error(e); + } return container.deleteAllItemsByPartitionKey(partitionKey, requestOptions); } diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/Constants.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/Constants.java index c06ae1b71f2ca..888752d3959f1 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/Constants.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/Constants.java @@ -15,4 +15,6 @@ public class Constants { public static final String ALLOW_CACHED_READS_HEADER = "x-ms-cosmos-allow-cachedreads"; public static final String DATABASE_RID_HEADER = "x-ms-cosmos-database-rid"; + + public static final String PROPERTY_NAME_ID = "id"; } diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java index 42032b0f2947f..d5bb27f39c4de 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.BooleanNode; import com.fasterxml.jackson.databind.node.DoubleNode; +import com.fasterxml.jackson.databind.node.JsonNodeType; import com.fasterxml.jackson.databind.node.LongNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; @@ -41,6 +42,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -48,6 +50,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; public class EncryptionProcessor { private final static Logger LOGGER = LoggerFactory.getLogger(EncryptionProcessor.class); @@ -58,6 +61,7 @@ public class EncryptionProcessor { private ClientEncryptionPolicy clientEncryptionPolicy; private String containerRid; private String databaseRid; + private List partitionKeyPaths; private CosmosClientEncryptionKeyProperties cosmosClientEncryptionKeyProperties; private final EncryptionKeyStoreProviderImpl encryptionKeyStoreProviderImpl; private final static ImplementationBridgeHelpers.CosmosContainerPropertiesHelper.CosmosContainerPropertiesAccessor cosmosContainerPropertiesAccessor = ImplementationBridgeHelpers.CosmosContainerPropertiesHelper.getCosmosContainerPropertiesAccessor(); @@ -99,7 +103,15 @@ public Mono initializeEncryptionSettingsAsync(boolean isRetry) { { this.containerRid = cosmosContainerProperties.getResourceId(); this.databaseRid = cosmosContainerPropertiesAccessor.getSelfLink(cosmosContainerProperties).split("/")[1]; + + if (cosmosContainerProperties.getPartitionKeyDefinition().getPaths().isEmpty()) { + this.partitionKeyPaths = cosmosContainerProperties.getPartitionKeyDefinition().getPaths(); + } else { + this.partitionKeyPaths = new ArrayList<>(); + } + this.encryptionSettings.setDatabaseRid(this.databaseRid); + this.encryptionSettings.setPartitionKeyPaths(partitionKeyPaths); if (cosmosContainerProperties.getClientEncryptionPolicy() == null) { this.isEncryptionSettingsInitDone.set(true); return Mono.empty(); @@ -206,7 +218,7 @@ public Mono initEncryptionSettingsIfNotInitializedAsync() { return Mono.empty(); } - ClientEncryptionPolicy getClientEncryptionPolicy() { + public ClientEncryptionPolicy getClientEncryptionPolicy() { return clientEncryptionPolicy; } @@ -438,16 +450,33 @@ public void encryptAndSerializeProperty(EncryptionSettings encryptionSettings, J } } + public String encryptAndSerializeValue(EncryptionSettings encryptionSettings, String propertyValue, String propertyName) throws MicrosoftDataEncryptionException { + JsonNode propertyValueHolder = toJsonNode(propertyValue.getBytes(), TypeMarker.STRING); + return convertToBase64UriSafeString(encryptAndSerializeValue(encryptionSettings, null, propertyValueHolder, propertyName)); + } + public byte[] encryptAndSerializeValue(EncryptionSettings encryptionSettings, ObjectNode objectNode, JsonNode propertyValueHolder, String propertyName) throws MicrosoftDataEncryptionException { byte[] cipherText; byte[] cipherTextWithTypeMarker; + if (propertyName.equals(Constants.PROPERTY_NAME_ID)) { + if (propertyValueHolder.getNodeType() != JsonNodeType.STRING) { + throw new IllegalArgumentException("Unsupported argument type. The value to escape has to be string " + + "type. Please refer to https://aka.ms/CosmosClientEncryption for more details."); + } + } Pair typeMarkerPair = toByteArray(propertyValueHolder); cipherText = encryptionSettings.getAeadAes256CbcHmac256EncryptionAlgorithm().encrypt(typeMarkerPair.getRight()); cipherTextWithTypeMarker = new byte[cipherText.length + 1]; cipherTextWithTypeMarker[0] = (byte) typeMarkerPair.getLeft().getValue(); System.arraycopy(cipherText, 0, cipherTextWithTypeMarker, 1, cipherText.length); + + if (propertyName.equals(Constants.PROPERTY_NAME_ID)) { + // case: id does not support '/','\','?','#'. Convert Base64 string to Uri safe string + cipherTextWithTypeMarker = convertToBase64UriSafeString(cipherTextWithTypeMarker).getBytes(); + } + if (objectNode != null && !objectNode.isNull()) { objectNode.put(propertyName, cipherTextWithTypeMarker); } @@ -567,7 +596,15 @@ public JsonNode decryptAndSerializeValue(EncryptionSettings encryptionSettings, JsonNode propertyValueHolder, String propertyName) throws MicrosoftDataEncryptionException, IOException { byte[] cipherText; byte[] cipherTextWithTypeMarker; - cipherTextWithTypeMarker = propertyValueHolder.binaryValue(); + if (propertyName.equals(Constants.PROPERTY_NAME_ID)) { + if (propertyValueHolder.getNodeType() == JsonNodeType.NULL) { + return null; + } + cipherTextWithTypeMarker = convertFromBase64UriSafeString(propertyValueHolder.toString()); + } else { + cipherTextWithTypeMarker = propertyValueHolder.binaryValue(); + } + cipherText = new byte[cipherTextWithTypeMarker.length - 1]; System.arraycopy(cipherTextWithTypeMarker, 1, cipherText, 0, cipherTextWithTypeMarker.length - 1); @@ -627,6 +664,20 @@ public static JsonNode toJsonNode(byte[] serializedBytes, TypeMarker typeMarker) throw BridgeInternal.createCosmosException(0, "Invalid or Unsupported Data Type Passed " + typeMarker); } + private String convertToBase64UriSafeString(byte[] bytesToProcess) { + String base64String = new String(Base64.getDecoder().decode(bytesToProcess)); + + // Base 64 Encoding with URL and Filename Safe Alphabet https://datatracker.ietf.org/doc/html/rfc4648#section-5 + // https://docs.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-item-limits, due to base64 conversion and encryption + // the permissible size of the property will further reduce. + return base64String.replaceAll("/","_").replaceAll("\\+","-"); + } + + private byte[] convertFromBase64UriSafeString(String base64UriSafeString) { + base64UriSafeString.replaceAll("_", "/").replaceAll("-", "\\+"); + return Base64.getEncoder().encode(base64UriSafeString.getBytes()); + } + public enum TypeMarker { NULL(1), // not used BOOLEAN(2), diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionSettings.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionSettings.java index 8c45d9be3e33e..e25c13be68e07 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionSettings.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionSettings.java @@ -25,6 +25,7 @@ import java.security.InvalidKeyException; import java.time.Duration; import java.time.Instant; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -39,6 +40,7 @@ public final class EncryptionSettings { private EncryptionType encryptionType; private String databaseRid; private CosmosClientEncryptionKeyProperties cosmosClientEncryptionKeyProperties; + private List partitionKeyPaths; private final static EncryptionImplementationBridgeHelpers.CosmosEncryptionAsyncClientHelper.CosmosEncryptionAsyncClientAccessor cosmosEncryptionAsyncClientAccessor = EncryptionImplementationBridgeHelpers.CosmosEncryptionAsyncClientHelper.getCosmosEncryptionAsyncClientAccessor(); @@ -226,6 +228,14 @@ public void setEncryptionSettingForProperty(String propertyName, EncryptionSetti this.encryptionSettingCacheByPropertyName.set(propertyName, cachedEncryptionSettings); } + public List getPartitionKeyPaths() { + return partitionKeyPaths; + } + + public void setPartitionKeyPaths(List partitionKeyPaths) { + this.partitionKeyPaths = partitionKeyPaths; + } + static EncryptionSettings create( EncryptionSettings settingsForKey, EncryptionType encryptionType) throws MicrosoftDataEncryptionException { diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionUtils.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionUtils.java index c2cb21f3cb9e7..5fbc16dd391b2 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionUtils.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionUtils.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.nio.ByteBuffer; +import java.util.Base64; public class EncryptionUtils { private static final ObjectMapper simpleObjectMapper = new ObjectMapper(); From 56b18cf31105a989fbde5d35746cfafe55a36c62 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Wed, 2 Nov 2022 09:01:22 -0700 Subject: [PATCH 08/31] removing unused imports --- .../cosmos/encryption/CosmosEncryptionAsyncContainer.java | 3 --- .../cosmos/encryption/implementation/EncryptionProcessor.java | 1 - .../cosmos/encryption/implementation/EncryptionUtils.java | 1 - .../java/com/azure/cosmos/models/ClientEncryptionPolicy.java | 3 +-- 4 files changed, 1 insertion(+), 7 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java index 6d3d29123dd8d..1089f9473ca19 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java @@ -24,7 +24,6 @@ import com.azure.cosmos.implementation.patch.PatchOperationCore; import com.azure.cosmos.implementation.patch.PatchOperationType; import com.azure.cosmos.implementation.query.Transformer; -import com.azure.cosmos.models.ClientEncryptionPolicy; import com.azure.cosmos.models.CosmosBatch; import com.azure.cosmos.models.CosmosBatchOperationResult; import com.azure.cosmos.models.CosmosBatchRequestOptions; @@ -65,9 +64,7 @@ import java.util.stream.Collectors; import static com.azure.cosmos.implementation.Utils.getEffectiveCosmosChangeFeedRequestOptions; -import static com.azure.cosmos.implementation.Utils.isEmpty; import static com.azure.cosmos.implementation.Utils.setContinuationTokenAndMaxItemCount; -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkElementIndex; import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; /** diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java index d5bb27f39c4de..ce54e9d51ea5e 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java @@ -50,7 +50,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import java.util.regex.Pattern; public class EncryptionProcessor { private final static Logger LOGGER = LoggerFactory.getLogger(EncryptionProcessor.class); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionUtils.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionUtils.java index 5fbc16dd391b2..c2cb21f3cb9e7 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionUtils.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionUtils.java @@ -9,7 +9,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.nio.ByteBuffer; -import java.util.Base64; public class EncryptionUtils { private static final ObjectMapper simpleObjectMapper = new ObjectMapper(); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java index f430d5fad5bf7..5952e9988638f 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java @@ -6,7 +6,6 @@ import com.azure.cosmos.implementation.Constants; import com.azure.cosmos.implementation.JsonSerializable; import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.util.Beta; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -164,7 +163,7 @@ private static void validateClientEncryptionIncludedPath(ClientEncryptionInclude throw new IllegalArgumentException("Invalid path " + clientEncryptionIncludedPath.getPath()); } - if (clientEncryptionIncludedPath.getPath().substring(1).equals("id")) { + if (clientEncryptionIncludedPath.getPath().substring(1).equals(Constants.Properties.ID)) { if (policyFormatVersion < 2) { throw new IllegalArgumentException(String.format("Path %s cannot be encrypted with policyFormatVersion %s.", clientEncryptionIncludedPath.getPath(), policyFormatVersion)); } From 06ed153904af77c898f1a22beacd38b9b9fa4bef Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Wed, 2 Nov 2022 12:16:18 -0700 Subject: [PATCH 09/31] Fixing CI build --- .../cosmos/encryption/CosmosEncryptionAsyncContainer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java index 1089f9473ca19..93a67cf5d459f 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java @@ -211,7 +211,7 @@ public Mono> deleteItem(String itemId, } private String checkAndGetEncryptedId(String itemId, EncryptionSettings encryptionSettings) throws MicrosoftDataEncryptionException { - if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream().filter(includedPath -> includedPath.getPath().substring(1).equals(Constants.PROPERTY_NAME_ID)).findFirst().isEmpty()) { + if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream().anyMatch(includedPath -> includedPath.getPath().substring(1).equals(Constants.PROPERTY_NAME_ID))) { return itemId; } return this.encryptionProcessor.encryptAndSerializeValue(encryptionSettings, itemId, Constants.PROPERTY_NAME_ID); @@ -232,7 +232,7 @@ private PartitionKey checkAndGetEncryptedPartitionKey(PartitionKey partitionKey, String partitionKeyPath = path.split("/")[0]; String childPartitionKey = arrayNode.elements().next().toString(); - if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream().filter(includedPath -> includedPath.getPath().substring(1).equals(partitionKeyPath)).findFirst().isEmpty()) { + if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream().anyMatch(includedPath -> includedPath.getPath().substring(1).equals(partitionKeyPath))) { partitionKeyBuilder.add(childPartitionKey); continue; } @@ -246,7 +246,7 @@ private PartitionKey checkAndGetEncryptedPartitionKey(PartitionKey partitionKey, throw new MicrosoftDataEncryptionException("There should only be 1 PartitionKeyPath."); } String partitionKeyPath = encryptionSettings.getPartitionKeyPaths().get(0); - if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream().filter(includedPath -> includedPath.getPath().substring(1).equals(partitionKeyPath)).findFirst().isEmpty()) { + if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream().anyMatch(includedPath -> includedPath.getPath().substring(1).equals(partitionKeyPath))) { return partitionKey; } return new PartitionKey(this.encryptionProcessor.encryptAndSerializeValue(encryptionSettings, partitionKey.toString(), partitionKeyPath)); From e8a16fcd78b50ac695f29a505bb0a7649f0a2f9b Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Wed, 2 Nov 2022 13:27:20 -0700 Subject: [PATCH 10/31] Adding test cases --- .../CosmosEncryptionClientCachesTest.java | 2 +- .../encryption/DotNetCompatibleTest.java | 2 +- .../EncryptionAsyncApiCrudTest.java | 115 +++++++++++++++++- ...ryptionCosmosEncryptionChangeFeedTest.java | 2 +- .../cosmos/encryption/TestSuiteBase.java | 21 +++- 5 files changed, 134 insertions(+), 8 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionClientCachesTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionClientCachesTest.java index efc84fa304dce..8335580f58e71 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionClientCachesTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionClientCachesTest.java @@ -68,7 +68,7 @@ public void before_CosmosItemTest() { CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName(), metadata2).block(); //Create collection with clientEncryptionPolicy - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths()); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(false)); CosmosContainerProperties containerProperties = new CosmosContainerProperties("TestCollForEncryptionCacheTest" , "/mypk"); containerProperties.setClientEncryptionPolicy(clientEncryptionPolicy); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java index 58fe54893f845..c7f036be371a2 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java @@ -70,7 +70,7 @@ public void before_CosmosItemTest() throws IOException { cosmosEncryptionAsyncClient.getCosmosEncryptionAsyncDatabase(cosmosAsyncDatabase.getId()); ClientEncryptionPolicy clientEncryptionPolicy = - new ClientEncryptionPolicy(getPaths()); + new ClientEncryptionPolicy(getPaths(false)); String containerId = UUID.randomUUID().toString(); CosmosContainerProperties containerProperties = new CosmosContainerProperties(containerId, "/mypk"); containerProperties.setClientEncryptionPolicy(clientEncryptionPolicy); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java index a5415f557db45..03b7e5552d670 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java @@ -57,6 +57,7 @@ public class EncryptionAsyncApiCrudTest extends TestSuiteBase { private CosmosAsyncClient client; private CosmosEncryptionAsyncClient cosmosEncryptionAsyncClient; private CosmosEncryptionAsyncContainer encryptionContainerWithIncompatiblePolicyVersion; + private CosmosContainerProperties propertiesWithIdAndPartitionKeyPath; CosmosEncryptionAsyncContainer cosmosEncryptionAsyncContainer; CosmosEncryptionAsyncDatabase cosmosEncryptionAsyncDatabase; @@ -76,12 +77,17 @@ public void before_CosmosItemTest() { cosmosEncryptionAsyncDatabase = getSharedEncryptionDatabase(cosmosEncryptionAsyncClient); cosmosEncryptionAsyncContainer = getSharedEncryptionContainer(cosmosEncryptionAsyncClient); - ClientEncryptionPolicy clientEncryptionWithPolicyFormatVersion2 = new ClientEncryptionPolicy(getPaths()); - ReflectionUtils.setPolicyFormatVersion(clientEncryptionWithPolicyFormatVersion2, 2); + ClientEncryptionPolicy clientEncryptionWithPolicyFormatVersion2 = new ClientEncryptionPolicy(getPaths(false), 2); String containerId = UUID.randomUUID().toString(); CosmosContainerProperties properties = new CosmosContainerProperties(containerId, "/mypk"); properties.setClientEncryptionPolicy(clientEncryptionWithPolicyFormatVersion2); cosmosEncryptionAsyncDatabase.getCosmosAsyncDatabase().createContainer(properties).block(); + + // ClientEncryptionPolicy with Id and PartitionKey included, and policyFormatVersion 2 + ClientEncryptionPolicy clientEncryptionPolicyWithIdAndPartitionKeyPath = new ClientEncryptionPolicy(getPaths(true), 2); + CosmosContainerProperties propertiesWithIdAndPartitionKeyPath = new CosmosContainerProperties(containerId, "/mypk"); + propertiesWithIdAndPartitionKeyPath.setClientEncryptionPolicy(clientEncryptionPolicyWithIdAndPartitionKeyPath); + encryptionContainerWithIncompatiblePolicyVersion = cosmosEncryptionAsyncDatabase.getCosmosEncryptionAsyncContainer(containerId); } @@ -365,7 +371,7 @@ public void crudQueryStaleCache() { cosmosEncryptionAsyncClient.getCosmosEncryptionAsyncDatabase(asyncClient.getDatabase(databaseId)); String containerId = UUID.randomUUID().toString(); - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths()); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(false)); createEncryptionContainer(cosmosEncryptionAsyncDatabase, clientEncryptionPolicy, containerId); CosmosEncryptionAsyncContainer encryptionAsyncContainerOriginal = cosmosEncryptionAsyncDatabase.getCosmosEncryptionAsyncContainer(containerId); @@ -1011,6 +1017,109 @@ public void crudOnDifferentOverload() { assertThat(deleteResponse3.getStatusCode()).isEqualTo(200); } + @Test(groups = {"encryption"}, timeOut = TIMEOUT) + public void crudOperationsWithIdAndPartitionKeyEncrypted() { + cosmosEncryptionAsyncDatabase.getCosmosAsyncDatabase().createContainer(propertiesWithIdAndPartitionKeyPath).block(); + List actualProperties = new ArrayList<>(); + // Read item + EncryptionPojo properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse itemResponse = cosmosEncryptionAsyncContainer.createItem(properties).block(); + assertThat(itemResponse.getRequestCharge()).isGreaterThan(0); + EncryptionPojo responseItem = itemResponse.getItem(); + validateResponse(properties, responseItem); + actualProperties.add(properties); + + properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse itemResponse1 = cosmosEncryptionAsyncContainer.createItem(properties, new CosmosItemRequestOptions()).block(); + assertThat(itemResponse1.getRequestCharge()).isGreaterThan(0); + EncryptionPojo responseItem1 = itemResponse1.getItem(); + validateResponse(properties, responseItem1); + actualProperties.add(properties); + + //Upsert Item + properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse upsertResponse1 = cosmosEncryptionAsyncContainer.upsertItem(properties).block(); + assertThat(upsertResponse1.getRequestCharge()).isGreaterThan(0); + EncryptionPojo responseItem2 = upsertResponse1.getItem(); + validateResponse(properties, responseItem2); + actualProperties.add(properties); + + properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse upsertResponse2 = cosmosEncryptionAsyncContainer.upsertItem(properties, new CosmosItemRequestOptions()).block(); + assertThat(upsertResponse2.getRequestCharge()).isGreaterThan(0); + EncryptionPojo responseItem3 = upsertResponse2.getItem(); + validateResponse(properties, responseItem3); + actualProperties.add(properties); + + //Read Item + EncryptionPojo readItem = cosmosEncryptionAsyncContainer.readItem(actualProperties.get(0).getId(), + new PartitionKey(actualProperties.get(0).getMypk()), EncryptionPojo.class).block().getItem(); + validateResponse(actualProperties.get(0), readItem); + + //Query Item + String query = String.format("SELECT * from c where c.id = '%s'", actualProperties.get(1).getId()); + + CosmosPagedFlux feedResponseIterator = + cosmosEncryptionAsyncContainer.queryItems(query, EncryptionPojo.class); + List feedResponse = feedResponseIterator.byPage().blockFirst().getResults(); + assertThat(feedResponse.size()).isGreaterThanOrEqualTo(1); + for (EncryptionPojo pojo : feedResponse) { + if (pojo.getId().equals(actualProperties.get(1).getId())) { + validateResponse(pojo, responseItem1); + } + } + + CosmosQueryRequestOptions cosmosQueryRequestOptions1 = new CosmosQueryRequestOptions(); + + CosmosPagedFlux feedResponseIterator1 = + cosmosEncryptionAsyncContainer.queryItems(query, cosmosQueryRequestOptions1, EncryptionPojo.class); + List feedResponse1 = feedResponseIterator1.byPage().blockFirst().getResults(); + assertThat(feedResponse1.size()).isGreaterThanOrEqualTo(1); + for (EncryptionPojo pojo : feedResponse1) { + if (pojo.getId().equals(actualProperties.get(1).getId())) { + validateResponse(pojo, responseItem1); + } + } + + CosmosQueryRequestOptions cosmosQueryRequestOptions2 = new CosmosQueryRequestOptions(); + SqlQuerySpec querySpec = new SqlQuerySpec(query); + + CosmosPagedFlux feedResponseIterator2 = + cosmosEncryptionAsyncContainer.queryItems(querySpec, cosmosQueryRequestOptions2, EncryptionPojo.class); + List feedResponse2 = feedResponseIterator2.byPage().blockFirst().getResults(); + assertThat(feedResponse2.size()).isGreaterThanOrEqualTo(1); + for (EncryptionPojo pojo : feedResponse2) { + if (pojo.getId().equals(actualProperties.get(1).getId())) { + validateResponse(pojo, responseItem1); + } + } + + //Replace Item + CosmosItemResponse replaceResponse = + cosmosEncryptionAsyncContainer.replaceItem(actualProperties.get(2), actualProperties.get(2).getId(), + new PartitionKey(actualProperties.get(2).getMypk())).block(); + assertThat(upsertResponse1.getRequestCharge()).isGreaterThan(0); + responseItem = replaceResponse.getItem(); + validateResponse(actualProperties.get(2), responseItem); + + //Delete Item + CosmosItemResponse deleteResponse = cosmosEncryptionAsyncContainer.deleteItem(actualProperties.get(0).getId(), + new PartitionKey(actualProperties.get(0).getMypk())).block(); + assertThat(deleteResponse.getStatusCode()).isEqualTo(204); + + CosmosItemResponse deleteResponse1 = cosmosEncryptionAsyncContainer.deleteItem(actualProperties.get(1).getId(), + new PartitionKey(actualProperties.get(1).getMypk()), new CosmosItemRequestOptions()).block(); + assertThat(deleteResponse1.getStatusCode()).isEqualTo(204); + + CosmosItemResponse deleteResponse2 = cosmosEncryptionAsyncContainer.deleteItem(actualProperties.get(2), + new CosmosItemRequestOptions()).block(); + assertThat(deleteResponse2.getStatusCode()).isEqualTo(204); + + CosmosItemResponse deleteResponse3 = cosmosEncryptionAsyncContainer.deleteAllItemsByPartitionKey(new PartitionKey(actualProperties.get(3).getMypk()), + new CosmosItemRequestOptions()).block(); + assertThat(deleteResponse3.getStatusCode()).isEqualTo(200); + } + static void validateResponseWithOneFieldEncryption(EncryptionPojo originalItem, EncryptionPojo result) { assertThat(result.getId()).isEqualTo(originalItem.getId()); assertThat(result.getNonSensitive()).isEqualTo(originalItem.getNonSensitive()); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionCosmosEncryptionChangeFeedTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionCosmosEncryptionChangeFeedTest.java index 497487b8deeb8..38c9ae0561585 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionCosmosEncryptionChangeFeedTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionCosmosEncryptionChangeFeedTest.java @@ -264,7 +264,7 @@ private CosmosAsyncContainer createLeaseCollection(int provisionedThroughput) { } private CosmosEncryptionAsyncContainer createFeedCollection() { - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths()); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(false)); String containerId = UUID.randomUUID().toString(); CosmosContainerProperties properties = new CosmosContainerProperties(containerId, "/mypk"); properties.setClientEncryptionPolicy(clientEncryptionPolicy); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java index 3d09303f79bfc..510b2439d14a2 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java @@ -240,7 +240,7 @@ public static void beforeSuite() { SHARED_ENCRYPTION_DATABASE.createClientEncryptionKey("key2", CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName(), metadata2).block(); - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths()); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(false),2); String containerId = UUID.randomUUID().toString(); CosmosContainerProperties properties = new CosmosContainerProperties(containerId, "/mypk"); properties.setClientEncryptionPolicy(clientEncryptionPolicy); @@ -1213,7 +1213,7 @@ public KeyEncryptionKey buildKeyEncryptionKey(String keyId) { } } - protected static List getPaths() { + protected static List getPaths(boolean shouldEncryptIdAndPartitionKey) { ClientEncryptionIncludedPath includedPath1 = new ClientEncryptionIncludedPath(); includedPath1.setClientEncryptionKeyId("key1"); includedPath1.setPath("/sensitiveString"); @@ -1292,6 +1292,18 @@ protected static List getPaths() { includedPath13.setEncryptionType(CosmosEncryptionType.DETERMINISTIC.getName()); includedPath13.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName()); + ClientEncryptionIncludedPath includedPath14 = new ClientEncryptionIncludedPath(); + includedPath13.setClientEncryptionKeyId("key1"); + includedPath13.setPath("/id"); + includedPath13.setEncryptionType(CosmosEncryptionType.DETERMINISTIC.getName()); + includedPath13.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName()); + + ClientEncryptionIncludedPath includedPath15 = new ClientEncryptionIncludedPath(); + includedPath13.setClientEncryptionKeyId("key1"); + includedPath13.setPath("/mypk"); + includedPath13.setEncryptionType(CosmosEncryptionType.DETERMINISTIC.getName()); + includedPath13.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName()); + List paths = new ArrayList<>(); paths.add(includedPath1); paths.add(includedPath2); @@ -1307,6 +1319,11 @@ protected static List getPaths() { paths.add(includedPath12); paths.add(includedPath13); + if (shouldEncryptIdAndPartitionKey) { + paths.add(includedPath14); + paths.add(includedPath15); + } + return paths; } From 65069b8728dcf5b1f5b0230d0a4e58d8b5e16824 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Wed, 2 Nov 2022 13:47:24 -0700 Subject: [PATCH 11/31] Fixing change log --- sdk/cosmos/azure-cosmos-benchmark/pom.xml | 2 +- sdk/cosmos/azure-cosmos-encryption/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-benchmark/pom.xml b/sdk/cosmos/azure-cosmos-benchmark/pom.xml index dc6e81052501d..5812ebf7d2f24 100644 --- a/sdk/cosmos/azure-cosmos-benchmark/pom.xml +++ b/sdk/cosmos/azure-cosmos-benchmark/pom.xml @@ -57,7 +57,7 @@ Licensed under the MIT License. com.azure azure-cosmos-encryption - 1.9.0-beta.1 + 2.0.0-beta.1 diff --git a/sdk/cosmos/azure-cosmos-encryption/pom.xml b/sdk/cosmos/azure-cosmos-encryption/pom.xml index 371a1b93fc33f..035dc5839131a 100644 --- a/sdk/cosmos/azure-cosmos-encryption/pom.xml +++ b/sdk/cosmos/azure-cosmos-encryption/pom.xml @@ -13,7 +13,7 @@ Licensed under the MIT License. com.azure azure-cosmos-encryption - 1.9.0-beta.1 + 2.0.0-beta.1 Encryption Plugin for Azure Cosmos DB SDK This Package contains Encryption Plugin for Microsoft Azure Cosmos SDK jar From 98a86a911adc6d48f8e8f511e6ab99121ce1c605 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Wed, 2 Nov 2022 14:18:42 -0700 Subject: [PATCH 12/31] Resolving comments --- .../encryption/AsyncEncryptionBenchmark.java | 2 +- .../encryption/EncryptionCodeSnippet.java | 2 +- .../com/azure/cosmos/encryption/Program.java | 2 +- .../cosmos/encryption/ReadmeSamples.java | 2 +- .../CosmosEncryptionClientCachesTest.java | 2 +- .../encryption/DotNetCompatibleTest.java | 2 +- .../EncryptionAsyncApiCrudTest.java | 4 +- ...ryptionCosmosEncryptionChangeFeedTest.java | 2 +- .../EncryptionProcessorAndSettingsTest.java | 4 +- .../cosmos/models/ClientEncryptionPolicy.java | 46 ++----------------- .../com/azure/cosmos/CosmosContainerTest.java | 4 +- 11 files changed, 16 insertions(+), 56 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/encryption/AsyncEncryptionBenchmark.java b/sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/encryption/AsyncEncryptionBenchmark.java index fc4f2ff397ecf..22d91a67c6ea0 100644 --- a/sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/encryption/AsyncEncryptionBenchmark.java +++ b/sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/encryption/AsyncEncryptionBenchmark.java @@ -549,7 +549,7 @@ private void createEncryptionDatabaseAndContainer() { includedPath.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName()); encryptionPaths.add(includedPath); } - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(encryptionPaths); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(encryptionPaths, 1); CosmosContainerProperties containerProperties = new CosmosContainerProperties(this.configuration.getCollectionId(), Configuration.DEFAULT_PARTITION_KEY_PATH); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/EncryptionCodeSnippet.java b/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/EncryptionCodeSnippet.java index 4eb0467d764c7..a67234a644b88 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/EncryptionCodeSnippet.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/EncryptionCodeSnippet.java @@ -124,7 +124,7 @@ void createContainerWithClientEncryptionPolicy(CosmosAsyncClient client) { paths.add(includedPath6); paths.add(includedPath7); - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 1); String containerId = "myCol"; CosmosContainerProperties properties = new CosmosContainerProperties(containerId, "/mypk"); properties.setClientEncryptionPolicy(clientEncryptionPolicy); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/Program.java b/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/Program.java index 4c8a47ec840a7..97666cb3bfcc1 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/Program.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/Program.java @@ -155,7 +155,7 @@ private static void initialize(CosmosEncryptionAsyncClient cosmosEncryptionAsync CosmosContainerProperties containerProperties = new CosmosContainerProperties(Program.containerId, "/purchaseOrderNumber"); - containerProperties.setClientEncryptionPolicy(new ClientEncryptionPolicy(paths)); + containerProperties.setClientEncryptionPolicy(new ClientEncryptionPolicy(paths, 1)); cosmosEncryptionAsyncDatabase.getCosmosAsyncDatabase().createContainer(containerProperties, ThroughputProperties.createManualThroughput(1000)).block(); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/ReadmeSamples.java b/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/ReadmeSamples.java index 3e9dbd3104c7d..33792e5df217b 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/ReadmeSamples.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/ReadmeSamples.java @@ -86,7 +86,7 @@ public void createCosmosEncryptionContainer() { List paths = new ArrayList<>(); paths.add(includedPath); - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 1); CosmosContainerProperties properties = new CosmosContainerProperties("", "/mypk"); properties.setClientEncryptionPolicy(clientEncryptionPolicy); return cosmosEncryptionAsyncDatabase.getCosmosAsyncDatabase().createContainer(properties); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionClientCachesTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionClientCachesTest.java index 8335580f58e71..7896378e4a83e 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionClientCachesTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionClientCachesTest.java @@ -68,7 +68,7 @@ public void before_CosmosItemTest() { CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName(), metadata2).block(); //Create collection with clientEncryptionPolicy - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(false)); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(false), 1); CosmosContainerProperties containerProperties = new CosmosContainerProperties("TestCollForEncryptionCacheTest" , "/mypk"); containerProperties.setClientEncryptionPolicy(clientEncryptionPolicy); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java index c7f036be371a2..81a31b184fbce 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java @@ -70,7 +70,7 @@ public void before_CosmosItemTest() throws IOException { cosmosEncryptionAsyncClient.getCosmosEncryptionAsyncDatabase(cosmosAsyncDatabase.getId()); ClientEncryptionPolicy clientEncryptionPolicy = - new ClientEncryptionPolicy(getPaths(false)); + new ClientEncryptionPolicy(getPaths(false), 1); String containerId = UUID.randomUUID().toString(); CosmosContainerProperties containerProperties = new CosmosContainerProperties(containerId, "/mypk"); containerProperties.setClientEncryptionPolicy(clientEncryptionPolicy); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java index 03b7e5552d670..c4c21d269f56b 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java @@ -371,7 +371,7 @@ public void crudQueryStaleCache() { cosmosEncryptionAsyncClient.getCosmosEncryptionAsyncDatabase(asyncClient.getDatabase(databaseId)); String containerId = UUID.randomUUID().toString(); - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(false)); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(false), 1); createEncryptionContainer(cosmosEncryptionAsyncDatabase, clientEncryptionPolicy, containerId); CosmosEncryptionAsyncContainer encryptionAsyncContainerOriginal = cosmosEncryptionAsyncDatabase.getCosmosEncryptionAsyncContainer(containerId); @@ -404,7 +404,7 @@ public void crudQueryStaleCache() { //Deleting and creating container encryptionAsyncContainerOriginal.getCosmosAsyncContainer().delete().block(); - ClientEncryptionPolicy policyWithOneEncryptionPolicy = new ClientEncryptionPolicy(getPathWithOneEncryptionField()); + ClientEncryptionPolicy policyWithOneEncryptionPolicy = new ClientEncryptionPolicy(getPathWithOneEncryptionField(), 1); createEncryptionContainer(cosmosEncryptionAsyncDatabase, policyWithOneEncryptionPolicy, containerId); CosmosEncryptionAsyncContainer encryptionAsyncContainerNew = getNewEncryptionContainerProxyObject(cosmosEncryptionAsyncDatabase.getCosmosAsyncDatabase().getId(), containerId); encryptionAsyncContainerNew.createItem(encryptionPojo, diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionCosmosEncryptionChangeFeedTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionCosmosEncryptionChangeFeedTest.java index 38c9ae0561585..401f85b659d4e 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionCosmosEncryptionChangeFeedTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionCosmosEncryptionChangeFeedTest.java @@ -264,7 +264,7 @@ private CosmosAsyncContainer createLeaseCollection(int provisionedThroughput) { } private CosmosEncryptionAsyncContainer createFeedCollection() { - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(false)); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(false), 1); String containerId = UUID.randomUUID().toString(); CosmosContainerProperties properties = new CosmosContainerProperties(containerId, "/mypk"); properties.setClientEncryptionPolicy(clientEncryptionPolicy); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/implementation/EncryptionProcessorAndSettingsTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/implementation/EncryptionProcessorAndSettingsTest.java index 066ae900e80fb..18913e31c1790 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/implementation/EncryptionProcessorAndSettingsTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/implementation/EncryptionProcessorAndSettingsTest.java @@ -206,7 +206,7 @@ private ClientEncryptionPolicy generateClientEncryptionPolicy() { includedPath1.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName()); List paths = new ArrayList<>(); paths.add(includedPath1); - return new ClientEncryptionPolicy(paths); + return new ClientEncryptionPolicy(paths, 1); } private CosmosContainerProperties generateContainerWithCosmosEncryptionPolicy() { @@ -219,7 +219,7 @@ private CosmosContainerProperties generateContainerWithCosmosEncryptionPolicy() includedPath1.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName()); List paths = new ArrayList<>(); paths.add(includedPath1); - return containerProperties.setClientEncryptionPolicy(new ClientEncryptionPolicy(paths)); + return containerProperties.setClientEncryptionPolicy(new ClientEncryptionPolicy(paths, 1)); } private CosmosClientEncryptionKeyProperties generateClientEncryptionKeyProperties() throws JsonProcessingException { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java index 5952e9988638f..75a277bfdea77 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java @@ -20,35 +20,20 @@ */ public final class ClientEncryptionPolicy { - private JsonSerializable jsonSerializable; - /** * Paths of the item that need encryption along with path-specific settings. */ @JsonProperty("includedPaths") - private List includedPaths; + private final List includedPaths; @JsonProperty("policyFormatVersion") - private int policyFormatVersion; - - /** - * Constructor. - * - * @param paths list of path of the item that need encryption along with path-specific settings. - * the PolicyFormatVersion will be set to 1 which is the default value. - * Note: If you need to include partition key or id field paths as part of the ClientEncryptionPolicy, please set PolicyFormatVersion to 2. - */ - public ClientEncryptionPolicy(List paths) { - this.policyFormatVersion = 1; - validateIncludedPaths(paths, policyFormatVersion); - this.includedPaths = paths; - } + private final int policyFormatVersion; /** * Constructor. * * @param paths list of path of the item that need encryption along with path-specific settings. - * @param policyFormatVersion version of the client encryption policy definition. Current supported versions are 1 and 2. Default version is 1. + * @param policyFormatVersion version of the client encryption policy definition. Current supported versions are 1 and 2. * Note: If you need to include partition key or id field paths as part of the ClientEncryptionPolicy, please set PolicyFormatVersion to 2. */ public ClientEncryptionPolicy(List paths, int policyFormatVersion) { @@ -60,31 +45,6 @@ public ClientEncryptionPolicy(List paths, int poli this.includedPaths = paths; } - /** - * Constructor. - */ - public ClientEncryptionPolicy() { - this.jsonSerializable = new JsonSerializable(); - } - - /** - * Constructor. - * - * @param jsonString the json string that represents the client encryption policy. - */ - ClientEncryptionPolicy(String jsonString) { - this.jsonSerializable = new JsonSerializable(jsonString); - } - - /** - * Constructor. - * - * @param objectNode the object node that represents the client encryption policy. - */ - ClientEncryptionPolicy(ObjectNode objectNode) { - this.jsonSerializable = new JsonSerializable(objectNode); - } - /** * Gets the list of paths of the item that need encryption along with path-specific settings. * @return includedPaths diff --git a/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerTest.java b/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerTest.java index dfed1d3c30d87..0a622597376c7 100644 --- a/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerTest.java +++ b/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerTest.java @@ -164,7 +164,7 @@ public void createContainer_withPartitionKeyWrongPolicyFormatVersion() { paths.add(path1); paths.add(path2); - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 1); CosmosContainerResponse containerResponse = null; //Verify partition key in CosmosContainerProperties constructor with encrypted field. @@ -239,7 +239,7 @@ public void createEncryptionPolicy_withIdWrongPolicyFormatVersion() { paths.add(path2); try { - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 1); fail("clientEncryptionPolicy should fail as id which is part of the partition key cannot be encrypted with " + "PolicyFormatVersion 1."); } catch (IllegalArgumentException ex) { From 8e67cb040e44b10e795d62be610b551d00bf04bc Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Wed, 2 Nov 2022 14:52:02 -0700 Subject: [PATCH 13/31] Fixing builds --- .../azure/cosmos/models/ClientEncryptionPolicy.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java index 75a277bfdea77..1120c9fb2b3c8 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java @@ -20,14 +20,16 @@ */ public final class ClientEncryptionPolicy { + private JsonSerializable jsonSerializable; + /** * Paths of the item that need encryption along with path-specific settings. */ @JsonProperty("includedPaths") - private final List includedPaths; + private List includedPaths; @JsonProperty("policyFormatVersion") - private final int policyFormatVersion; + private int policyFormatVersion; /** * Constructor. @@ -45,6 +47,13 @@ public ClientEncryptionPolicy(List paths, int poli this.includedPaths = paths; } + /** + * Constructor. + */ + public ClientEncryptionPolicy() { + this.jsonSerializable = new JsonSerializable(); + } + /** * Gets the list of paths of the item that need encryption along with path-specific settings. * @return includedPaths From 1cd45bf95fa52b9f35426846723360b75490bfac Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Wed, 2 Nov 2022 15:43:54 -0700 Subject: [PATCH 14/31] Resolving comments --- .../CosmosEncryptionAsyncContainer.java | 24 ++++++++++++------- .../implementation/EncryptionProcessor.java | 23 ++++++++++++++---- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java index 93a67cf5d459f..2567ea08b1f8c 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java @@ -210,14 +210,17 @@ public Mono> deleteItem(String itemId, return container.deleteItem(itemId, partitionKey, requestOptions); } - private String checkAndGetEncryptedId(String itemId, EncryptionSettings encryptionSettings) throws MicrosoftDataEncryptionException { - if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream().anyMatch(includedPath -> includedPath.getPath().substring(1).equals(Constants.PROPERTY_NAME_ID))) { + private String checkAndGetEncryptedId(String itemId, EncryptionSettings encryptionSettings) + throws MicrosoftDataEncryptionException { + if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream(). + anyMatch(includedPath -> includedPath.getPath().substring(1).equals(Constants.PROPERTY_NAME_ID))) { return itemId; } return this.encryptionProcessor.encryptAndSerializeValue(encryptionSettings, itemId, Constants.PROPERTY_NAME_ID); } - private PartitionKey checkAndGetEncryptedPartitionKey(PartitionKey partitionKey, EncryptionSettings encryptionSettings) throws JsonProcessingException, MicrosoftDataEncryptionException { + private PartitionKey checkAndGetEncryptedPartitionKey(PartitionKey partitionKey, EncryptionSettings encryptionSettings) + throws JsonProcessingException, MicrosoftDataEncryptionException { if (encryptionSettings.getPartitionKeyPaths().isEmpty() || partitionKey == null) { return partitionKey; } @@ -228,15 +231,18 @@ private PartitionKey checkAndGetEncryptedPartitionKey(PartitionKey partitionKey, for (String path : encryptionSettings.getPartitionKeyPaths()) { // case: partition key path is /a/b/c and the client encryption policy has /a in path. - // hence encrypt the partition key value with using its top level path /a since /c would have been encrypted in the document using /a's policy. + // hence encrypt the partition key value with using its top level path /a since + // /c would have been encrypted in the document using /a's policy. String partitionKeyPath = path.split("/")[0]; String childPartitionKey = arrayNode.elements().next().toString(); - if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream().anyMatch(includedPath -> includedPath.getPath().substring(1).equals(partitionKeyPath))) { + if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream(). + anyMatch(includedPath -> includedPath.getPath().substring(1).equals(partitionKeyPath))) { partitionKeyBuilder.add(childPartitionKey); continue; } - partitionKeyBuilder.add(this.encryptionProcessor.encryptAndSerializeValue(encryptionSettings, childPartitionKey, partitionKeyPath)); + partitionKeyBuilder.add(this.encryptionProcessor. + encryptAndSerializeValue(encryptionSettings, childPartitionKey, partitionKeyPath)); } return partitionKeyBuilder.build(); } @@ -246,10 +252,12 @@ private PartitionKey checkAndGetEncryptedPartitionKey(PartitionKey partitionKey, throw new MicrosoftDataEncryptionException("There should only be 1 PartitionKeyPath."); } String partitionKeyPath = encryptionSettings.getPartitionKeyPaths().get(0); - if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream().anyMatch(includedPath -> includedPath.getPath().substring(1).equals(partitionKeyPath))) { + if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream(). + anyMatch(includedPath -> includedPath.getPath().substring(1).equals(partitionKeyPath))) { return partitionKey; } - return new PartitionKey(this.encryptionProcessor.encryptAndSerializeValue(encryptionSettings, partitionKey.toString(), partitionKeyPath)); + return new PartitionKey(this.encryptionProcessor. + encryptAndSerializeValue(encryptionSettings, partitionKey.toString(), partitionKeyPath)); } } diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java index ce54e9d51ea5e..10f9a29ffda4a 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java @@ -664,17 +664,32 @@ public static JsonNode toJsonNode(byte[] serializedBytes, TypeMarker typeMarker) } private String convertToBase64UriSafeString(byte[] bytesToProcess) { - String base64String = new String(Base64.getDecoder().decode(bytesToProcess)); + StringBuilder base64String = new StringBuilder(new String(Base64.getDecoder().decode(bytesToProcess))); // Base 64 Encoding with URL and Filename Safe Alphabet https://datatracker.ietf.org/doc/html/rfc4648#section-5 // https://docs.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-item-limits, due to base64 conversion and encryption // the permissible size of the property will further reduce. - return base64String.replaceAll("/","_").replaceAll("\\+","-"); + replaceString(base64String, "/", "_"); + replaceString(base64String, "\\+", "-"); + return base64String.toString(); } private byte[] convertFromBase64UriSafeString(String base64UriSafeString) { - base64UriSafeString.replaceAll("_", "/").replaceAll("-", "\\+"); - return Base64.getEncoder().encode(base64UriSafeString.getBytes()); + StringBuilder base64String = new StringBuilder(base64UriSafeString); + + replaceString(base64String, "_", "/"); + replaceString(base64String, "-", "\\+"); + return Base64.getEncoder().encode(base64String.toString().getBytes()); + } + + private StringBuilder replaceString(StringBuilder sb, + String toReplace, + String replacement) { + int index = -1; + while ((index = sb.lastIndexOf(toReplace)) != -1) { + sb.replace(index, index + toReplace.length(), replacement); + } + return sb; } public enum TypeMarker { From 22ad6c99a57efe2a55ffe90066c1bce29780652d Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Wed, 2 Nov 2022 16:29:21 -0700 Subject: [PATCH 15/31] Fixing test cases --- .../cosmos/models/ClientEncryptionPolicy.java | 15 ++++++++++++++- .../com/azure/cosmos/CosmosContainerTest.java | 6 +++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java index 1120c9fb2b3c8..41ead1a19d99b 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java @@ -35,7 +35,20 @@ public final class ClientEncryptionPolicy { * Constructor. * * @param paths list of path of the item that need encryption along with path-specific settings. - * @param policyFormatVersion version of the client encryption policy definition. Current supported versions are 1 and 2. + * the PolicyFormatVersion will be set to 1 which is the default value. + * Note: If you need to include partition key or id field paths as part of the ClientEncryptionPolicy, please set PolicyFormatVersion to 2. + */ + public ClientEncryptionPolicy(List paths) { + this.policyFormatVersion = 1; + validateIncludedPaths(paths, policyFormatVersion); + this.includedPaths = paths; + } + + /** + * Constructor. + * + * @param paths list of path of the item that need encryption along with path-specific settings. + * @param policyFormatVersion version of the client encryption policy definition. Current supported versions are 1 and 2. Default version is 1. * Note: If you need to include partition key or id field paths as part of the ClientEncryptionPolicy, please set PolicyFormatVersion to 2. */ public ClientEncryptionPolicy(List paths, int policyFormatVersion) { diff --git a/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerTest.java b/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerTest.java index 0a622597376c7..3a12249a7559f 100644 --- a/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerTest.java +++ b/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerTest.java @@ -174,7 +174,7 @@ public void createContainer_withPartitionKeyWrongPolicyFormatVersion() { fail("createContainer should fail as mypk which is part of the partition key cannot be encrypted with " + "PolicyFormatVersion 1."); } catch (IllegalArgumentException ex) { - assertThat(ex.getMessage()).isEqualTo("Path mypk which is part of the partition key cannot be encrypted with PolicyFormatVersion 1"); + assertThat(ex.getMessage()).isEqualTo("Path mypk which is part of the partition key cannot be encrypted with PolicyFormatVersion 1. Please use PolicyFormatVersion 2."); } @@ -187,7 +187,7 @@ public void createContainer_withPartitionKeyWrongPolicyFormatVersion() { fail("createContainer should fail as mypk which is part of the partition key cannot be encrypted with " + "PolicyFormatVersion 1."); } catch (IllegalArgumentException ex) { - assertThat(ex.getMessage()).isEqualTo("Path mypk which is part of the partition key cannot be encrypted with PolicyFormatVersion 1"); + assertThat(ex.getMessage()).isEqualTo("Path mypk which is part of the partition key cannot be encrypted with PolicyFormatVersion 1. Please use PolicyFormatVersion 2."); } @@ -205,7 +205,7 @@ public void createContainer_withPartitionKeyWrongPolicyFormatVersion() { fail("createContainer should fail as mypk which is part of the partition key cannot be encrypted with " + "PolicyFormatVersion 1."); } catch (IllegalArgumentException ex) { - assertThat(ex.getMessage()).isEqualTo("Path mypk which is part of the partition key cannot be encrypted with PolicyFormatVersion 1"); + assertThat(ex.getMessage()).isEqualTo("Path mypk which is part of the partition key cannot be encrypted with PolicyFormatVersion 1. Please use PolicyFormatVersion 2."); } //This should pass as we check only the first key of the composite key. From 13d32824ad84aacc071d055a345342f6984ce303 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Tue, 15 Nov 2022 11:48:49 -0800 Subject: [PATCH 16/31] FIxing builds --- sdk/cosmos/azure-cosmos-benchmark/pom.xml | 2 +- sdk/cosmos/azure-cosmos-encryption/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-benchmark/pom.xml b/sdk/cosmos/azure-cosmos-benchmark/pom.xml index 5812ebf7d2f24..dc6e81052501d 100644 --- a/sdk/cosmos/azure-cosmos-benchmark/pom.xml +++ b/sdk/cosmos/azure-cosmos-benchmark/pom.xml @@ -57,7 +57,7 @@ Licensed under the MIT License. com.azure azure-cosmos-encryption - 2.0.0-beta.1 + 1.9.0-beta.1 diff --git a/sdk/cosmos/azure-cosmos-encryption/pom.xml b/sdk/cosmos/azure-cosmos-encryption/pom.xml index 035dc5839131a..371a1b93fc33f 100644 --- a/sdk/cosmos/azure-cosmos-encryption/pom.xml +++ b/sdk/cosmos/azure-cosmos-encryption/pom.xml @@ -13,7 +13,7 @@ Licensed under the MIT License. com.azure azure-cosmos-encryption - 2.0.0-beta.1 + 1.9.0-beta.1 Encryption Plugin for Azure Cosmos DB SDK This Package contains Encryption Plugin for Microsoft Azure Cosmos SDK jar From fe789dbd397fabf1b6d9afc5e9adc8b00d03727f Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Tue, 15 Nov 2022 12:22:02 -0800 Subject: [PATCH 17/31] Fixing versioning --- eng/jacoco-test-coverage/pom.xml | 2 +- eng/versioning/version_client.txt | 2 +- sdk/cosmos/azure-cosmos-benchmark/pom.xml | 2 +- sdk/cosmos/azure-cosmos-encryption/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/jacoco-test-coverage/pom.xml b/eng/jacoco-test-coverage/pom.xml index e2d6dc2c32e8b..f48c330027a19 100644 --- a/eng/jacoco-test-coverage/pom.xml +++ b/eng/jacoco-test-coverage/pom.xml @@ -188,7 +188,7 @@ com.azure azure-cosmos-encryption - 1.9.0-beta.1 + 2.0.0-beta.1 com.azure diff --git a/eng/versioning/version_client.txt b/eng/versioning/version_client.txt index dae5959a385a7..61f077cb64cb5 100644 --- a/eng/versioning/version_client.txt +++ b/eng/versioning/version_client.txt @@ -93,7 +93,7 @@ com.azure:azure-cosmos-dotnet-benchmark;4.0.1-beta.1;4.0.1-beta.1 com.azure.cosmos.spark:azure-cosmos-spark_3_2-12;1.0.0-beta.1;1.0.0-beta.1 com.azure.cosmos.spark:azure-cosmos-spark_3-1_2-12;4.14.1;4.15.0-beta.1 com.azure.cosmos.spark:azure-cosmos-spark_3-2_2-12;4.14.1;4.15.0-beta.1 -com.azure:azure-cosmos-encryption;1.8.1;1.9.0-beta.1 +com.azure:azure-cosmos-encryption;1.8.1;2.0.0-beta.1 com.azure:azure-data-appconfiguration;1.3.8;1.4.0-beta.1 com.azure:azure-data-appconfiguration-perf;1.0.0-beta.1;1.0.0-beta.1 com.azure:azure-data-schemaregistry;1.3.0;1.4.0-beta.1 diff --git a/sdk/cosmos/azure-cosmos-benchmark/pom.xml b/sdk/cosmos/azure-cosmos-benchmark/pom.xml index dc6e81052501d..5812ebf7d2f24 100644 --- a/sdk/cosmos/azure-cosmos-benchmark/pom.xml +++ b/sdk/cosmos/azure-cosmos-benchmark/pom.xml @@ -57,7 +57,7 @@ Licensed under the MIT License. com.azure azure-cosmos-encryption - 1.9.0-beta.1 + 2.0.0-beta.1 diff --git a/sdk/cosmos/azure-cosmos-encryption/pom.xml b/sdk/cosmos/azure-cosmos-encryption/pom.xml index 371a1b93fc33f..035dc5839131a 100644 --- a/sdk/cosmos/azure-cosmos-encryption/pom.xml +++ b/sdk/cosmos/azure-cosmos-encryption/pom.xml @@ -13,7 +13,7 @@ Licensed under the MIT License. com.azure azure-cosmos-encryption - 1.9.0-beta.1 + 2.0.0-beta.1 Encryption Plugin for Azure Cosmos DB SDK This Package contains Encryption Plugin for Microsoft Azure Cosmos SDK jar From 03bf1bbf10ab2fdd1574ecf742c074589d7f3d9d Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Tue, 15 Nov 2022 12:31:16 -0800 Subject: [PATCH 18/31] Resolving commentsg --- sdk/cosmos/azure-cosmos-encryption/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/CHANGELOG.md b/sdk/cosmos/azure-cosmos-encryption/CHANGELOG.md index 8b06f1d446cbc..4df6f4a7d13bc 100644 --- a/sdk/cosmos/azure-cosmos-encryption/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-encryption/CHANGELOG.md @@ -4,9 +4,9 @@ #### Features Added * Added support for allowing partition key path and id to be part of client encryption policy - See [PR 30678](https://github.com/Azure/azure-sdk-for-java/pull/30678) -* Added support for allowing the encryption of partition key and id - See [PR 30678](https://github.com/Azure/azure-sdk-for-java/pull/30678) #### Breaking Changes +* Adds code to support ParititonKey and Id encryption, when the PolicyFormatVersion is set to 2 - See [PR 30678](https://github.com/Azure/azure-sdk-for-java/pull/30678) #### Bugs Fixed From a1bfcce7a48a4b159643d59c4a81a45e0eaea8d3 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Tue, 15 Nov 2022 12:44:31 -0800 Subject: [PATCH 19/31] Resolving commentsg --- eng/versioning/version_client.txt | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/eng/versioning/version_client.txt b/eng/versioning/version_client.txt index 0a03d8a4ff3b3..40df360e8e372 100644 --- a/eng/versioning/version_client.txt +++ b/eng/versioning/version_client.txt @@ -32,7 +32,6 @@ # When adding a new library the dependency-version and current-version will be the same until the first release is # performed. So, if com.azure:azure-new-library is added with major version 2 it'll use # com.azure:azure-new-library;2.0.0-beta.1;2.0.0-beta.1 as its initial version tag. - com.azure:azure-sdk-all;1.0.0;1.0.0 com.azure:azure-sdk-parent;1.6.0;1.6.0 com.azure:azure-client-sdk-parent;1.7.0;1.7.0 @@ -92,9 +91,10 @@ com.azure:azure-cosmos-benchmark;4.0.1-beta.1;4.0.1-beta.1 com.azure:azure-cosmos-dotnet-benchmark;4.0.1-beta.1;4.0.1-beta.1 com.azure.cosmos.spark:azure-cosmos-spark_3_2-12;1.0.0-beta.1;1.0.0-beta.1 com.azure.cosmos.spark:azure-cosmos-spark_3-1_2-12;4.14.1;4.15.0-beta.1 +com.azure.cosmos.spark:azure-cosmos-spark_3-2_2-12;4.14.1;4.15.0-beta.1 com.azure.cosmos.spark:azure-cosmos-spark_3-3_2-12;4.15.0-beta.1;4.15.0-beta.1 -com.azure:azure-cosmos-encryption;1.8.1;2.0.0-beta.1 -com.azure:azure-data-appconfiguration;1.3.8;1.4.0-beta.1 +com.azure:azure-cosmos-encryption;1.8.1;1.9.0-beta.1 +com.azure:azure-data-appconfiguration;1.3.9;1.4.0-beta.1 com.azure:azure-data-appconfiguration-perf;1.0.0-beta.1;1.0.0-beta.1 com.azure:azure-data-schemaregistry;1.3.0;1.4.0-beta.1 com.azure:azure-data-schemaregistry-apacheavro;1.1.0;1.2.0-beta.1 @@ -389,7 +389,6 @@ com.azure.resourcemanager:azure-resourcemanager-securitydevops;1.0.0-beta.1;1.0. com.azure.resourcemanager:azure-resourcemanager-appcomplianceautomation;1.0.0-beta.1;1.0.0-beta.2 com.azure.tools:azure-sdk-archetype;1.0.0;1.2.0-beta.1 com.azure.tools:azure-sdk-build-tool;1.0.0-beta.1;1.0.0-beta.2 - # Unreleased dependencies: Copy the entry from above, prepend "unreleased_" and remove the current # version. Unreleased dependencies are only valid for dependency versions. # Format; @@ -398,8 +397,6 @@ com.azure.tools:azure-sdk-build-tool;1.0.0-beta.1;1.0.0-beta.2 # In the pom, the version update tag after the version should name the unreleased package and the dependency version: # unreleased_com.azure:azure-core;1.35.0-beta.1 - - # Released Beta dependencies: Copy the entry from above, prepend "beta_", remove the current # version and set the version to the released beta. Released beta dependencies are only valid # for dependency versions. These entries are specifically for when we've released a beta for @@ -408,4 +405,4 @@ unreleased_com.azure:azure-core;1.35.0-beta.1 # beta_:;dependency-version # note: Released beta versions will not be manipulated with the automatic PR creation code. beta_com.azure:azure-communication-common;1.3.0-beta.1 -beta_com.azure:azure-identity;1.7.0-beta.1 +beta_com.azure:azure-identity;1.7.0-beta.1 \ No newline at end of file From 3c6c807a94a4792581f52b887a38ffe49fea85f6 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Tue, 15 Nov 2022 12:49:12 -0800 Subject: [PATCH 20/31] Fixing versioning --- eng/versioning/version_client.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eng/versioning/version_client.txt b/eng/versioning/version_client.txt index 40df360e8e372..fb48820a61a6d 100644 --- a/eng/versioning/version_client.txt +++ b/eng/versioning/version_client.txt @@ -32,6 +32,7 @@ # When adding a new library the dependency-version and current-version will be the same until the first release is # performed. So, if com.azure:azure-new-library is added with major version 2 it'll use # com.azure:azure-new-library;2.0.0-beta.1;2.0.0-beta.1 as its initial version tag. + com.azure:azure-sdk-all;1.0.0;1.0.0 com.azure:azure-sdk-parent;1.6.0;1.6.0 com.azure:azure-client-sdk-parent;1.7.0;1.7.0 @@ -93,7 +94,7 @@ com.azure.cosmos.spark:azure-cosmos-spark_3_2-12;1.0.0-beta.1;1.0.0-beta.1 com.azure.cosmos.spark:azure-cosmos-spark_3-1_2-12;4.14.1;4.15.0-beta.1 com.azure.cosmos.spark:azure-cosmos-spark_3-2_2-12;4.14.1;4.15.0-beta.1 com.azure.cosmos.spark:azure-cosmos-spark_3-3_2-12;4.15.0-beta.1;4.15.0-beta.1 -com.azure:azure-cosmos-encryption;1.8.1;1.9.0-beta.1 +com.azure:azure-cosmos-encryption;1.8.1;2.0.0-beta.1 com.azure:azure-data-appconfiguration;1.3.9;1.4.0-beta.1 com.azure:azure-data-appconfiguration-perf;1.0.0-beta.1;1.0.0-beta.1 com.azure:azure-data-schemaregistry;1.3.0;1.4.0-beta.1 @@ -389,6 +390,7 @@ com.azure.resourcemanager:azure-resourcemanager-securitydevops;1.0.0-beta.1;1.0. com.azure.resourcemanager:azure-resourcemanager-appcomplianceautomation;1.0.0-beta.1;1.0.0-beta.2 com.azure.tools:azure-sdk-archetype;1.0.0;1.2.0-beta.1 com.azure.tools:azure-sdk-build-tool;1.0.0-beta.1;1.0.0-beta.2 + # Unreleased dependencies: Copy the entry from above, prepend "unreleased_" and remove the current # version. Unreleased dependencies are only valid for dependency versions. # Format; @@ -397,6 +399,8 @@ com.azure.tools:azure-sdk-build-tool;1.0.0-beta.1;1.0.0-beta.2 # In the pom, the version update tag after the version should name the unreleased package and the dependency version: # unreleased_com.azure:azure-core;1.35.0-beta.1 + + # Released Beta dependencies: Copy the entry from above, prepend "beta_", remove the current # version and set the version to the released beta. Released beta dependencies are only valid # for dependency versions. These entries are specifically for when we've released a beta for From 366928724a1d84ed725692f90f0800127db89788 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Tue, 15 Nov 2022 12:52:17 -0800 Subject: [PATCH 21/31] Resolving comments --- .../azure/cosmos/models/ClientEncryptionPolicy.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java index 41ead1a19d99b..14b54958d032b 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java @@ -7,7 +7,6 @@ import com.azure.cosmos.implementation.JsonSerializable; import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.ArrayList; import java.util.List; @@ -29,7 +28,7 @@ public final class ClientEncryptionPolicy { private List includedPaths; @JsonProperty("policyFormatVersion") - private int policyFormatVersion; + private final int policyFormatVersion; /** * Constructor. @@ -60,13 +59,6 @@ public ClientEncryptionPolicy(List paths, int poli this.includedPaths = paths; } - /** - * Constructor. - */ - public ClientEncryptionPolicy() { - this.jsonSerializable = new JsonSerializable(); - } - /** * Gets the list of paths of the item that need encryption along with path-specific settings. * @return includedPaths From 598b675f12fadf01456871bdb8364c00a34646e0 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Tue, 15 Nov 2022 14:00:37 -0800 Subject: [PATCH 22/31] Fixing builds --- .../com/azure/cosmos/models/ClientEncryptionPolicy.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java index 14b54958d032b..5ea5efe4ac4e0 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ClientEncryptionPolicy.java @@ -28,7 +28,7 @@ public final class ClientEncryptionPolicy { private List includedPaths; @JsonProperty("policyFormatVersion") - private final int policyFormatVersion; + private int policyFormatVersion; /** * Constructor. @@ -59,6 +59,13 @@ public ClientEncryptionPolicy(List paths, int poli this.includedPaths = paths; } + /** + * Constructor. + */ + public ClientEncryptionPolicy() { + this.jsonSerializable = new JsonSerializable(); + } + /** * Gets the list of paths of the item that need encryption along with path-specific settings. * @return includedPaths From 6f81259a25238c1b55e48bdcb6653f30e896c974 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Tue, 15 Nov 2022 14:57:08 -0800 Subject: [PATCH 23/31] Fixing spotbugs --- .../encryption/implementation/EncryptionProcessor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java index 10f9a29ffda4a..0a61ebe1a05bb 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java @@ -450,7 +450,7 @@ public void encryptAndSerializeProperty(EncryptionSettings encryptionSettings, J } public String encryptAndSerializeValue(EncryptionSettings encryptionSettings, String propertyValue, String propertyName) throws MicrosoftDataEncryptionException { - JsonNode propertyValueHolder = toJsonNode(propertyValue.getBytes(), TypeMarker.STRING); + JsonNode propertyValueHolder = toJsonNode(propertyValue.getBytes(StandardCharsets.UTF_8), TypeMarker.STRING); return convertToBase64UriSafeString(encryptAndSerializeValue(encryptionSettings, null, propertyValueHolder, propertyName)); } @@ -473,7 +473,7 @@ public byte[] encryptAndSerializeValue(EncryptionSettings encryptionSettings, Ob if (propertyName.equals(Constants.PROPERTY_NAME_ID)) { // case: id does not support '/','\','?','#'. Convert Base64 string to Uri safe string - cipherTextWithTypeMarker = convertToBase64UriSafeString(cipherTextWithTypeMarker).getBytes(); + cipherTextWithTypeMarker = convertToBase64UriSafeString(cipherTextWithTypeMarker).getBytes(StandardCharsets.UTF_8); } if (objectNode != null && !objectNode.isNull()) { @@ -664,7 +664,7 @@ public static JsonNode toJsonNode(byte[] serializedBytes, TypeMarker typeMarker) } private String convertToBase64UriSafeString(byte[] bytesToProcess) { - StringBuilder base64String = new StringBuilder(new String(Base64.getDecoder().decode(bytesToProcess))); + StringBuilder base64String = new StringBuilder(new String(Base64.getUrlDecoder().decode(bytesToProcess))); // Base 64 Encoding with URL and Filename Safe Alphabet https://datatracker.ietf.org/doc/html/rfc4648#section-5 // https://docs.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-item-limits, due to base64 conversion and encryption @@ -679,7 +679,7 @@ private byte[] convertFromBase64UriSafeString(String base64UriSafeString) { replaceString(base64String, "_", "/"); replaceString(base64String, "-", "\\+"); - return Base64.getEncoder().encode(base64String.toString().getBytes()); + return Base64.getUrlEncoder().encode(base64String.toString().getBytes(StandardCharsets.UTF_8)); } private StringBuilder replaceString(StringBuilder sb, From e555b03fa676fe5cf32f909bc347680d34e419f8 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Tue, 15 Nov 2022 15:23:16 -0800 Subject: [PATCH 24/31] Fixing spotbugs --- .../cosmos/encryption/implementation/EncryptionProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java index 0a61ebe1a05bb..96a5d42792d4d 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java @@ -664,7 +664,7 @@ public static JsonNode toJsonNode(byte[] serializedBytes, TypeMarker typeMarker) } private String convertToBase64UriSafeString(byte[] bytesToProcess) { - StringBuilder base64String = new StringBuilder(new String(Base64.getUrlDecoder().decode(bytesToProcess))); + StringBuilder base64String = new StringBuilder(new String(Base64.getUrlDecoder().decode(bytesToProcess), StandardCharsets.UTF_8)); // Base 64 Encoding with URL and Filename Safe Alphabet https://datatracker.ietf.org/doc/html/rfc4648#section-5 // https://docs.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-item-limits, due to base64 conversion and encryption From f9f293d20ac9d6d6aed0b26535f90943f7875806 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Tue, 15 Nov 2022 18:54:53 -0800 Subject: [PATCH 25/31] Fixing builds --- .../azure/cosmos/encryption/CosmosEncryptionAsyncClient.java | 2 +- .../java/com/azure/cosmos/encryption/DotNetCompatibleTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClient.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClient.java index 9de68048575ff..aa42d6639164b 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClient.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClient.java @@ -222,7 +222,7 @@ private CosmosContainerProperties getContainerPropertiesWithVersionValidation(Co throw new IllegalArgumentException("Container without client encryption policy cannot be used"); } - if (cosmosContainerResponse.getProperties().getClientEncryptionPolicy().getPolicyFormatVersion() > 1) { + if (cosmosContainerResponse.getProperties().getClientEncryptionPolicy().getPolicyFormatVersion() > 2) { throw new UnsupportedOperationException("This version of the Encryption library cannot be used with this " + "container. Please upgrade to the latest version of the same."); } diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java index 81a31b184fbce..ec8272b48c31c 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java @@ -70,7 +70,7 @@ public void before_CosmosItemTest() throws IOException { cosmosEncryptionAsyncClient.getCosmosEncryptionAsyncDatabase(cosmosAsyncDatabase.getId()); ClientEncryptionPolicy clientEncryptionPolicy = - new ClientEncryptionPolicy(getPaths(false), 1); + new ClientEncryptionPolicy(getPaths(false), 2); String containerId = UUID.randomUUID().toString(); CosmosContainerProperties containerProperties = new CosmosContainerProperties(containerId, "/mypk"); containerProperties.setClientEncryptionPolicy(clientEncryptionPolicy); From 96139391277fc0cca6cab30125ac46bc888faa34 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Wed, 16 Nov 2022 11:13:31 -0800 Subject: [PATCH 26/31] Fixing test cases --- .../implementation/EncryptionProcessor.java | 20 ++++++++-------- .../CosmosEncryptionClientCachesTest.java | 2 +- .../cosmos/encryption/TestSuiteBase.java | 23 +++++++++---------- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java index 96a5d42792d4d..dad38e1dfc76b 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java @@ -664,22 +664,24 @@ public static JsonNode toJsonNode(byte[] serializedBytes, TypeMarker typeMarker) } private String convertToBase64UriSafeString(byte[] bytesToProcess) { - StringBuilder base64String = new StringBuilder(new String(Base64.getUrlDecoder().decode(bytesToProcess), StandardCharsets.UTF_8)); +// StringBuilder base64String = new StringBuilder(new String(Base64.getDecoder().decode(bytesToProcess), StandardCharsets.UTF_8)); // Base 64 Encoding with URL and Filename Safe Alphabet https://datatracker.ietf.org/doc/html/rfc4648#section-5 // https://docs.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-item-limits, due to base64 conversion and encryption // the permissible size of the property will further reduce. - replaceString(base64String, "/", "_"); - replaceString(base64String, "\\+", "-"); - return base64String.toString(); +// replaceString(base64String, "/", "_"); +// replaceString(base64String, "\\+", "-"); +// return base64String.toString(); + return Base64.getUrlEncoder().encodeToString(bytesToProcess); } private byte[] convertFromBase64UriSafeString(String base64UriSafeString) { - StringBuilder base64String = new StringBuilder(base64UriSafeString); - - replaceString(base64String, "_", "/"); - replaceString(base64String, "-", "\\+"); - return Base64.getUrlEncoder().encode(base64String.toString().getBytes(StandardCharsets.UTF_8)); +// StringBuilder base64String = new StringBuilder(base64UriSafeString); +// +// replaceString(base64String, "_", "/"); +// replaceString(base64String, "-", "\\+"); +// return Base64.getEncoder().encode(base64String.toString().getBytes(StandardCharsets.UTF_8)); + return Base64.getUrlDecoder().decode(base64UriSafeString); } private StringBuilder replaceString(StringBuilder sb, diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionClientCachesTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionClientCachesTest.java index 7896378e4a83e..d54425485729f 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionClientCachesTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionClientCachesTest.java @@ -68,7 +68,7 @@ public void before_CosmosItemTest() { CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName(), metadata2).block(); //Create collection with clientEncryptionPolicy - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(false), 1); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(false), 2); CosmosContainerProperties containerProperties = new CosmosContainerProperties("TestCollForEncryptionCacheTest" , "/mypk"); containerProperties.setClientEncryptionPolicy(clientEncryptionPolicy); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java index 510b2439d14a2..61cc3f4cfbbdf 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java @@ -1213,7 +1213,7 @@ public KeyEncryptionKey buildKeyEncryptionKey(String keyId) { } } - protected static List getPaths(boolean shouldEncryptIdAndPartitionKey) { + protected static List getPaths(boolean shouldEncryptId) { ClientEncryptionIncludedPath includedPath1 = new ClientEncryptionIncludedPath(); includedPath1.setClientEncryptionKeyId("key1"); includedPath1.setPath("/sensitiveString"); @@ -1293,16 +1293,16 @@ protected static List getPaths(boolean shouldEncry includedPath13.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName()); ClientEncryptionIncludedPath includedPath14 = new ClientEncryptionIncludedPath(); - includedPath13.setClientEncryptionKeyId("key1"); - includedPath13.setPath("/id"); - includedPath13.setEncryptionType(CosmosEncryptionType.DETERMINISTIC.getName()); - includedPath13.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName()); + includedPath14.setClientEncryptionKeyId("key1"); + includedPath14.setPath("/id"); + includedPath14.setEncryptionType(CosmosEncryptionType.DETERMINISTIC.getName()); + includedPath14.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName()); - ClientEncryptionIncludedPath includedPath15 = new ClientEncryptionIncludedPath(); - includedPath13.setClientEncryptionKeyId("key1"); - includedPath13.setPath("/mypk"); - includedPath13.setEncryptionType(CosmosEncryptionType.DETERMINISTIC.getName()); - includedPath13.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName()); +// ClientEncryptionIncludedPath includedPath15 = new ClientEncryptionIncludedPath(); +// includedPath13.setClientEncryptionKeyId("key1"); +// includedPath13.setPath("/mypk"); +// includedPath13.setEncryptionType(CosmosEncryptionType.DETERMINISTIC.getName()); +// includedPath13.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName()); List paths = new ArrayList<>(); paths.add(includedPath1); @@ -1319,9 +1319,8 @@ protected static List getPaths(boolean shouldEncry paths.add(includedPath12); paths.add(includedPath13); - if (shouldEncryptIdAndPartitionKey) { + if (shouldEncryptId) { paths.add(includedPath14); - paths.add(includedPath15); } return paths; From a071e8de4b70574cb608a8dd0072b9b134d58bd4 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Wed, 16 Nov 2022 12:19:16 -0800 Subject: [PATCH 27/31] Fixing test cases --- .../implementation/EncryptionProcessor.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java index dad38e1dfc76b..4a0f1d6a8059e 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java @@ -676,12 +676,12 @@ private String convertToBase64UriSafeString(byte[] bytesToProcess) { } private byte[] convertFromBase64UriSafeString(String base64UriSafeString) { -// StringBuilder base64String = new StringBuilder(base64UriSafeString); -// -// replaceString(base64String, "_", "/"); -// replaceString(base64String, "-", "\\+"); -// return Base64.getEncoder().encode(base64String.toString().getBytes(StandardCharsets.UTF_8)); - return Base64.getUrlDecoder().decode(base64UriSafeString); + StringBuilder base64String = new StringBuilder(base64UriSafeString); + + replaceString(base64String, "_", "/"); + replaceString(base64String, "-", "\\+"); + return Base64.getEncoder().encode(base64String.toString().getBytes(StandardCharsets.UTF_8)); +// return Base64.getUrlDecoder().decode(base64UriSafeString); } private StringBuilder replaceString(StringBuilder sb, From 921f1568f2f050d8fa7ca086d89ee68f0483045e Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Thu, 17 Nov 2022 10:15:54 -0800 Subject: [PATCH 28/31] Fixing test cases --- .../encryption/AsyncEncryptionBenchmark.java | 2 +- .../implementation/EncryptionProcessor.java | 12 +- .../encryption/EncryptionCodeSnippet.java | 2 +- .../com/azure/cosmos/encryption/Program.java | 2 +- .../cosmos/encryption/ReadmeSamples.java | 2 +- .../CosmosEncryptionClientCachesTest.java | 2 +- .../encryption/DotNetCompatibleTest.java | 2 +- .../EncryptionAsyncApiCrudTest.java | 221 +++++++++--------- ...ryptionCosmosEncryptionChangeFeedTest.java | 2 +- .../cosmos/encryption/TestSuiteBase.java | 15 +- 10 files changed, 128 insertions(+), 134 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/encryption/AsyncEncryptionBenchmark.java b/sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/encryption/AsyncEncryptionBenchmark.java index 22d91a67c6ea0..0a5f7446501df 100644 --- a/sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/encryption/AsyncEncryptionBenchmark.java +++ b/sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/encryption/AsyncEncryptionBenchmark.java @@ -549,7 +549,7 @@ private void createEncryptionDatabaseAndContainer() { includedPath.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName()); encryptionPaths.add(includedPath); } - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(encryptionPaths, 1); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(encryptionPaths, 2); CosmosContainerProperties containerProperties = new CosmosContainerProperties(this.configuration.getCollectionId(), Configuration.DEFAULT_PARTITION_KEY_PATH); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java index 4a0f1d6a8059e..1c7587862ef5d 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java @@ -676,12 +676,12 @@ private String convertToBase64UriSafeString(byte[] bytesToProcess) { } private byte[] convertFromBase64UriSafeString(String base64UriSafeString) { - StringBuilder base64String = new StringBuilder(base64UriSafeString); - - replaceString(base64String, "_", "/"); - replaceString(base64String, "-", "\\+"); - return Base64.getEncoder().encode(base64String.toString().getBytes(StandardCharsets.UTF_8)); -// return Base64.getUrlDecoder().decode(base64UriSafeString); +// StringBuilder base64String = new StringBuilder(base64UriSafeString); +// +// replaceString(base64String, "_", "/"); +// replaceString(base64String, "-", "\\+"); +// return Base64.getDecoder().decode(base64String.toString().getBytes(StandardCharsets.UTF_8)); + return Base64.getUrlDecoder().decode(base64UriSafeString); } private StringBuilder replaceString(StringBuilder sb, diff --git a/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/EncryptionCodeSnippet.java b/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/EncryptionCodeSnippet.java index a67234a644b88..e8f9e2898701a 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/EncryptionCodeSnippet.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/EncryptionCodeSnippet.java @@ -124,7 +124,7 @@ void createContainerWithClientEncryptionPolicy(CosmosAsyncClient client) { paths.add(includedPath6); paths.add(includedPath7); - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 1); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 2); String containerId = "myCol"; CosmosContainerProperties properties = new CosmosContainerProperties(containerId, "/mypk"); properties.setClientEncryptionPolicy(clientEncryptionPolicy); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/Program.java b/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/Program.java index 97666cb3bfcc1..9aad3fb512193 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/Program.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/Program.java @@ -155,7 +155,7 @@ private static void initialize(CosmosEncryptionAsyncClient cosmosEncryptionAsync CosmosContainerProperties containerProperties = new CosmosContainerProperties(Program.containerId, "/purchaseOrderNumber"); - containerProperties.setClientEncryptionPolicy(new ClientEncryptionPolicy(paths, 1)); + containerProperties.setClientEncryptionPolicy(new ClientEncryptionPolicy(paths, 2)); cosmosEncryptionAsyncDatabase.getCosmosAsyncDatabase().createContainer(containerProperties, ThroughputProperties.createManualThroughput(1000)).block(); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/ReadmeSamples.java b/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/ReadmeSamples.java index 33792e5df217b..c95291168c19c 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/ReadmeSamples.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/samples/java/com/azure/cosmos/encryption/ReadmeSamples.java @@ -86,7 +86,7 @@ public void createCosmosEncryptionContainer() { List paths = new ArrayList<>(); paths.add(includedPath); - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 1); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 2); CosmosContainerProperties properties = new CosmosContainerProperties("", "/mypk"); properties.setClientEncryptionPolicy(clientEncryptionPolicy); return cosmosEncryptionAsyncDatabase.getCosmosAsyncDatabase().createContainer(properties); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionClientCachesTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionClientCachesTest.java index d54425485729f..a2fb743d14fe1 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionClientCachesTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionClientCachesTest.java @@ -68,7 +68,7 @@ public void before_CosmosItemTest() { CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName(), metadata2).block(); //Create collection with clientEncryptionPolicy - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(false), 2); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(), 2); CosmosContainerProperties containerProperties = new CosmosContainerProperties("TestCollForEncryptionCacheTest" , "/mypk"); containerProperties.setClientEncryptionPolicy(clientEncryptionPolicy); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java index ec8272b48c31c..136a71caf2f50 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java @@ -70,7 +70,7 @@ public void before_CosmosItemTest() throws IOException { cosmosEncryptionAsyncClient.getCosmosEncryptionAsyncDatabase(cosmosAsyncDatabase.getId()); ClientEncryptionPolicy clientEncryptionPolicy = - new ClientEncryptionPolicy(getPaths(false), 2); + new ClientEncryptionPolicy(getPaths(), 2); String containerId = UUID.randomUUID().toString(); CosmosContainerProperties containerProperties = new CosmosContainerProperties(containerId, "/mypk"); containerProperties.setClientEncryptionPolicy(clientEncryptionPolicy); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java index c4c21d269f56b..3056235786f99 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java @@ -77,16 +77,16 @@ public void before_CosmosItemTest() { cosmosEncryptionAsyncDatabase = getSharedEncryptionDatabase(cosmosEncryptionAsyncClient); cosmosEncryptionAsyncContainer = getSharedEncryptionContainer(cosmosEncryptionAsyncClient); - ClientEncryptionPolicy clientEncryptionWithPolicyFormatVersion2 = new ClientEncryptionPolicy(getPaths(false), 2); + ClientEncryptionPolicy clientEncryptionWithPolicyFormatVersion2 = new ClientEncryptionPolicy(getPaths(), 2); String containerId = UUID.randomUUID().toString(); CosmosContainerProperties properties = new CosmosContainerProperties(containerId, "/mypk"); properties.setClientEncryptionPolicy(clientEncryptionWithPolicyFormatVersion2); cosmosEncryptionAsyncDatabase.getCosmosAsyncDatabase().createContainer(properties).block(); // ClientEncryptionPolicy with Id and PartitionKey included, and policyFormatVersion 2 - ClientEncryptionPolicy clientEncryptionPolicyWithIdAndPartitionKeyPath = new ClientEncryptionPolicy(getPaths(true), 2); - CosmosContainerProperties propertiesWithIdAndPartitionKeyPath = new CosmosContainerProperties(containerId, "/mypk"); - propertiesWithIdAndPartitionKeyPath.setClientEncryptionPolicy(clientEncryptionPolicyWithIdAndPartitionKeyPath); +// ClientEncryptionPolicy clientEncryptionPolicyWithIdAndPartitionKeyPath = new ClientEncryptionPolicy(getPaths(), 2); +// CosmosContainerProperties propertiesWithIdAndPartitionKeyPath = new CosmosContainerProperties(containerId, "/mypk"); +// propertiesWithIdAndPartitionKeyPath.setClientEncryptionPolicy(clientEncryptionPolicyWithIdAndPartitionKeyPath); encryptionContainerWithIncompatiblePolicyVersion = cosmosEncryptionAsyncDatabase.getCosmosEncryptionAsyncContainer(containerId); @@ -100,7 +100,10 @@ public void afterClass() { @Test(groups = {"encryption"}, timeOut = TIMEOUT) public void createItemEncrypt_readItemDecrypt() { - EncryptionPojo properties = getItem(UUID.randomUUID().toString()); +// EncryptionPojo properties = getItem(UUID.randomUUID().toString()); + + EncryptionPojo properties = getItem("7d925bf9-d583-4c6d-a49f-137bee7cff91"); + CosmosItemResponse itemResponse = cosmosEncryptionAsyncContainer.createItem(properties, new PartitionKey(properties.getMypk()), new CosmosItemRequestOptions()).block(); assertThat(itemResponse.getRequestCharge()).isGreaterThan(0); @@ -371,7 +374,7 @@ public void crudQueryStaleCache() { cosmosEncryptionAsyncClient.getCosmosEncryptionAsyncDatabase(asyncClient.getDatabase(databaseId)); String containerId = UUID.randomUUID().toString(); - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(false), 1); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(), 2); createEncryptionContainer(cosmosEncryptionAsyncDatabase, clientEncryptionPolicy, containerId); CosmosEncryptionAsyncContainer encryptionAsyncContainerOriginal = cosmosEncryptionAsyncDatabase.getCosmosEncryptionAsyncContainer(containerId); @@ -404,7 +407,7 @@ public void crudQueryStaleCache() { //Deleting and creating container encryptionAsyncContainerOriginal.getCosmosAsyncContainer().delete().block(); - ClientEncryptionPolicy policyWithOneEncryptionPolicy = new ClientEncryptionPolicy(getPathWithOneEncryptionField(), 1); + ClientEncryptionPolicy policyWithOneEncryptionPolicy = new ClientEncryptionPolicy(getPathWithOneEncryptionField(), 2); createEncryptionContainer(cosmosEncryptionAsyncDatabase, policyWithOneEncryptionPolicy, containerId); CosmosEncryptionAsyncContainer encryptionAsyncContainerNew = getNewEncryptionContainerProxyObject(cosmosEncryptionAsyncDatabase.getCosmosAsyncDatabase().getId(), containerId); encryptionAsyncContainerNew.createItem(encryptionPojo, @@ -1017,108 +1020,108 @@ public void crudOnDifferentOverload() { assertThat(deleteResponse3.getStatusCode()).isEqualTo(200); } - @Test(groups = {"encryption"}, timeOut = TIMEOUT) - public void crudOperationsWithIdAndPartitionKeyEncrypted() { - cosmosEncryptionAsyncDatabase.getCosmosAsyncDatabase().createContainer(propertiesWithIdAndPartitionKeyPath).block(); - List actualProperties = new ArrayList<>(); - // Read item - EncryptionPojo properties = getItem(UUID.randomUUID().toString()); - CosmosItemResponse itemResponse = cosmosEncryptionAsyncContainer.createItem(properties).block(); - assertThat(itemResponse.getRequestCharge()).isGreaterThan(0); - EncryptionPojo responseItem = itemResponse.getItem(); - validateResponse(properties, responseItem); - actualProperties.add(properties); - - properties = getItem(UUID.randomUUID().toString()); - CosmosItemResponse itemResponse1 = cosmosEncryptionAsyncContainer.createItem(properties, new CosmosItemRequestOptions()).block(); - assertThat(itemResponse1.getRequestCharge()).isGreaterThan(0); - EncryptionPojo responseItem1 = itemResponse1.getItem(); - validateResponse(properties, responseItem1); - actualProperties.add(properties); - - //Upsert Item - properties = getItem(UUID.randomUUID().toString()); - CosmosItemResponse upsertResponse1 = cosmosEncryptionAsyncContainer.upsertItem(properties).block(); - assertThat(upsertResponse1.getRequestCharge()).isGreaterThan(0); - EncryptionPojo responseItem2 = upsertResponse1.getItem(); - validateResponse(properties, responseItem2); - actualProperties.add(properties); - - properties = getItem(UUID.randomUUID().toString()); - CosmosItemResponse upsertResponse2 = cosmosEncryptionAsyncContainer.upsertItem(properties, new CosmosItemRequestOptions()).block(); - assertThat(upsertResponse2.getRequestCharge()).isGreaterThan(0); - EncryptionPojo responseItem3 = upsertResponse2.getItem(); - validateResponse(properties, responseItem3); - actualProperties.add(properties); - - //Read Item - EncryptionPojo readItem = cosmosEncryptionAsyncContainer.readItem(actualProperties.get(0).getId(), - new PartitionKey(actualProperties.get(0).getMypk()), EncryptionPojo.class).block().getItem(); - validateResponse(actualProperties.get(0), readItem); - - //Query Item - String query = String.format("SELECT * from c where c.id = '%s'", actualProperties.get(1).getId()); - - CosmosPagedFlux feedResponseIterator = - cosmosEncryptionAsyncContainer.queryItems(query, EncryptionPojo.class); - List feedResponse = feedResponseIterator.byPage().blockFirst().getResults(); - assertThat(feedResponse.size()).isGreaterThanOrEqualTo(1); - for (EncryptionPojo pojo : feedResponse) { - if (pojo.getId().equals(actualProperties.get(1).getId())) { - validateResponse(pojo, responseItem1); - } - } - - CosmosQueryRequestOptions cosmosQueryRequestOptions1 = new CosmosQueryRequestOptions(); - - CosmosPagedFlux feedResponseIterator1 = - cosmosEncryptionAsyncContainer.queryItems(query, cosmosQueryRequestOptions1, EncryptionPojo.class); - List feedResponse1 = feedResponseIterator1.byPage().blockFirst().getResults(); - assertThat(feedResponse1.size()).isGreaterThanOrEqualTo(1); - for (EncryptionPojo pojo : feedResponse1) { - if (pojo.getId().equals(actualProperties.get(1).getId())) { - validateResponse(pojo, responseItem1); - } - } - - CosmosQueryRequestOptions cosmosQueryRequestOptions2 = new CosmosQueryRequestOptions(); - SqlQuerySpec querySpec = new SqlQuerySpec(query); - - CosmosPagedFlux feedResponseIterator2 = - cosmosEncryptionAsyncContainer.queryItems(querySpec, cosmosQueryRequestOptions2, EncryptionPojo.class); - List feedResponse2 = feedResponseIterator2.byPage().blockFirst().getResults(); - assertThat(feedResponse2.size()).isGreaterThanOrEqualTo(1); - for (EncryptionPojo pojo : feedResponse2) { - if (pojo.getId().equals(actualProperties.get(1).getId())) { - validateResponse(pojo, responseItem1); - } - } - - //Replace Item - CosmosItemResponse replaceResponse = - cosmosEncryptionAsyncContainer.replaceItem(actualProperties.get(2), actualProperties.get(2).getId(), - new PartitionKey(actualProperties.get(2).getMypk())).block(); - assertThat(upsertResponse1.getRequestCharge()).isGreaterThan(0); - responseItem = replaceResponse.getItem(); - validateResponse(actualProperties.get(2), responseItem); - - //Delete Item - CosmosItemResponse deleteResponse = cosmosEncryptionAsyncContainer.deleteItem(actualProperties.get(0).getId(), - new PartitionKey(actualProperties.get(0).getMypk())).block(); - assertThat(deleteResponse.getStatusCode()).isEqualTo(204); - - CosmosItemResponse deleteResponse1 = cosmosEncryptionAsyncContainer.deleteItem(actualProperties.get(1).getId(), - new PartitionKey(actualProperties.get(1).getMypk()), new CosmosItemRequestOptions()).block(); - assertThat(deleteResponse1.getStatusCode()).isEqualTo(204); - - CosmosItemResponse deleteResponse2 = cosmosEncryptionAsyncContainer.deleteItem(actualProperties.get(2), - new CosmosItemRequestOptions()).block(); - assertThat(deleteResponse2.getStatusCode()).isEqualTo(204); - - CosmosItemResponse deleteResponse3 = cosmosEncryptionAsyncContainer.deleteAllItemsByPartitionKey(new PartitionKey(actualProperties.get(3).getMypk()), - new CosmosItemRequestOptions()).block(); - assertThat(deleteResponse3.getStatusCode()).isEqualTo(200); - } +// @Test(groups = {"encryption"}, timeOut = TIMEOUT) +// public void crudOperationsWithIdAndPartitionKeyEncrypted() { +// cosmosEncryptionAsyncDatabase.getCosmosAsyncDatabase().createContainer(propertiesWithIdAndPartitionKeyPath).block(); +// List actualProperties = new ArrayList<>(); +// // Read item +// EncryptionPojo properties = getItem(UUID.randomUUID().toString()); +// CosmosItemResponse itemResponse = cosmosEncryptionAsyncContainer.createItem(properties).block(); +// assertThat(itemResponse.getRequestCharge()).isGreaterThan(0); +// EncryptionPojo responseItem = itemResponse.getItem(); +// validateResponse(properties, responseItem); +// actualProperties.add(properties); +// +// properties = getItem(UUID.randomUUID().toString()); +// CosmosItemResponse itemResponse1 = cosmosEncryptionAsyncContainer.createItem(properties, new CosmosItemRequestOptions()).block(); +// assertThat(itemResponse1.getRequestCharge()).isGreaterThan(0); +// EncryptionPojo responseItem1 = itemResponse1.getItem(); +// validateResponse(properties, responseItem1); +// actualProperties.add(properties); +// +// //Upsert Item +// properties = getItem(UUID.randomUUID().toString()); +// CosmosItemResponse upsertResponse1 = cosmosEncryptionAsyncContainer.upsertItem(properties).block(); +// assertThat(upsertResponse1.getRequestCharge()).isGreaterThan(0); +// EncryptionPojo responseItem2 = upsertResponse1.getItem(); +// validateResponse(properties, responseItem2); +// actualProperties.add(properties); +// +// properties = getItem(UUID.randomUUID().toString()); +// CosmosItemResponse upsertResponse2 = cosmosEncryptionAsyncContainer.upsertItem(properties, new CosmosItemRequestOptions()).block(); +// assertThat(upsertResponse2.getRequestCharge()).isGreaterThan(0); +// EncryptionPojo responseItem3 = upsertResponse2.getItem(); +// validateResponse(properties, responseItem3); +// actualProperties.add(properties); +// +// //Read Item +// EncryptionPojo readItem = cosmosEncryptionAsyncContainer.readItem(actualProperties.get(0).getId(), +// new PartitionKey(actualProperties.get(0).getMypk()), EncryptionPojo.class).block().getItem(); +// validateResponse(actualProperties.get(0), readItem); +// +// //Query Item +// String query = String.format("SELECT * from c where c.id = '%s'", actualProperties.get(1).getId()); +// +// CosmosPagedFlux feedResponseIterator = +// cosmosEncryptionAsyncContainer.queryItems(query, EncryptionPojo.class); +// List feedResponse = feedResponseIterator.byPage().blockFirst().getResults(); +// assertThat(feedResponse.size()).isGreaterThanOrEqualTo(1); +// for (EncryptionPojo pojo : feedResponse) { +// if (pojo.getId().equals(actualProperties.get(1).getId())) { +// validateResponse(pojo, responseItem1); +// } +// } +// +// CosmosQueryRequestOptions cosmosQueryRequestOptions1 = new CosmosQueryRequestOptions(); +// +// CosmosPagedFlux feedResponseIterator1 = +// cosmosEncryptionAsyncContainer.queryItems(query, cosmosQueryRequestOptions1, EncryptionPojo.class); +// List feedResponse1 = feedResponseIterator1.byPage().blockFirst().getResults(); +// assertThat(feedResponse1.size()).isGreaterThanOrEqualTo(1); +// for (EncryptionPojo pojo : feedResponse1) { +// if (pojo.getId().equals(actualProperties.get(1).getId())) { +// validateResponse(pojo, responseItem1); +// } +// } +// +// CosmosQueryRequestOptions cosmosQueryRequestOptions2 = new CosmosQueryRequestOptions(); +// SqlQuerySpec querySpec = new SqlQuerySpec(query); +// +// CosmosPagedFlux feedResponseIterator2 = +// cosmosEncryptionAsyncContainer.queryItems(querySpec, cosmosQueryRequestOptions2, EncryptionPojo.class); +// List feedResponse2 = feedResponseIterator2.byPage().blockFirst().getResults(); +// assertThat(feedResponse2.size()).isGreaterThanOrEqualTo(1); +// for (EncryptionPojo pojo : feedResponse2) { +// if (pojo.getId().equals(actualProperties.get(1).getId())) { +// validateResponse(pojo, responseItem1); +// } +// } +// +// //Replace Item +// CosmosItemResponse replaceResponse = +// cosmosEncryptionAsyncContainer.replaceItem(actualProperties.get(2), actualProperties.get(2).getId(), +// new PartitionKey(actualProperties.get(2).getMypk())).block(); +// assertThat(upsertResponse1.getRequestCharge()).isGreaterThan(0); +// responseItem = replaceResponse.getItem(); +// validateResponse(actualProperties.get(2), responseItem); +// +// //Delete Item +// CosmosItemResponse deleteResponse = cosmosEncryptionAsyncContainer.deleteItem(actualProperties.get(0).getId(), +// new PartitionKey(actualProperties.get(0).getMypk())).block(); +// assertThat(deleteResponse.getStatusCode()).isEqualTo(204); +// +// CosmosItemResponse deleteResponse1 = cosmosEncryptionAsyncContainer.deleteItem(actualProperties.get(1).getId(), +// new PartitionKey(actualProperties.get(1).getMypk()), new CosmosItemRequestOptions()).block(); +// assertThat(deleteResponse1.getStatusCode()).isEqualTo(204); +// +// CosmosItemResponse deleteResponse2 = cosmosEncryptionAsyncContainer.deleteItem(actualProperties.get(2), +// new CosmosItemRequestOptions()).block(); +// assertThat(deleteResponse2.getStatusCode()).isEqualTo(204); +// +// CosmosItemResponse deleteResponse3 = cosmosEncryptionAsyncContainer.deleteAllItemsByPartitionKey(new PartitionKey(actualProperties.get(3).getMypk()), +// new CosmosItemRequestOptions()).block(); +// assertThat(deleteResponse3.getStatusCode()).isEqualTo(200); +// } static void validateResponseWithOneFieldEncryption(EncryptionPojo originalItem, EncryptionPojo result) { assertThat(result.getId()).isEqualTo(originalItem.getId()); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionCosmosEncryptionChangeFeedTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionCosmosEncryptionChangeFeedTest.java index 401f85b659d4e..8b670a266048c 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionCosmosEncryptionChangeFeedTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionCosmosEncryptionChangeFeedTest.java @@ -264,7 +264,7 @@ private CosmosAsyncContainer createLeaseCollection(int provisionedThroughput) { } private CosmosEncryptionAsyncContainer createFeedCollection() { - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(false), 1); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(), 2); String containerId = UUID.randomUUID().toString(); CosmosContainerProperties properties = new CosmosContainerProperties(containerId, "/mypk"); properties.setClientEncryptionPolicy(clientEncryptionPolicy); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java index 61cc3f4cfbbdf..f21742cc4e535 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java @@ -240,7 +240,7 @@ public static void beforeSuite() { SHARED_ENCRYPTION_DATABASE.createClientEncryptionKey("key2", CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName(), metadata2).block(); - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(false),2); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(),2); String containerId = UUID.randomUUID().toString(); CosmosContainerProperties properties = new CosmosContainerProperties(containerId, "/mypk"); properties.setClientEncryptionPolicy(clientEncryptionPolicy); @@ -1213,7 +1213,7 @@ public KeyEncryptionKey buildKeyEncryptionKey(String keyId) { } } - protected static List getPaths(boolean shouldEncryptId) { + protected static List getPaths() { ClientEncryptionIncludedPath includedPath1 = new ClientEncryptionIncludedPath(); includedPath1.setClientEncryptionKeyId("key1"); includedPath1.setPath("/sensitiveString"); @@ -1298,12 +1298,6 @@ protected static List getPaths(boolean shouldEncry includedPath14.setEncryptionType(CosmosEncryptionType.DETERMINISTIC.getName()); includedPath14.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName()); -// ClientEncryptionIncludedPath includedPath15 = new ClientEncryptionIncludedPath(); -// includedPath13.setClientEncryptionKeyId("key1"); -// includedPath13.setPath("/mypk"); -// includedPath13.setEncryptionType(CosmosEncryptionType.DETERMINISTIC.getName()); -// includedPath13.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName()); - List paths = new ArrayList<>(); paths.add(includedPath1); paths.add(includedPath2); @@ -1318,10 +1312,7 @@ protected static List getPaths(boolean shouldEncry paths.add(includedPath11); paths.add(includedPath12); paths.add(includedPath13); - - if (shouldEncryptId) { - paths.add(includedPath14); - } + paths.add(includedPath14); return paths; } From 3522fc83a5d2d6a9861a6e704aa0fa97932eeda4 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Thu, 17 Nov 2022 11:33:02 -0800 Subject: [PATCH 29/31] Fixing test cases --- .../implementation/EncryptionProcessor.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java index 1c7587862ef5d..085870bd0bf9d 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java @@ -664,24 +664,24 @@ public static JsonNode toJsonNode(byte[] serializedBytes, TypeMarker typeMarker) } private String convertToBase64UriSafeString(byte[] bytesToProcess) { -// StringBuilder base64String = new StringBuilder(new String(Base64.getDecoder().decode(bytesToProcess), StandardCharsets.UTF_8)); + StringBuilder base64String = new StringBuilder(new String(Base64.getEncoder().encode(bytesToProcess), StandardCharsets.UTF_8)); // Base 64 Encoding with URL and Filename Safe Alphabet https://datatracker.ietf.org/doc/html/rfc4648#section-5 // https://docs.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-item-limits, due to base64 conversion and encryption // the permissible size of the property will further reduce. -// replaceString(base64String, "/", "_"); -// replaceString(base64String, "\\+", "-"); -// return base64String.toString(); - return Base64.getUrlEncoder().encodeToString(bytesToProcess); + replaceString(base64String, "/", "_"); + replaceString(base64String, "\\+", "-"); + return base64String.toString(); +// return Base64.getUrlEncoder().encodeToString(bytesToProcess); } private byte[] convertFromBase64UriSafeString(String base64UriSafeString) { -// StringBuilder base64String = new StringBuilder(base64UriSafeString); -// -// replaceString(base64String, "_", "/"); -// replaceString(base64String, "-", "\\+"); -// return Base64.getDecoder().decode(base64String.toString().getBytes(StandardCharsets.UTF_8)); - return Base64.getUrlDecoder().decode(base64UriSafeString); + StringBuilder base64String = new StringBuilder(base64UriSafeString); + + replaceString(base64String, "_", "/"); + replaceString(base64String, "-", "\\+"); + return Base64.getDecoder().decode(base64String.toString().getBytes(StandardCharsets.UTF_8)); +// return Base64.getUrlDecoder().decode(base64UriSafeString); } private StringBuilder replaceString(StringBuilder sb, From a23d43d0d502502137da3de7393f19d229c2f51b Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Wed, 15 Feb 2023 18:01:42 -0800 Subject: [PATCH 30/31] Fixing test cases --- .../CosmosEncryptionAsyncContainer.java | 50 ++++++++++++++++--- .../implementation/EncryptionProcessor.java | 26 ++++------ 2 files changed, 54 insertions(+), 22 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java index 2567ea08b1f8c..da0212f772a82 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java @@ -60,6 +60,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.stream.Collectors; @@ -212,11 +213,14 @@ public Mono> deleteItem(String itemId, private String checkAndGetEncryptedId(String itemId, EncryptionSettings encryptionSettings) throws MicrosoftDataEncryptionException { + AtomicReference encryptedId = new AtomicReference<>(); if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream(). anyMatch(includedPath -> includedPath.getPath().substring(1).equals(Constants.PROPERTY_NAME_ID))) { - return itemId; + + encryptedId.set(getEncryptedItem(encryptionSettings, Constants.PROPERTY_NAME_ID, itemId)); } - return this.encryptionProcessor.encryptAndSerializeValue(encryptionSettings, itemId, Constants.PROPERTY_NAME_ID); + + return encryptedId.get() != null ? encryptedId.get() : itemId; } private PartitionKey checkAndGetEncryptedPartitionKey(PartitionKey partitionKey, EncryptionSettings encryptionSettings) @@ -241,8 +245,7 @@ private PartitionKey checkAndGetEncryptedPartitionKey(PartitionKey partitionKey, partitionKeyBuilder.add(childPartitionKey); continue; } - partitionKeyBuilder.add(this.encryptionProcessor. - encryptAndSerializeValue(encryptionSettings, childPartitionKey, partitionKeyPath)); + partitionKeyBuilder.add(getEncryptedItem(encryptionSettings, partitionKeyPath, childPartitionKey)); } return partitionKeyBuilder.build(); } @@ -256,11 +259,26 @@ private PartitionKey checkAndGetEncryptedPartitionKey(PartitionKey partitionKey, anyMatch(includedPath -> includedPath.getPath().substring(1).equals(partitionKeyPath))) { return partitionKey; } - return new PartitionKey(this.encryptionProcessor. - encryptAndSerializeValue(encryptionSettings, partitionKey.toString(), partitionKeyPath)); + return new PartitionKey(getEncryptedItem(encryptionSettings, partitionKeyPath, partitionKey.toString())); } } + private String getEncryptedItem(EncryptionSettings encryptionSettings, String propertyName, String propertyValue) { + AtomicReference encryptedItem = new AtomicReference<>(); + Mono encryptionSettingsForProperty = encryptionSettings.getEncryptionSettingForPropertyAsync(propertyName, this.encryptionProcessor); + + encryptionSettingsForProperty.flatMap(settings -> { + try { + encryptedItem.set(this.encryptionProcessor. + encryptAndSerializeValue(settings, propertyValue, propertyName)); + } catch (MicrosoftDataEncryptionException ex) { + return Mono.error(ex); + } + return Mono.empty(); + }).block(); + return encryptedItem.get(); + } + /** * Deletes the item. @@ -459,6 +477,16 @@ public Mono> readItem(String id, requestOptions = new CosmosItemRequestOptions(); } + this.encryptionProcessor.initEncryptionSettingsIfNotInitializedAsync(); + EncryptionSettings encryptionSettings = this.encryptionProcessor.getEncryptionSettings(); + + try { + id = checkAndGetEncryptedId(id, encryptionSettings); + partitionKey = checkAndGetEncryptedPartitionKey(partitionKey, encryptionSettings); + } catch (JsonProcessingException | MicrosoftDataEncryptionException e) { + return Mono.error(e); + } + Mono> responseMessageMono = this.readItemHelper(id, partitionKey, requestOptions, false); return responseMessageMono.publishOn(encryptionScheduler).flatMap(cosmosItemResponse -> setByteArrayContent(cosmosItemResponse, @@ -633,6 +661,16 @@ public Mono> patchItem( options = new CosmosPatchItemRequestOptions(); } + this.encryptionProcessor.initEncryptionSettingsIfNotInitializedAsync(); + EncryptionSettings encryptionSettings = this.encryptionProcessor.getEncryptionSettings(); + + try { + itemId = checkAndGetEncryptedId(itemId, encryptionSettings); + partitionKey = checkAndGetEncryptedPartitionKey(partitionKey, encryptionSettings); + } catch (JsonProcessingException | MicrosoftDataEncryptionException e) { + return Mono.error(e); + } + return patchItemHelper(itemId, partitionKey, cosmosPatchOperations, options, itemType); } diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java index 085870bd0bf9d..a10f3849cd206 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java @@ -450,8 +450,8 @@ public void encryptAndSerializeProperty(EncryptionSettings encryptionSettings, J } public String encryptAndSerializeValue(EncryptionSettings encryptionSettings, String propertyValue, String propertyName) throws MicrosoftDataEncryptionException { - JsonNode propertyValueHolder = toJsonNode(propertyValue.getBytes(StandardCharsets.UTF_8), TypeMarker.STRING); - return convertToBase64UriSafeString(encryptAndSerializeValue(encryptionSettings, null, propertyValueHolder, propertyName)); + JsonNode propertyValueHolder = toJsonNode(propertyValue.getBytes(StandardCharsets.US_ASCII), TypeMarker.STRING); + return new String(encryptAndSerializeValue(encryptionSettings, null, propertyValueHolder, propertyName), StandardCharsets.US_ASCII); } public byte[] encryptAndSerializeValue(EncryptionSettings encryptionSettings, ObjectNode objectNode, @@ -473,7 +473,11 @@ public byte[] encryptAndSerializeValue(EncryptionSettings encryptionSettings, Ob if (propertyName.equals(Constants.PROPERTY_NAME_ID)) { // case: id does not support '/','\','?','#'. Convert Base64 string to Uri safe string - cipherTextWithTypeMarker = convertToBase64UriSafeString(cipherTextWithTypeMarker).getBytes(StandardCharsets.UTF_8); + String base64UriSafeString = convertToBase64UriSafeString(cipherTextWithTypeMarker); + if (objectNode != null && !objectNode.isNull()) { + objectNode.put(propertyName, base64UriSafeString); + } + return base64UriSafeString.getBytes(StandardCharsets.UTF_8); } if (objectNode != null && !objectNode.isNull()) { @@ -599,7 +603,7 @@ public JsonNode decryptAndSerializeValue(EncryptionSettings encryptionSettings, if (propertyValueHolder.getNodeType() == JsonNodeType.NULL) { return null; } - cipherTextWithTypeMarker = convertFromBase64UriSafeString(propertyValueHolder.toString()); + cipherTextWithTypeMarker = convertFromBase64UriSafeString(propertyValueHolder.asText()); } else { cipherTextWithTypeMarker = propertyValueHolder.binaryValue(); } @@ -664,24 +668,14 @@ public static JsonNode toJsonNode(byte[] serializedBytes, TypeMarker typeMarker) } private String convertToBase64UriSafeString(byte[] bytesToProcess) { - StringBuilder base64String = new StringBuilder(new String(Base64.getEncoder().encode(bytesToProcess), StandardCharsets.UTF_8)); - // Base 64 Encoding with URL and Filename Safe Alphabet https://datatracker.ietf.org/doc/html/rfc4648#section-5 // https://docs.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-item-limits, due to base64 conversion and encryption // the permissible size of the property will further reduce. - replaceString(base64String, "/", "_"); - replaceString(base64String, "\\+", "-"); - return base64String.toString(); -// return Base64.getUrlEncoder().encodeToString(bytesToProcess); + return Base64.getUrlEncoder().encodeToString(bytesToProcess); } private byte[] convertFromBase64UriSafeString(String base64UriSafeString) { - StringBuilder base64String = new StringBuilder(base64UriSafeString); - - replaceString(base64String, "_", "/"); - replaceString(base64String, "-", "\\+"); - return Base64.getDecoder().decode(base64String.toString().getBytes(StandardCharsets.UTF_8)); -// return Base64.getUrlDecoder().decode(base64UriSafeString); + return Base64.getUrlDecoder().decode(base64UriSafeString); } private StringBuilder replaceString(StringBuilder sb, From bcd030a74c64d18577db0c5032fe0d475c1523e0 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Tue, 21 Feb 2023 12:33:21 -0800 Subject: [PATCH 31/31] Fixing test cases --- .../CosmosEncryptionAsyncContainer.java | 226 +++++++++--------- .../encryption/DotNetCompatibleTest.java | 5 +- .../EncryptionAsyncApiCrudTest.java | 9 +- .../cosmos/encryption/TestSuiteBase.java | 7 + .../dotnetEncryption/EncryptedPOCO.json | 42 ++-- 5 files changed, 147 insertions(+), 142 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java index da0212f772a82..aeb9672090a78 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncContainer.java @@ -10,9 +10,11 @@ import com.azure.cosmos.encryption.implementation.Constants; import com.azure.cosmos.encryption.implementation.CosmosResponseFactory; import com.azure.cosmos.encryption.implementation.EncryptionImplementationBridgeHelpers; +import com.azure.cosmos.encryption.implementation.EncryptionProcessor; import com.azure.cosmos.encryption.implementation.EncryptionSettings; import com.azure.cosmos.encryption.implementation.EncryptionUtils; import com.azure.cosmos.encryption.implementation.mdesrc.cryptography.MicrosoftDataEncryptionException; +import com.azure.cosmos.encryption.models.SqlQuerySpecWithEncryption; import com.azure.cosmos.implementation.CosmosPagedFluxOptions; import com.azure.cosmos.implementation.HttpConstants; import com.azure.cosmos.implementation.ImplementationBridgeHelpers; @@ -46,8 +48,6 @@ import com.azure.cosmos.models.SqlQuerySpec; import com.azure.cosmos.util.CosmosPagedFlux; import com.azure.cosmos.util.UtilBridgeInternal; -import com.azure.cosmos.encryption.implementation.EncryptionProcessor; -import com.azure.cosmos.encryption.models.SqlQuerySpecWithEncryption; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -60,7 +60,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; +import java.util.Objects; +import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; @@ -200,83 +201,99 @@ public Mono> deleteItem(String itemId, CosmosItemRequestOptions requestOptions) { this.encryptionProcessor.initEncryptionSettingsIfNotInitializedAsync(); - EncryptionSettings encryptionSettings = this.encryptionProcessor.getEncryptionSettings(); - - try { - itemId = checkAndGetEncryptedId(itemId, encryptionSettings); - partitionKey = checkAndGetEncryptedPartitionKey(partitionKey, encryptionSettings); - } catch (JsonProcessingException | MicrosoftDataEncryptionException e) { - return Mono.error(e); - } - return container.deleteItem(itemId, partitionKey, requestOptions); + return Mono.just(this.encryptionProcessor.getEncryptionSettings()) + .flatMap(settings -> { + try { + return Mono.zip( + checkAndGetEncryptedId(itemId, settings), + checkAndGetEncryptedPartitionKey(partitionKey, settings) + ).flatMap(encryptedIdPartitionTuple -> container.deleteItem(encryptedIdPartitionTuple.getT1(), encryptedIdPartitionTuple.getT2(), requestOptions)); + } catch (Exception ex) { + return Mono.error(ex); + } + }); } - private String checkAndGetEncryptedId(String itemId, EncryptionSettings encryptionSettings) - throws MicrosoftDataEncryptionException { - AtomicReference encryptedId = new AtomicReference<>(); + private Mono checkAndGetEncryptedId(String itemId, EncryptionSettings encryptionSettings) + { if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream(). anyMatch(includedPath -> includedPath.getPath().substring(1).equals(Constants.PROPERTY_NAME_ID))) { - - encryptedId.set(getEncryptedItem(encryptionSettings, Constants.PROPERTY_NAME_ID, itemId)); + return this.getEncryptedItem(encryptionSettings, Constants.PROPERTY_NAME_ID, itemId); } - - return encryptedId.get() != null ? encryptedId.get() : itemId; + return Mono.just(itemId); } - private PartitionKey checkAndGetEncryptedPartitionKey(PartitionKey partitionKey, EncryptionSettings encryptionSettings) - throws JsonProcessingException, MicrosoftDataEncryptionException { + private Mono checkAndGetEncryptedPartitionKey(PartitionKey partitionKey, EncryptionSettings encryptionSettings) { if (encryptionSettings.getPartitionKeyPaths().isEmpty() || partitionKey == null) { - return partitionKey; + return Mono.just(partitionKey); + } + + JsonNode partitionKeyNode; + try { + partitionKeyNode = EncryptionUtils.getSimpleObjectMapper().readTree(partitionKey.toString()); + } catch (JsonProcessingException ex) { + return Mono.error(ex); } - JsonNode partitionKeyNode = EncryptionUtils.getSimpleObjectMapper().readTree(partitionKey.toString()); + if (partitionKeyNode.isArray()) { ArrayNode arrayNode = (ArrayNode) partitionKeyNode; - PartitionKeyBuilder partitionKeyBuilder = new PartitionKeyBuilder(); - - for (String path : encryptionSettings.getPartitionKeyPaths()) { - // case: partition key path is /a/b/c and the client encryption policy has /a in path. - // hence encrypt the partition key value with using its top level path /a since - // /c would have been encrypted in the document using /a's policy. - String partitionKeyPath = path.split("/")[0]; - - String childPartitionKey = arrayNode.elements().next().toString(); - if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream(). - anyMatch(includedPath -> includedPath.getPath().substring(1).equals(partitionKeyPath))) { - partitionKeyBuilder.add(childPartitionKey); - continue; - } - partitionKeyBuilder.add(getEncryptedItem(encryptionSettings, partitionKeyPath, childPartitionKey)); - } - return partitionKeyBuilder.build(); - } - else { - if (encryptionSettings.getPartitionKeyPaths().size() > 1) { - throw new MicrosoftDataEncryptionException("There should only be 1 PartitionKeyPath."); - } - String partitionKeyPath = encryptionSettings.getPartitionKeyPaths().get(0); - if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream(). - anyMatch(includedPath -> includedPath.getPath().substring(1).equals(partitionKeyPath))) { - return partitionKey; - } - return new PartitionKey(getEncryptedItem(encryptionSettings, partitionKeyPath, partitionKey.toString())); + return Mono.just(new PartitionKeyBuilder()) + .flatMap(partitionKeyBuilder -> { + return Flux.fromIterable(encryptionSettings.getPartitionKeyPaths()) + .flatMap(path -> { + // case: partition key path is /a/b/c and the client encryption policy has /a in path. + // hence encrypt the partition key value with using its top level path /a since + // /c would have been encrypted in the document using /a's policy. + String partitionKeyPath = path.split("/")[0]; + + String childPartitionKey = arrayNode.elements().next().toString(); + if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream(). + anyMatch(includedPath -> includedPath.getPath().substring(1).equals(partitionKeyPath))) { + partitionKeyBuilder.add(childPartitionKey); + return Mono.empty(); + } + return getEncryptedItem(encryptionSettings, partitionKeyPath, childPartitionKey); + }) + .collectList() + .flatMap(encryptedItems -> { + for (String encryptedItem : encryptedItems) { + partitionKeyBuilder.add(encryptedItem); + } + return Mono.just(partitionKeyBuilder.build()); + }); + }); + } else { + return Mono.just((encryptionSettings.getPartitionKeyPaths().size() > 1)) + .flatMap(multiplePartitionPath -> { + if (multiplePartitionPath) { + return Mono.error(new MicrosoftDataEncryptionException("There should only be 1 PartitionKeyPath.")); + } + + return Mono.just(encryptionSettings.getPartitionKeyPaths().get(0)) + .flatMap(partitionKeyPath -> { + if (this.encryptionProcessor.getClientEncryptionPolicy().getIncludedPaths().stream(). + anyMatch(includedPath -> includedPath.getPath().substring(1).equals(partitionKeyPath))) { + return Mono.just(partitionKey); + } + return getEncryptedItem(encryptionSettings, partitionKeyPath, partitionKey.toString()); + }) + .map(encryptedPartitionKeyPath -> new PartitionKey(encryptedPartitionKeyPath)); + }); } } - private String getEncryptedItem(EncryptionSettings encryptionSettings, String propertyName, String propertyValue) { - AtomicReference encryptedItem = new AtomicReference<>(); - Mono encryptionSettingsForProperty = encryptionSettings.getEncryptionSettingForPropertyAsync(propertyName, this.encryptionProcessor); - - encryptionSettingsForProperty.flatMap(settings -> { - try { - encryptedItem.set(this.encryptionProcessor. - encryptAndSerializeValue(settings, propertyValue, propertyName)); - } catch (MicrosoftDataEncryptionException ex) { - return Mono.error(ex); - } - return Mono.empty(); - }).block(); - return encryptedItem.get(); + private Mono getEncryptedItem(EncryptionSettings encryptionSettings, String propertyName, String propertyValue) { + return encryptionSettings + .getEncryptionSettingForPropertyAsync(propertyName, this.encryptionProcessor) + .flatMap(settings -> { + try { + return Mono.just( + this.encryptionProcessor.encryptAndSerializeValue(settings, propertyValue, propertyName)); + } catch (MicrosoftDataEncryptionException ex) { + return Mono.error(ex); + } + }); } @@ -309,19 +326,14 @@ public Mono> deleteItem(T item, CosmosItemRequest */ // TODO Make this api public once it is GA in cosmos core library Mono> deleteAllItemsByPartitionKey(PartitionKey partitionKey, CosmosItemRequestOptions requestOptions) { - if (requestOptions == null) { - requestOptions = new CosmosItemRequestOptions(); - } - - this.encryptionProcessor.initEncryptionSettingsIfNotInitializedAsync(); - EncryptionSettings encryptionSettings = this.encryptionProcessor.getEncryptionSettings(); - - try { - partitionKey = checkAndGetEncryptedPartitionKey(partitionKey, encryptionSettings); - } catch (JsonProcessingException | MicrosoftDataEncryptionException e) { - return Mono.error(e); - } - return container.deleteAllItemsByPartitionKey(partitionKey, requestOptions); + Objects.requireNonNull(partitionKey, "partitionKey cannot be null"); + final CosmosItemRequestOptions options = Optional.ofNullable(requestOptions) + .orElse(new CosmosItemRequestOptions()); + + return this.encryptionProcessor.initEncryptionSettingsIfNotInitializedAsync() + .thenReturn(this.encryptionProcessor.getEncryptionSettings()) + .flatMap(encryptedSettings -> checkAndGetEncryptedPartitionKey(partitionKey, encryptedSettings)) + .flatMap(encryptedPartitionKey -> container.deleteAllItemsByPartitionKey(partitionKey, options)); } /** @@ -473,25 +485,23 @@ public Mono> readItem(String id, PartitionKey partitionKey, CosmosItemRequestOptions requestOptions, Class classType) { - if (requestOptions == null) { - requestOptions = new CosmosItemRequestOptions(); - } - - this.encryptionProcessor.initEncryptionSettingsIfNotInitializedAsync(); - EncryptionSettings encryptionSettings = this.encryptionProcessor.getEncryptionSettings(); - - try { - id = checkAndGetEncryptedId(id, encryptionSettings); - partitionKey = checkAndGetEncryptedPartitionKey(partitionKey, encryptionSettings); - } catch (JsonProcessingException | MicrosoftDataEncryptionException e) { - return Mono.error(e); - } - - Mono> responseMessageMono = this.readItemHelper(id, partitionKey, requestOptions, false); - - return responseMessageMono.publishOn(encryptionScheduler).flatMap(cosmosItemResponse -> setByteArrayContent(cosmosItemResponse, - this.encryptionProcessor.decrypt(cosmosItemResponseBuilderAccessor.getByteArrayContent(cosmosItemResponse))) - .map(bytes -> this.responseFactory.createItemResponse(cosmosItemResponse, classType))); + final CosmosItemRequestOptions options = Optional.ofNullable(requestOptions) + .orElse(new CosmosItemRequestOptions()); + + return this.encryptionProcessor.initEncryptionSettingsIfNotInitializedAsync() + .thenReturn(this.encryptionProcessor.getEncryptionSettings()) + .flatMap(encryptedSettings -> Mono.zip( + checkAndGetEncryptedId(id, encryptedSettings), + checkAndGetEncryptedPartitionKey(partitionKey, encryptedSettings)) + .flatMap(encryptedIdPartitionKeyTuple -> + this.readItemHelper(encryptedIdPartitionKeyTuple.getT1(), + encryptedIdPartitionKeyTuple.getT2(), options, false)) + .publishOn(encryptionScheduler) + .flatMap(cosmosItemResponse -> setByteArrayContent( + cosmosItemResponse, + this.encryptionProcessor.decrypt(cosmosItemResponseBuilderAccessor + .getByteArrayContent(cosmosItemResponse)) + ).map(bytes -> this.responseFactory.createItemResponse(cosmosItemResponse, classType)))); } /** @@ -657,21 +667,15 @@ public Mono> patchItem( checkNotNull(partitionKey, "expected non-null partitionKey for patchItem"); checkNotNull(cosmosPatchOperations, "expected non-null cosmosPatchOperations"); - if (options == null) { - options = new CosmosPatchItemRequestOptions(); - } - - this.encryptionProcessor.initEncryptionSettingsIfNotInitializedAsync(); - EncryptionSettings encryptionSettings = this.encryptionProcessor.getEncryptionSettings(); - - try { - itemId = checkAndGetEncryptedId(itemId, encryptionSettings); - partitionKey = checkAndGetEncryptedPartitionKey(partitionKey, encryptionSettings); - } catch (JsonProcessingException | MicrosoftDataEncryptionException e) { - return Mono.error(e); - } + final CosmosPatchItemRequestOptions patchOptions = Optional.ofNullable(options) + .orElse(new CosmosPatchItemRequestOptions()); - return patchItemHelper(itemId, partitionKey, cosmosPatchOperations, options, itemType); + return this.encryptionProcessor.initEncryptionSettingsIfNotInitializedAsync() + .thenReturn(this.encryptionProcessor.getEncryptionSettings()) + .flatMap(encryptionSettings -> Mono.zip( + checkAndGetEncryptedId(itemId, encryptionSettings), + checkAndGetEncryptedPartitionKey(partitionKey, encryptionSettings)) + .flatMap(encryptedIdPartitionKeyTuple -> patchItemHelper(encryptedIdPartitionKeyTuple.getT1(), encryptedIdPartitionKeyTuple.getT2(), cosmosPatchOperations, patchOptions, itemType))); } private Mono> patchItemHelper(String itemId, diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java index 136a71caf2f50..db181101ba0b8 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/DotNetCompatibleTest.java @@ -100,12 +100,13 @@ public void createItemEncrypt_readItemDecrypt() throws IOException { this.cosmosEncryptionAsyncContainer.getCosmosAsyncContainer().createItem(dotNetEncryptedPocoJsonNode, partitionKey, requestOptions).block(); + JsonNode dotNetPOCOJsonNode = MAPPER.readTree(new File("src/test/resources/dotnetEncryption/POCO.json")); + partitionKey = new PartitionKey(dotNetPOCOJsonNode.get("mypk").asText()); //reading above saved .net encrypted json via java encryption library EncryptionPojo unencryptedPojo = - this.cosmosEncryptionAsyncContainer.readItem(dotNetEncryptedPocoJsonNode.get("id").asText(), partitionKey + this.cosmosEncryptionAsyncContainer.readItem(dotNetPOCOJsonNode.get("id").asText(), partitionKey , new CosmosItemRequestOptions(), EncryptionPojo.class).block().getItem(); - JsonNode dotNetPOCOJsonNode = MAPPER.readTree(new File("src/test/resources/dotnetEncryption/POCO.json")); EncryptionPojo unencryptedPoco = MAPPER.treeToValue(dotNetPOCOJsonNode, EncryptionPojo.class); //validating java decrypted pojo similar to original .net unencrypted poco diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java index 3056235786f99..ba2702015ac0b 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java @@ -83,11 +83,6 @@ public void before_CosmosItemTest() { properties.setClientEncryptionPolicy(clientEncryptionWithPolicyFormatVersion2); cosmosEncryptionAsyncDatabase.getCosmosAsyncDatabase().createContainer(properties).block(); - // ClientEncryptionPolicy with Id and PartitionKey included, and policyFormatVersion 2 -// ClientEncryptionPolicy clientEncryptionPolicyWithIdAndPartitionKeyPath = new ClientEncryptionPolicy(getPaths(), 2); -// CosmosContainerProperties propertiesWithIdAndPartitionKeyPath = new CosmosContainerProperties(containerId, "/mypk"); -// propertiesWithIdAndPartitionKeyPath.setClientEncryptionPolicy(clientEncryptionPolicyWithIdAndPartitionKeyPath); - encryptionContainerWithIncompatiblePolicyVersion = cosmosEncryptionAsyncDatabase.getCosmosEncryptionAsyncContainer(containerId); } @@ -100,9 +95,7 @@ public void afterClass() { @Test(groups = {"encryption"}, timeOut = TIMEOUT) public void createItemEncrypt_readItemDecrypt() { -// EncryptionPojo properties = getItem(UUID.randomUUID().toString()); - - EncryptionPojo properties = getItem("7d925bf9-d583-4c6d-a49f-137bee7cff91"); + EncryptionPojo properties = getItem(UUID.randomUUID().toString()); CosmosItemResponse itemResponse = cosmosEncryptionAsyncContainer.createItem(properties, new PartitionKey(properties.getMypk()), new CosmosItemRequestOptions()).block(); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java index f21742cc4e535..16077a9bba142 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java @@ -1298,6 +1298,12 @@ protected static List getPaths() { includedPath14.setEncryptionType(CosmosEncryptionType.DETERMINISTIC.getName()); includedPath14.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName()); + ClientEncryptionIncludedPath includedPath15 = new ClientEncryptionIncludedPath(); + includedPath15.setClientEncryptionKeyId("key1"); + includedPath15.setPath("/pk"); + includedPath15.setEncryptionType(CosmosEncryptionType.DETERMINISTIC.getName()); + includedPath15.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAD_AES_256_CBC_HMAC_SHA256.getName()); + List paths = new ArrayList<>(); paths.add(includedPath1); paths.add(includedPath2); @@ -1313,6 +1319,7 @@ protected static List getPaths() { paths.add(includedPath12); paths.add(includedPath13); paths.add(includedPath14); + paths.add(includedPath15); return paths; } diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/dotnetEncryption/EncryptedPOCO.json b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/dotnetEncryption/EncryptedPOCO.json index 448e61486361c..7f1e2684fbcdc 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/dotnetEncryption/EncryptedPOCO.json +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/dotnetEncryption/EncryptedPOCO.json @@ -1,20 +1,20 @@ { - "id": "7d925bf9-d583-4c6d-a49f-137bee7cff91", + "id": "BQF1OMHlXj-zDxKasrXQFwusq5lZf-AjwlS5Z4ZFdtljRO9ljKsPhPdZ4bGsXYbgtO4WrsHdqVy7r_EB101kO7xfjmGMvTOBiRt_M80AcNwiwJ1vFX9-uLudp3ttD-wKyZs=", "mypk": "7d925bf9-d583-4c6d-a49f-137bee7cff91", "nonSensitive": "cf012e6a-3dd1-4f1a-bd93-dc2b7fee5e97", "sensitiveString": "BQE8sgzM2aDfOlOiU9tYsgR5CghB/WZ/jsluf4HoYotfeUBqDBn3hVfx3A+o4m2mm5dEvnJyOR2Sl+NZezsYN71d", "sensitiveInt": "BAF+h8xlYiIqaH9TmN8/mgzX8ulodNhbYPASSPEGIVb+3qTR8J4qo/jZoz/qQdkxkEv8TV/aj0UdPEpGF4EBe+88", - "sensitiveFloat": "AwHBSYf0AqKfOX60Ppn/yj9ycfuFlOWEHGQ230BypHacyeBX/o3jRmlgnRNJuZpr1I9n0pNQl1yCfJkscMUVs8ao", + "sensitiveFloat": "BAFjPNt6v/gmO/v1uZCeTPgT6Tioja+rOzr1I8OK3hjC6tU6eZlXk0TUBVtu6ZkbIMo+cMAxH7IVC5diCnJfkjlS", "sensitiveLong": "BAG8flsdV8r870XeoPwTErbRBPCU5wdoAPSICreKQigwYblfexcEAosleZ+FO261u/m8X97ObH5m8hkxEuQzPURL", - "sensitiveDouble": "AwHu2zdLESFrjQ7L9KbCenSeEXHOXKjV9d2UCfhNhR/ImYmODXymlKGxapHhzmUabdEp6hKP6ZNIjqJ5xvFD6EYs", + "sensitiveDouble": "AwHMrObWBev4hnUulDL5dvLxybF22O3VzY7//PXyano3Hrl8A967TAp3fdRATdaFHydATozMzgREtbhjJNk1WT1d", "sensitiveBoolean": "AgHOBKDUuHWgivJCu+I8JgIqI3CuSgNKYKpn6nNUN8nqtyPkwnOJXYIeKpXS55/UzzaMAfRd1CZbT2oHdkIzUfuN", "sensitiveNestedPojo": { - "id": "BQERuT+l/kKc51OtABiPG3/lu8hpIJNGwUgJVzr71dKJk9wcuuzZbMqtsxj+Ue4B0oY0OYbKCuOYF3wLctg7P0cO", + "id": "BQERuT-l_kKc51OtABiPG3_lu8hpIJNGwUgJVzr71dKJk9wcuuzZbMqtsxj-Ue4B0oY0OYbKCuOYF3wLctg7P0cO", "mypk": "BQERuT+l/kKc51OtABiPG3/lu8hpIJNGwUgJVzr71dKJk9wcuuzZbMqtsxj+Ue4B0oY0OYbKCuOYF3wLctg7P0cO", "nonSensitive": null, "sensitiveString": "BQERuT+l/kKc51OtABiPG3/lu8hpIJNGwUgJVzr71dKJk9wcuuzZbMqtsxj+Ue4B0oY0OYbKCuOYF3wLctg7P0cO", "sensitiveInt": "BAFZBTqQK+9Z7UC1W1NBh+LHLzAEcfFgCc53BD6EtcfGxLZXjKm/F9OFNB+gMeM3T4SFj6jG5DH6TtjVh1XRum4/", - "sensitiveFloat": "AwHRoUMnYT23ocPkUoERjaSVcWxN05sjBivH0Dt4S5rC92YyZoyZokxGa5wnfpKuK49wr9wPynxF5QurRIha2d+c", + "sensitiveFloat": "BAHRoUMnYT23ocPkUoERjaSVcWxN05sjBivH0Dt4S5rC92YyZoyZokxGa5wnfpKuK49wr9wPynxF5QurRIha2d+c", "sensitiveLong": "BAG8flsdV8r870XeoPwTErbRBPCU5wdoAPSICreKQigwYblfexcEAosleZ+FO261u/m8X97ObH5m8hkxEuQzPURL", "sensitiveDouble": "AwHAQXtks16DzqIkLNWYZ7TWcjNCLPIYdiTe3NG7hXqg+DJirv5y2hx/fPiGuSKQ30ocEKkiqnxwISegmxZQt1De", "sensitiveBoolean": "AgHOBKDUuHWgivJCu+I8JgIqI3CuSgNKYKpn6nNUN8nqtyPkwnOJXYIeKpXS55/UzzaMAfRd1CZbT2oHdkIzUfuN", @@ -82,12 +82,12 @@ "sensitiveChildPojo2DArray": [ [ { - "id": "BQHRkzWrw6w88Rak29a/mWMh57hvictUtZEK0YuYV+LNRuvCBPU74+btLrxseJ4VZ3zesySbaBGkICzT7zq44M1j", + "id": "BQHRkzWrw6w88Rak29a_mWMh57hvictUtZEK0YuYV-LNRuvCBPU74-btLrxseJ4VZ3zesySbaBGkICzT7zq44M1j", "mypk": null, "nonSensitive": null, "sensitiveString": "BQFfH6UGQgHuCZ1SKb8uDdEEdpjZ9+oFgKXDZGR5awXt5hY7B0D5i+6WQQNp2y2Vwsql32ZhCGEyg2vpew/heajPaIqKEPCCklz0WiVc/lrePg==", "sensitiveInt": "BAFZBTqQK+9Z7UC1W1NBh+LHLzAEcfFgCc53BD6EtcfGxLZXjKm/F9OFNB+gMeM3T4SFj6jG5DH6TtjVh1XRum4/", - "sensitiveFloat": "AwHRoUMnYT23ocPkUoERjaSVcWxN05sjBivH0Dt4S5rC92YyZoyZokxGa5wnfpKuK49wr9wPynxF5QurRIha2d+c", + "sensitiveFloat": "BAHRoUMnYT23ocPkUoERjaSVcWxN05sjBivH0Dt4S5rC92YyZoyZokxGa5wnfpKuK49wr9wPynxF5QurRIha2d+c", "sensitiveLong": "BAG8flsdV8r870XeoPwTErbRBPCU5wdoAPSICreKQigwYblfexcEAosleZ+FO261u/m8X97ObH5m8hkxEuQzPURL", "sensitiveDouble": "AwHAQXtks16DzqIkLNWYZ7TWcjNCLPIYdiTe3NG7hXqg+DJirv5y2hx/fPiGuSKQ30ocEKkiqnxwISegmxZQt1De", "sensitiveBoolean": "AgHOBKDUuHWgivJCu+I8JgIqI3CuSgNKYKpn6nNUN8nqtyPkwnOJXYIeKpXS55/UzzaMAfRd1CZbT2oHdkIzUfuN", @@ -123,12 +123,12 @@ "sensitiveChildPojoList": null }, { - "id": "BQEUkOOtKpvfp4tbzdsgHToSUv8fUstDi025Q6n6oOvi5mFelP1C/wEauxKAYYswqUJ6DWd5eiIsFB/bjGViIq5A", + "id": "BQEUkOOtKpvfp4tbzdsgHToSUv8fUstDi025Q6n6oOvi5mFelP1C_wEauxKAYYswqUJ6DWd5eiIsFB_bjGViIq5A", "mypk": null, "nonSensitive": null, "sensitiveString": "BQG3BuavLLquGcjrQw50+NLMm/86/+qVs+75b0jVTcfgjdkwXpTyFY7bRb/SSvxb7DYG7/za32PJ9Q5L4t5d3jvluLs2JWe6wb0fzjw5HSVuEA==", "sensitiveInt": "BAFZBTqQK+9Z7UC1W1NBh+LHLzAEcfFgCc53BD6EtcfGxLZXjKm/F9OFNB+gMeM3T4SFj6jG5DH6TtjVh1XRum4/", - "sensitiveFloat": "AwHRoUMnYT23ocPkUoERjaSVcWxN05sjBivH0Dt4S5rC92YyZoyZokxGa5wnfpKuK49wr9wPynxF5QurRIha2d+c", + "sensitiveFloat": "BAHRoUMnYT23ocPkUoERjaSVcWxN05sjBivH0Dt4S5rC92YyZoyZokxGa5wnfpKuK49wr9wPynxF5QurRIha2d+c", "sensitiveLong": "BAG8flsdV8r870XeoPwTErbRBPCU5wdoAPSICreKQigwYblfexcEAosleZ+FO261u/m8X97ObH5m8hkxEuQzPURL", "sensitiveDouble": "AwHAQXtks16DzqIkLNWYZ7TWcjNCLPIYdiTe3NG7hXqg+DJirv5y2hx/fPiGuSKQ30ocEKkiqnxwISegmxZQt1De", "sensitiveBoolean": "AgHOBKDUuHWgivJCu+I8JgIqI3CuSgNKYKpn6nNUN8nqtyPkwnOJXYIeKpXS55/UzzaMAfRd1CZbT2oHdkIzUfuN", @@ -142,12 +142,12 @@ ], [ { - "id": "BQHRkzWrw6w88Rak29a/mWMh57hvictUtZEK0YuYV+LNRuvCBPU74+btLrxseJ4VZ3zesySbaBGkICzT7zq44M1j", + "id": "BQHRkzWrw6w88Rak29a_mWMh57hvictUtZEK0YuYV-LNRuvCBPU74-btLrxseJ4VZ3zesySbaBGkICzT7zq44M1j", "mypk": null, "nonSensitive": null, "sensitiveString": "BQFfH6UGQgHuCZ1SKb8uDdEEdpjZ9+oFgKXDZGR5awXt5hY7B0D5i+6WQQNp2y2Vwsql32ZhCGEyg2vpew/heajPaIqKEPCCklz0WiVc/lrePg==", "sensitiveInt": "BAFZBTqQK+9Z7UC1W1NBh+LHLzAEcfFgCc53BD6EtcfGxLZXjKm/F9OFNB+gMeM3T4SFj6jG5DH6TtjVh1XRum4/", - "sensitiveFloat": "AwHRoUMnYT23ocPkUoERjaSVcWxN05sjBivH0Dt4S5rC92YyZoyZokxGa5wnfpKuK49wr9wPynxF5QurRIha2d+c", + "sensitiveFloat": "BAHRoUMnYT23ocPkUoERjaSVcWxN05sjBivH0Dt4S5rC92YyZoyZokxGa5wnfpKuK49wr9wPynxF5QurRIha2d+c", "sensitiveLong": "BAG8flsdV8r870XeoPwTErbRBPCU5wdoAPSICreKQigwYblfexcEAosleZ+FO261u/m8X97ObH5m8hkxEuQzPURL", "sensitiveDouble": "AwHAQXtks16DzqIkLNWYZ7TWcjNCLPIYdiTe3NG7hXqg+DJirv5y2hx/fPiGuSKQ30ocEKkiqnxwISegmxZQt1De", "sensitiveBoolean": "AgHOBKDUuHWgivJCu+I8JgIqI3CuSgNKYKpn6nNUN8nqtyPkwnOJXYIeKpXS55/UzzaMAfRd1CZbT2oHdkIzUfuN", @@ -183,12 +183,12 @@ "sensitiveChildPojoList": null }, { - "id": "BQEUkOOtKpvfp4tbzdsgHToSUv8fUstDi025Q6n6oOvi5mFelP1C/wEauxKAYYswqUJ6DWd5eiIsFB/bjGViIq5A", + "id": "BQEUkOOtKpvfp4tbzdsgHToSUv8fUstDi025Q6n6oOvi5mFelP1C_wEauxKAYYswqUJ6DWd5eiIsFB_bjGViIq5A", "mypk": null, "nonSensitive": null, "sensitiveString": "BQG3BuavLLquGcjrQw50+NLMm/86/+qVs+75b0jVTcfgjdkwXpTyFY7bRb/SSvxb7DYG7/za32PJ9Q5L4t5d3jvluLs2JWe6wb0fzjw5HSVuEA==", "sensitiveInt": "BAFZBTqQK+9Z7UC1W1NBh+LHLzAEcfFgCc53BD6EtcfGxLZXjKm/F9OFNB+gMeM3T4SFj6jG5DH6TtjVh1XRum4/", - "sensitiveFloat": "AwHRoUMnYT23ocPkUoERjaSVcWxN05sjBivH0Dt4S5rC92YyZoyZokxGa5wnfpKuK49wr9wPynxF5QurRIha2d+c", + "sensitiveFloat": "BAHRoUMnYT23ocPkUoERjaSVcWxN05sjBivH0Dt4S5rC92YyZoyZokxGa5wnfpKuK49wr9wPynxF5QurRIha2d+c", "sensitiveLong": "BAG8flsdV8r870XeoPwTErbRBPCU5wdoAPSICreKQigwYblfexcEAosleZ+FO261u/m8X97ObH5m8hkxEuQzPURL", "sensitiveDouble": "AwHAQXtks16DzqIkLNWYZ7TWcjNCLPIYdiTe3NG7hXqg+DJirv5y2hx/fPiGuSKQ30ocEKkiqnxwISegmxZQt1De", "sensitiveBoolean": "AgHOBKDUuHWgivJCu+I8JgIqI3CuSgNKYKpn6nNUN8nqtyPkwnOJXYIeKpXS55/UzzaMAfRd1CZbT2oHdkIzUfuN", @@ -203,12 +203,12 @@ ], "sensitiveChildPojoList": [ { - "id": "BQHRkzWrw6w88Rak29a/mWMh57hvictUtZEK0YuYV+LNRuvCBPU74+btLrxseJ4VZ3zesySbaBGkICzT7zq44M1j", + "id": "BQHRkzWrw6w88Rak29a_mWMh57hvictUtZEK0YuYV-LNRuvCBPU74-btLrxseJ4VZ3zesySbaBGkICzT7zq44M1j", "mypk": null, "nonSensitive": null, "sensitiveString": "BQFfH6UGQgHuCZ1SKb8uDdEEdpjZ9+oFgKXDZGR5awXt5hY7B0D5i+6WQQNp2y2Vwsql32ZhCGEyg2vpew/heajPaIqKEPCCklz0WiVc/lrePg==", "sensitiveInt": "BAFZBTqQK+9Z7UC1W1NBh+LHLzAEcfFgCc53BD6EtcfGxLZXjKm/F9OFNB+gMeM3T4SFj6jG5DH6TtjVh1XRum4/", - "sensitiveFloat": "AwHRoUMnYT23ocPkUoERjaSVcWxN05sjBivH0Dt4S5rC92YyZoyZokxGa5wnfpKuK49wr9wPynxF5QurRIha2d+c", + "sensitiveFloat": "BAHRoUMnYT23ocPkUoERjaSVcWxN05sjBivH0Dt4S5rC92YyZoyZokxGa5wnfpKuK49wr9wPynxF5QurRIha2d+c", "sensitiveLong": "BAG8flsdV8r870XeoPwTErbRBPCU5wdoAPSICreKQigwYblfexcEAosleZ+FO261u/m8X97ObH5m8hkxEuQzPURL", "sensitiveDouble": "AwHAQXtks16DzqIkLNWYZ7TWcjNCLPIYdiTe3NG7hXqg+DJirv5y2hx/fPiGuSKQ30ocEKkiqnxwISegmxZQt1De", "sensitiveBoolean": "AgHOBKDUuHWgivJCu+I8JgIqI3CuSgNKYKpn6nNUN8nqtyPkwnOJXYIeKpXS55/UzzaMAfRd1CZbT2oHdkIzUfuN", @@ -244,12 +244,12 @@ "sensitiveChildPojoList": null }, { - "id": "BQEUkOOtKpvfp4tbzdsgHToSUv8fUstDi025Q6n6oOvi5mFelP1C/wEauxKAYYswqUJ6DWd5eiIsFB/bjGViIq5A", + "id": "BQEUkOOtKpvfp4tbzdsgHToSUv8fUstDi025Q6n6oOvi5mFelP1C_wEauxKAYYswqUJ6DWd5eiIsFB_bjGViIq5A", "mypk": null, "nonSensitive": null, "sensitiveString": "BQG3BuavLLquGcjrQw50+NLMm/86/+qVs+75b0jVTcfgjdkwXpTyFY7bRb/SSvxb7DYG7/za32PJ9Q5L4t5d3jvluLs2JWe6wb0fzjw5HSVuEA==", "sensitiveInt": "BAFZBTqQK+9Z7UC1W1NBh+LHLzAEcfFgCc53BD6EtcfGxLZXjKm/F9OFNB+gMeM3T4SFj6jG5DH6TtjVh1XRum4/", - "sensitiveFloat": "AwHRoUMnYT23ocPkUoERjaSVcWxN05sjBivH0Dt4S5rC92YyZoyZokxGa5wnfpKuK49wr9wPynxF5QurRIha2d+c", + "sensitiveFloat": "BAHRoUMnYT23ocPkUoERjaSVcWxN05sjBivH0Dt4S5rC92YyZoyZokxGa5wnfpKuK49wr9wPynxF5QurRIha2d+c", "sensitiveLong": "BAG8flsdV8r870XeoPwTErbRBPCU5wdoAPSICreKQigwYblfexcEAosleZ+FO261u/m8X97ObH5m8hkxEuQzPURL", "sensitiveDouble": "AwHAQXtks16DzqIkLNWYZ7TWcjNCLPIYdiTe3NG7hXqg+DJirv5y2hx/fPiGuSKQ30ocEKkiqnxwISegmxZQt1De", "sensitiveBoolean": "AgHOBKDUuHWgivJCu+I8JgIqI3CuSgNKYKpn6nNUN8nqtyPkwnOJXYIeKpXS55/UzzaMAfRd1CZbT2oHdkIzUfuN", @@ -261,9 +261,9 @@ "sensitiveChildPojoList": null } ], - "_rid": "-dspAMVPSPMBAAAAAAAAAA==", - "_self": "dbs/-dspAA==/colls/-dspAMVPSPM=/docs/-dspAMVPSPMBAAAAAAAAAA==/", - "_etag": "00000000-0000-0000-984f-74dd1e6e01d7", + "_rid": "frBRAOuISWIBAAAAAAAAAA==", + "_self": "dbs/frBRAA==/colls/frBRAOuISWI=/docs/frBRAOuISWIBAAAAAAAAAA==/", + "_etag": "\"01094fbb-0000-0300-0000-63f528970000\"", "_attachments": "attachments/", - "_ts": 1629744506 + "_ts": 1677011095 }