Skip to content

Commit

Permalink
3.18.1-preview: Adds release notes and ports hotfixes for query (#2532)
Browse files Browse the repository at this point in the history 
* Query: Fixes InvalidOperationException on merge to a single partition (#2510)

* Add support to FlakyDocumentContainer send back 410s based on a delegate

* Fix bug in CrossPartitionRangePageAsyncEnumerator where we dont handle a merge to a single partition

* Fix InvalidOperationException on merge to single partition in OrderByCrossPartitionQueryPipelineStage

* Incorporate CR feedback

* more CR feedback

Co-authored-by: Samer Boshra <sboshra@microsoft.com>

* Query: Fixes handling of pipeline execution on partition merge (#2531)

* fix

* fixing inmemorycontainer

* fixing merge test

* undo small change

* OrderBy fix too

* Correct orderby handling

* more tests

* text

Co-authored-by: j82w <j82w@users.noreply.github.com>

* [Internal] Client Encryption : Adds test to verify that update of Client Encryption Policy is not allowed via ReplaceContainer (#2349)

Adds / updates existing tests to verify -
a. Update of Client Encryption Policy is not allowed via ReplaceContainer
b. CreateContainer request ensures that the ClientEncryptionKey exists when creating Client Encryption Policy

Co-authored-by: neildsh <35383880+neildsh@users.noreply.github.com>
Co-authored-by: Samer Boshra <sboshra@microsoft.com>
Co-authored-by: j82w <j82w@users.noreply.github.com>
Co-authored-by: anujtoshniwal <62551957+anujtoshniwal@users.noreply.github.com>
  • Loading branch information
@ealsur @neildsh
@sboshra @j82w @anujtoshniwal
5 people authored Jun 14, 2021
1 parent 561bfda commit e303be9
Show file tree
Hide file tree
Showing 11 changed files with 590 additions and 103 deletions.
30 changes: 0 additions & 30 deletions Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,36 +362,6 @@ public async Task EncryptionResourceTokenAuthRestricted()
}
}

[TestMethod]
public async Task EncryptionFailsWithUnknownClientEncryptionKey()
{
ClientEncryptionIncludedPath unknownKeyConfigured = new ClientEncryptionIncludedPath()
{
Path = "/",
ClientEncryptionKeyId = "unknownKey",
EncryptionType = "Deterministic",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
};

Collection<ClientEncryptionIncludedPath> paths = new Collection<ClientEncryptionIncludedPath> { unknownKeyConfigured };
ClientEncryptionPolicy clientEncryptionPolicyId = new ClientEncryptionPolicy(paths);

ContainerProperties containerProperties = new ContainerProperties(Guid.NewGuid().ToString(), "/PK") { ClientEncryptionPolicy = clientEncryptionPolicyId };

Container encryptionContainer = await database.CreateContainerAsync(containerProperties, 400);

try
{
await encryptionContainer.InitializeEncryptionAsync();
await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainer);
Assert.Fail("Expected item creation should fail since client encryption policy is configured with unknown key.");
}
catch (Exception ex)
{
Assert.IsTrue(ex is InvalidOperationException);
}
}

[TestMethod]
public async Task ClientEncryptionPolicyTests()
{
Expand Down
34 changes: 23 additions & 11 deletions Microsoft.Azure.Cosmos/src/Pagination/CrossPartitionRangePageAsyncEnumerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,12 +147,8 @@ public async ValueTask<bool> MoveNextAsync(ITrace trace)
currentPaginator.FeedRangeState.FeedRange,
childTrace,
this.cancellationToken);
if (childRanges.Count == 0)
{
throw new InvalidOperationException("Got back no children");
}

if (childRanges.Count == 1)
if (childRanges.Count <= 1)
{
// We optimistically assumed that the cache is not stale.
// In the event that it is (where we only get back one child / the partition that we think got split)
Expand All @@ -164,16 +160,32 @@ public async ValueTask<bool> MoveNextAsync(ITrace trace)
this.cancellationToken);
}

if (childRanges.Count() <= 1)
if (childRanges.Count < 1)
{
throw new InvalidOperationException("Expected more than 1 child");
string errorMessage = "SDK invariant violated 4795CC37: Must have at least one EPK range in a cross partition enumerator";
throw Resource.CosmosExceptions.CosmosExceptionFactory.CreateInternalServerErrorException(
message: errorMessage,
headers: null,
stackTrace: null,
trace: childTrace,
error: new Microsoft.Azure.Documents.Error { Code = "SDK_invariant_violated_4795CC37", Message = errorMessage });
}

foreach (FeedRangeInternal childRange in childRanges)
if (childRanges.Count == 1)
{
// On a merge, the 410/1002 results in a single parent
// We maintain the current enumerator's range and let the RequestInvokerHandler logic kick in
enumerators.Enqueue(currentPaginator);
}
else
{
PartitionRangePageAsyncEnumerator<TPage, TState> childPaginator = this.createPartitionRangeEnumerator(
new FeedRangeState<TState>(childRange, currentPaginator.FeedRangeState.State));
enumerators.Enqueue(childPaginator);
// Split
foreach (FeedRangeInternal childRange in childRanges)
{
PartitionRangePageAsyncEnumerator<TPage, TState> childPaginator = this.createPartitionRangeEnumerator(
new FeedRangeState<TState>(childRange, currentPaginator.FeedRangeState.State));
enumerators.Enqueue(childPaginator);
}
}

// Recursively retry
Expand Down
42 changes: 31 additions & 11 deletions ...src/Query/Core/Pipeline/CrossPartition/OrderBy/OrderByCrossPartitionQueryPipelineStage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,12 +294,8 @@ private async ValueTask<bool> MoveNextAsync_InitializeAsync_HandleSplitAsync(
uninitializedEnumerator.FeedRangeState.FeedRange,
trace,
this.cancellationToken);
if (childRanges.Count == 0)
{
throw new InvalidOperationException("Got back no children");
}

if (childRanges.Count == 1)
if (childRanges.Count <= 1)
{
// We optimistically assumed that the cache is not stale.
// In the event that it is (where we only get back one child / the partition that we think got split)
Expand All @@ -311,25 +307,49 @@ private async ValueTask<bool> MoveNextAsync_InitializeAsync_HandleSplitAsync(
this.cancellationToken);
}

if (childRanges.Count() <= 1)
if (childRanges.Count < 1)
{
throw new InvalidOperationException("Expected more than 1 child");
string errorMessage = "SDK invariant violated 82086B2D: Must have at least one EPK range in a cross partition enumerator";
throw Resource.CosmosExceptions.CosmosExceptionFactory.CreateInternalServerErrorException(
message: errorMessage,
headers: null,
stackTrace: null,
trace: trace,
error: new Microsoft.Azure.Documents.Error { Code = "SDK_invariant_violated_82086B2D", Message = errorMessage });
}

foreach (FeedRangeInternal childRange in childRanges)
if (childRanges.Count == 1)
{
this.cancellationToken.ThrowIfCancellationRequested();

// On a merge, the 410/1002 results in a single parent
// We maintain the current enumerator's range and let the RequestInvokerHandler logic kick in
OrderByQueryPartitionRangePageAsyncEnumerator childPaginator = new OrderByQueryPartitionRangePageAsyncEnumerator(
this.documentContainer,
uninitializedEnumerator.SqlQuerySpec,
new FeedRangeState<QueryState>(childRange, uninitializedEnumerator.StartOfPageState),
new FeedRangeState<QueryState>(uninitializedEnumerator.FeedRangeState.FeedRange, uninitializedEnumerator.StartOfPageState),
partitionKey: null,
uninitializedEnumerator.QueryPaginationOptions,
uninitializedEnumerator.Filter,
this.cancellationToken);
this.uninitializedEnumeratorsAndTokens.Enqueue((childPaginator, token));
}
else
{
// Split
foreach (FeedRangeInternal childRange in childRanges)
{
this.cancellationToken.ThrowIfCancellationRequested();

OrderByQueryPartitionRangePageAsyncEnumerator childPaginator = new OrderByQueryPartitionRangePageAsyncEnumerator(
this.documentContainer,
uninitializedEnumerator.SqlQuerySpec,
new FeedRangeState<QueryState>(childRange, uninitializedEnumerator.StartOfPageState),
partitionKey: null,
uninitializedEnumerator.QueryPaginationOptions,
uninitializedEnumerator.Filter,
this.cancellationToken);
this.uninitializedEnumeratorsAndTokens.Enqueue((childPaginator, token));
}
}

// Recursively retry
return await this.MoveNextAsync(trace);
Expand Down
58 changes: 58 additions & 0 deletions Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosContainerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1320,9 +1320,41 @@ public async Task TimeToLivePropertyPath()
Assert.AreEqual(HttpStatusCode.NoContent, containerResponse.StatusCode);
}

[TestMethod]
public async Task ContainerCreationFailsWithUnknownClientEncryptionKey()
{
ClientEncryptionIncludedPath unknownKeyConfigured = new ClientEncryptionIncludedPath()
{
Path = "/",
ClientEncryptionKeyId = "unknownKey",
EncryptionType = "Deterministic",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
};

Collection<ClientEncryptionIncludedPath> paths = new Collection<ClientEncryptionIncludedPath> { unknownKeyConfigured };
ClientEncryptionPolicy clientEncryptionPolicyId = new ClientEncryptionPolicy(paths);

ContainerProperties containerProperties = new ContainerProperties(Guid.NewGuid().ToString(), "/PK") { ClientEncryptionPolicy = clientEncryptionPolicyId };

try
{
await this.cosmosDatabase.CreateContainerAsync(containerProperties, 400);
Assert.Fail("Expected container creation should fail since client encryption policy is configured with unknown key.");
}
catch (CosmosException ex)
{
Assert.AreEqual(HttpStatusCode.BadRequest, ex.StatusCode);
Assert.IsTrue(ex.Message.Contains("ClientEncryptionKey with id '[unknownKey]' does not exist."));
}
}

[TestMethod]
public async Task ClientEncryptionPolicyTest()
{
DatabaseInlineCore databaseInlineCore = (DatabaseInlineCore)this.cosmosDatabase;
await TestCommon.CreateClientEncryptionKey("dekId1", databaseInlineCore);
await TestCommon.CreateClientEncryptionKey("dekId2", databaseInlineCore);

string containerName = Guid.NewGuid().ToString();
string partitionKeyPath = "/users";
Collection<ClientEncryptionIncludedPath> paths = new Collection<ClientEncryptionIncludedPath>()
Expand Down Expand Up @@ -1371,6 +1403,32 @@ public async Task ClientEncryptionPolicyTest()
ContainerResponse readResponse = await container.ReadContainerAsync();
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
Assert.IsNotNull(readResponse.Resource.ClientEncryptionPolicy);

// replace without updating CEP should be successful
readResponse.Resource.IndexingPolicy = new Cosmos.IndexingPolicy()
{
IndexingMode = Cosmos.IndexingMode.None,
Automatic = false
};

containerResponse = await container.ReplaceContainerAsync(readResponse.Resource);
Assert.AreEqual(HttpStatusCode.OK, containerResponse.StatusCode);
Assert.AreEqual(Cosmos.IndexingMode.None, containerResponse.Resource.IndexingPolicy.IndexingMode);
Assert.IsFalse(containerResponse.Resource.IndexingPolicy.Automatic);

// update CEP and attempt replace
readResponse.Resource.ClientEncryptionPolicy = null;
try
{
await container.ReplaceContainerAsync(readResponse.Resource);

Assert.Fail("ReplaceCollection with update to ClientEncryptionPolicy should have failed.");
}
catch (CosmosException ex)
{
Assert.AreEqual(HttpStatusCode.BadRequest, ex.StatusCode);
Assert.IsTrue(ex.Message.Contains("'clientEncryptionPolicy' cannot be changed as part of collection replace operation."));
}
}

[TestMethod]
Expand Down
34 changes: 28 additions & 6 deletions ....Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Fluent/ContainerSettingsTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ public async Task Cleanup()
[TestMethod]
public async Task ContainerContractTest()
{
DatabaseInlineCore databaseInlineCore = (DatabaseInlineCore)this.database;
await TestCommon.CreateClientEncryptionKey("dekId", databaseInlineCore);

ClientEncryptionIncludedPath clientEncryptionIncludedPath1 = new ClientEncryptionIncludedPath()
{
Path = "/path",
Expand Down Expand Up @@ -558,7 +561,7 @@ public async Task TimeToLivePropertyPath()
containerResponse = await this.database.DefineContainer(containerName, partitionKeyPath)
.WithTimeToLivePropertyPath("/creationDate")
.CreateAsync();
Assert.Fail("CreateColleciton with TtlPropertyPath and with no DefaultTimeToLive should have failed.");
Assert.Fail("CreateCollection with TtlPropertyPath and with no DefaultTimeToLive should have failed.");
}
catch (CosmosException exeption)
{
Expand Down Expand Up @@ -593,20 +596,25 @@ public async Task TimeToLivePropertyPath()
[TestMethod]
public async Task WithClientEncryptionPolicyTest()
{
// create ClientEncryptionKeys
DatabaseInlineCore databaseInlineCore = (DatabaseInlineCore)this.database;
await TestCommon.CreateClientEncryptionKey("dekId1", databaseInlineCore);
await TestCommon.CreateClientEncryptionKey("dekId2", databaseInlineCore);

string containerName = Guid.NewGuid().ToString();
string partitionKeyPath = "/users";
ClientEncryptionIncludedPath path1 = new ClientEncryptionIncludedPath()
{
Path = "/path1",
ClientEncryptionKeyId = "key1",
ClientEncryptionKeyId = "dekId1",
EncryptionType = "Randomized",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256"
};

ClientEncryptionIncludedPath path2 = new ClientEncryptionIncludedPath()
{
Path = "/path2",
ClientEncryptionKeyId = "key2",
ClientEncryptionKeyId = "dekId2",
EncryptionType = "Randomized",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
};
Expand All @@ -632,6 +640,20 @@ public async Task WithClientEncryptionPolicyTest()
ContainerResponse readResponse = await container.ReadContainerAsync();
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
Assert.IsNotNull(readResponse.Resource.ClientEncryptionPolicy);

// update CEP and replace container
readResponse.Resource.ClientEncryptionPolicy = null;
try
{
await container.ReplaceContainerAsync(readResponse.Resource);

Assert.Fail("ReplaceCollection with update to ClientEncryptionPolicy should have failed.");
}
catch (CosmosException ex)
{
Assert.AreEqual(HttpStatusCode.BadRequest, ex.StatusCode);
Assert.IsTrue(ex.Message.Contains("'clientEncryptionPolicy' cannot be changed as part of collection replace operation."));
}
}

[TestMethod]
Expand All @@ -655,7 +677,7 @@ public async Task WithClientEncryptionPolicyFailureTest()
.Attach()
.CreateAsync();

Assert.Fail("CreateColleciton with invalid ClientEncryptionPolicy should have failed.");
Assert.Fail("CreateCollection with invalid ClientEncryptionPolicy should have failed.");
}
catch (ArgumentNullException ex)
{
Expand All @@ -673,7 +695,7 @@ public async Task WithClientEncryptionPolicyFailureTest()
.Attach()
.CreateAsync();

Assert.Fail("CreateColleciton with invalid ClientEncryptionPolicy should have failed.");
Assert.Fail("CreateCollection with invalid ClientEncryptionPolicy should have failed.");
}
catch (ArgumentException ex)
{
Expand All @@ -691,7 +713,7 @@ public async Task WithClientEncryptionPolicyFailureTest()
.Attach()
.CreateAsync();

Assert.Fail("CreateColleciton with invalid ClientEncryptionPolicy should have failed.");
Assert.Fail("CreateCollection with invalid ClientEncryptionPolicy should have failed.");
}
catch (ArgumentException ex)
{
Expand Down
36 changes: 36 additions & 0 deletions Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/TestCommon.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
Expand Down Expand Up @@ -341,6 +342,41 @@ internal static void RouteToTheOnlyPartition(DocumentClient client, DocumentServ
request.RouteTo(new PartitionKeyRangeIdentity(collection.ResourceId, ranges.Single().Id));
}

internal static async Task CreateClientEncryptionKey(
string dekId,
DatabaseInlineCore databaseInlineCore)
{
EncryptionKeyWrapMetadata metadata = new EncryptionKeyWrapMetadata("custom", dekId, "tempMetadata");

byte[] wrappedDataEncryptionKey = new byte[32];
// Generate random bytes cryptographically.
using (RNGCryptoServiceProvider rngCsp = new RNGCryptoServiceProvider())
{
rngCsp.GetBytes(wrappedDataEncryptionKey);
}

ClientEncryptionKeyProperties clientEncryptionKeyProperties = new ClientEncryptionKeyProperties(
dekId,
"AEAD_AES_256_CBC_HMAC_SHA256",
wrappedDataEncryptionKey,
metadata);

try
{
await databaseInlineCore.CreateClientEncryptionKeyAsync(clientEncryptionKeyProperties);
}
catch(CosmosException ex)
{
if (ex.StatusCode == HttpStatusCode.Conflict &&
ex.Message.Contains("Resource with specified id, name, or unique index already exists."))
{
return;
}

throw;
}
}

internal static Database CreateOrGetDatabase(DocumentClient client)
{
IList<Database> databases = TestCommon.ListAll<Database>(
Expand Down
Loading

0 comments on commit e303be9

Please sign in to comment.