Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

3.18.1-preview: Adds release notes and ports hotfixes for query #2532

Merged
merged 3 commits into from
Jun 14, 2021
Merged

3.18.1-preview: Adds release notes and ports hotfixes for query #2532

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3073175
Query: Fixes InvalidOperationException on merge to a single partition…
neildsh Jun 2, 2021
90320a1
Query: Fixes handling of pipeline execution on partition merge (#2531)
ealsur Jun 11, 2021
fad325a
[Internal] Client Encryption : Adds test to verify that update of Cli…
anujtoshniwal Apr 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 0 additions & 30 deletions Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,36 +362,6 @@ public async Task EncryptionResourceTokenAuthRestricted()
}
}

[TestMethod]
public async Task EncryptionFailsWithUnknownClientEncryptionKey()
{
ClientEncryptionIncludedPath unknownKeyConfigured = new ClientEncryptionIncludedPath()
{
Path = "/",
ClientEncryptionKeyId = "unknownKey",
EncryptionType = "Deterministic",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
};

Collection<ClientEncryptionIncludedPath> paths = new Collection<ClientEncryptionIncludedPath> { unknownKeyConfigured };
ClientEncryptionPolicy clientEncryptionPolicyId = new ClientEncryptionPolicy(paths);

ContainerProperties containerProperties = new ContainerProperties(Guid.NewGuid().ToString(), "/PK") { ClientEncryptionPolicy = clientEncryptionPolicyId };

Container encryptionContainer = await database.CreateContainerAsync(containerProperties, 400);

try
{
await encryptionContainer.InitializeEncryptionAsync();
await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainer);
Assert.Fail("Expected item creation should fail since client encryption policy is configured with unknown key.");
}
catch (Exception ex)
{
Assert.IsTrue(ex is InvalidOperationException);
}
}

[TestMethod]
public async Task ClientEncryptionPolicyTests()
{
Expand Down
34 changes: 23 additions & 11 deletions Microsoft.Azure.Cosmos/src/Pagination/CrossPartitionRangePageAsyncEnumerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,12 +147,8 @@ public async ValueTask<bool> MoveNextAsync(ITrace trace)
currentPaginator.FeedRangeState.FeedRange,
childTrace,
this.cancellationToken);
if (childRanges.Count == 0)
Copy link
Contributor

@j82w j82w Jun 11, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we include the following fixes so the diagnostics are actually useful?
#2424
#2375

{
throw new InvalidOperationException("Got back no children");
}

if (childRanges.Count == 1)
if (childRanges.Count <= 1)
{
// We optimistically assumed that the cache is not stale.
// In the event that it is (where we only get back one child / the partition that we think got split)
Expand All @@ -164,16 +160,32 @@ public async ValueTask<bool> MoveNextAsync(ITrace trace)
this.cancellationToken);
}

if (childRanges.Count() <= 1)
if (childRanges.Count < 1)
{
throw new InvalidOperationException("Expected more than 1 child");
string errorMessage = "SDK invariant violated 4795CC37: Must have at least one EPK range in a cross partition enumerator";
throw Resource.CosmosExceptions.CosmosExceptionFactory.CreateInternalServerErrorException(
message: errorMessage,
headers: null,
stackTrace: null,
trace: childTrace,
error: new Microsoft.Azure.Documents.Error { Code = "SDK_invariant_violated_4795CC37", Message = errorMessage });
}

foreach (FeedRangeInternal childRange in childRanges)
if (childRanges.Count == 1)
{
// On a merge, the 410/1002 results in a single parent
// We maintain the current enumerator's range and let the RequestInvokerHandler logic kick in
enumerators.Enqueue(currentPaginator);
}
else
{
PartitionRangePageAsyncEnumerator<TPage, TState> childPaginator = this.createPartitionRangeEnumerator(
new FeedRangeState<TState>(childRange, currentPaginator.FeedRangeState.State));
enumerators.Enqueue(childPaginator);
// Split
foreach (FeedRangeInternal childRange in childRanges)
{
PartitionRangePageAsyncEnumerator<TPage, TState> childPaginator = this.createPartitionRangeEnumerator(
new FeedRangeState<TState>(childRange, currentPaginator.FeedRangeState.State));
enumerators.Enqueue(childPaginator);
}
}

// Recursively retry
Expand Down
42 changes: 31 additions & 11 deletions ...src/Query/Core/Pipeline/CrossPartition/OrderBy/OrderByCrossPartitionQueryPipelineStage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,12 +294,8 @@ private async ValueTask<bool> MoveNextAsync_InitializeAsync_HandleSplitAsync(
uninitializedEnumerator.FeedRangeState.FeedRange,
trace,
this.cancellationToken);
if (childRanges.Count == 0)
{
throw new InvalidOperationException("Got back no children");
}

if (childRanges.Count == 1)
if (childRanges.Count <= 1)
{
// We optimistically assumed that the cache is not stale.
// In the event that it is (where we only get back one child / the partition that we think got split)
Expand All @@ -311,25 +307,49 @@ private async ValueTask<bool> MoveNextAsync_InitializeAsync_HandleSplitAsync(
this.cancellationToken);
}

if (childRanges.Count() <= 1)
if (childRanges.Count < 1)
{
throw new InvalidOperationException("Expected more than 1 child");
string errorMessage = "SDK invariant violated 82086B2D: Must have at least one EPK range in a cross partition enumerator";
throw Resource.CosmosExceptions.CosmosExceptionFactory.CreateInternalServerErrorException(
message: errorMessage,
headers: null,
stackTrace: null,
trace: trace,
error: new Microsoft.Azure.Documents.Error { Code = "SDK_invariant_violated_82086B2D", Message = errorMessage });
}

foreach (FeedRangeInternal childRange in childRanges)
if (childRanges.Count == 1)
{
this.cancellationToken.ThrowIfCancellationRequested();

// On a merge, the 410/1002 results in a single parent
// We maintain the current enumerator's range and let the RequestInvokerHandler logic kick in
OrderByQueryPartitionRangePageAsyncEnumerator childPaginator = new OrderByQueryPartitionRangePageAsyncEnumerator(
this.documentContainer,
uninitializedEnumerator.SqlQuerySpec,
new FeedRangeState<QueryState>(childRange, uninitializedEnumerator.StartOfPageState),
new FeedRangeState<QueryState>(uninitializedEnumerator.FeedRangeState.FeedRange, uninitializedEnumerator.StartOfPageState),
partitionKey: null,
uninitializedEnumerator.QueryPaginationOptions,
uninitializedEnumerator.Filter,
this.cancellationToken);
this.uninitializedEnumeratorsAndTokens.Enqueue((childPaginator, token));
}
else
{
// Split
foreach (FeedRangeInternal childRange in childRanges)
{
this.cancellationToken.ThrowIfCancellationRequested();

OrderByQueryPartitionRangePageAsyncEnumerator childPaginator = new OrderByQueryPartitionRangePageAsyncEnumerator(
this.documentContainer,
uninitializedEnumerator.SqlQuerySpec,
new FeedRangeState<QueryState>(childRange, uninitializedEnumerator.StartOfPageState),
partitionKey: null,
uninitializedEnumerator.QueryPaginationOptions,
uninitializedEnumerator.Filter,
this.cancellationToken);
this.uninitializedEnumeratorsAndTokens.Enqueue((childPaginator, token));
}
}

// Recursively retry
return await this.MoveNextAsync(trace);
Expand Down
58 changes: 58 additions & 0 deletions Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosContainerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1320,9 +1320,41 @@ public async Task TimeToLivePropertyPath()
Assert.AreEqual(HttpStatusCode.NoContent, containerResponse.StatusCode);
}

[TestMethod]
public async Task ContainerCreationFailsWithUnknownClientEncryptionKey()
{
ClientEncryptionIncludedPath unknownKeyConfigured = new ClientEncryptionIncludedPath()
{
Path = "/",
ClientEncryptionKeyId = "unknownKey",
EncryptionType = "Deterministic",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
};

Collection<ClientEncryptionIncludedPath> paths = new Collection<ClientEncryptionIncludedPath> { unknownKeyConfigured };
ClientEncryptionPolicy clientEncryptionPolicyId = new ClientEncryptionPolicy(paths);

ContainerProperties containerProperties = new ContainerProperties(Guid.NewGuid().ToString(), "/PK") { ClientEncryptionPolicy = clientEncryptionPolicyId };

try
{
await this.cosmosDatabase.CreateContainerAsync(containerProperties, 400);
Assert.Fail("Expected container creation should fail since client encryption policy is configured with unknown key.");
}
catch (CosmosException ex)
{
Assert.AreEqual(HttpStatusCode.BadRequest, ex.StatusCode);
Assert.IsTrue(ex.Message.Contains("ClientEncryptionKey with id '[unknownKey]' does not exist."));
}
}

[TestMethod]
public async Task ClientEncryptionPolicyTest()
{
DatabaseInlineCore databaseInlineCore = (DatabaseInlineCore)this.cosmosDatabase;
await TestCommon.CreateClientEncryptionKey("dekId1", databaseInlineCore);
await TestCommon.CreateClientEncryptionKey("dekId2", databaseInlineCore);

string containerName = Guid.NewGuid().ToString();
string partitionKeyPath = "/users";
Collection<ClientEncryptionIncludedPath> paths = new Collection<ClientEncryptionIncludedPath>()
Expand Down Expand Up @@ -1371,6 +1403,32 @@ public async Task ClientEncryptionPolicyTest()
ContainerResponse readResponse = await container.ReadContainerAsync();
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
Assert.IsNotNull(readResponse.Resource.ClientEncryptionPolicy);

// replace without updating CEP should be successful
readResponse.Resource.IndexingPolicy = new Cosmos.IndexingPolicy()
{
IndexingMode = Cosmos.IndexingMode.None,
Automatic = false
};

containerResponse = await container.ReplaceContainerAsync(readResponse.Resource);
Assert.AreEqual(HttpStatusCode.OK, containerResponse.StatusCode);
Assert.AreEqual(Cosmos.IndexingMode.None, containerResponse.Resource.IndexingPolicy.IndexingMode);
Assert.IsFalse(containerResponse.Resource.IndexingPolicy.Automatic);

// update CEP and attempt replace
readResponse.Resource.ClientEncryptionPolicy = null;
try
{
await container.ReplaceContainerAsync(readResponse.Resource);

Assert.Fail("ReplaceCollection with update to ClientEncryptionPolicy should have failed.");
}
catch (CosmosException ex)
{
Assert.AreEqual(HttpStatusCode.BadRequest, ex.StatusCode);
Assert.IsTrue(ex.Message.Contains("'clientEncryptionPolicy' cannot be changed as part of collection replace operation."));
}
}

[TestMethod]
Expand Down
34 changes: 28 additions & 6 deletions ....Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Fluent/ContainerSettingsTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ public async Task Cleanup()
[TestMethod]
public async Task ContainerContractTest()
{
DatabaseInlineCore databaseInlineCore = (DatabaseInlineCore)this.database;
await TestCommon.CreateClientEncryptionKey("dekId", databaseInlineCore);

ClientEncryptionIncludedPath clientEncryptionIncludedPath1 = new ClientEncryptionIncludedPath()
{
Path = "/path",
Expand Down Expand Up @@ -558,7 +561,7 @@ public async Task TimeToLivePropertyPath()
containerResponse = await this.database.DefineContainer(containerName, partitionKeyPath)
.WithTimeToLivePropertyPath("/creationDate")
.CreateAsync();
Assert.Fail("CreateColleciton with TtlPropertyPath and with no DefaultTimeToLive should have failed.");
Assert.Fail("CreateCollection with TtlPropertyPath and with no DefaultTimeToLive should have failed.");
}
catch (CosmosException exeption)
{
Expand Down Expand Up @@ -593,20 +596,25 @@ public async Task TimeToLivePropertyPath()
[TestMethod]
public async Task WithClientEncryptionPolicyTest()
{
// create ClientEncryptionKeys
DatabaseInlineCore databaseInlineCore = (DatabaseInlineCore)this.database;
await TestCommon.CreateClientEncryptionKey("dekId1", databaseInlineCore);
await TestCommon.CreateClientEncryptionKey("dekId2", databaseInlineCore);

string containerName = Guid.NewGuid().ToString();
string partitionKeyPath = "/users";
ClientEncryptionIncludedPath path1 = new ClientEncryptionIncludedPath()
{
Path = "/path1",
ClientEncryptionKeyId = "key1",
ClientEncryptionKeyId = "dekId1",
EncryptionType = "Randomized",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256"
};

ClientEncryptionIncludedPath path2 = new ClientEncryptionIncludedPath()
{
Path = "/path2",
ClientEncryptionKeyId = "key2",
ClientEncryptionKeyId = "dekId2",
EncryptionType = "Randomized",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
};
Expand All @@ -632,6 +640,20 @@ public async Task WithClientEncryptionPolicyTest()
ContainerResponse readResponse = await container.ReadContainerAsync();
Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode);
Assert.IsNotNull(readResponse.Resource.ClientEncryptionPolicy);

// update CEP and replace container
readResponse.Resource.ClientEncryptionPolicy = null;
try
{
await container.ReplaceContainerAsync(readResponse.Resource);

Assert.Fail("ReplaceCollection with update to ClientEncryptionPolicy should have failed.");
}
catch (CosmosException ex)
{
Assert.AreEqual(HttpStatusCode.BadRequest, ex.StatusCode);
Assert.IsTrue(ex.Message.Contains("'clientEncryptionPolicy' cannot be changed as part of collection replace operation."));
}
}

[TestMethod]
Expand All @@ -655,7 +677,7 @@ public async Task WithClientEncryptionPolicyFailureTest()
.Attach()
.CreateAsync();

Assert.Fail("CreateColleciton with invalid ClientEncryptionPolicy should have failed.");
Assert.Fail("CreateCollection with invalid ClientEncryptionPolicy should have failed.");
}
catch (ArgumentNullException ex)
{
Expand All @@ -673,7 +695,7 @@ public async Task WithClientEncryptionPolicyFailureTest()
.Attach()
.CreateAsync();

Assert.Fail("CreateColleciton with invalid ClientEncryptionPolicy should have failed.");
Assert.Fail("CreateCollection with invalid ClientEncryptionPolicy should have failed.");
}
catch (ArgumentException ex)
{
Expand All @@ -691,7 +713,7 @@ public async Task WithClientEncryptionPolicyFailureTest()
.Attach()
.CreateAsync();

Assert.Fail("CreateColleciton with invalid ClientEncryptionPolicy should have failed.");
Assert.Fail("CreateCollection with invalid ClientEncryptionPolicy should have failed.");
}
catch (ArgumentException ex)
{
Expand Down
36 changes: 36 additions & 0 deletions Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/TestCommon.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
Expand Down Expand Up @@ -341,6 +342,41 @@ internal static void RouteToTheOnlyPartition(DocumentClient client, DocumentServ
request.RouteTo(new PartitionKeyRangeIdentity(collection.ResourceId, ranges.Single().Id));
}

internal static async Task CreateClientEncryptionKey(
string dekId,
DatabaseInlineCore databaseInlineCore)
{
EncryptionKeyWrapMetadata metadata = new EncryptionKeyWrapMetadata("custom", dekId, "tempMetadata");

byte[] wrappedDataEncryptionKey = new byte[32];
// Generate random bytes cryptographically.
using (RNGCryptoServiceProvider rngCsp = new RNGCryptoServiceProvider())
{
rngCsp.GetBytes(wrappedDataEncryptionKey);
}

ClientEncryptionKeyProperties clientEncryptionKeyProperties = new ClientEncryptionKeyProperties(
dekId,
"AEAD_AES_256_CBC_HMAC_SHA256",
wrappedDataEncryptionKey,
metadata);

try
{
await databaseInlineCore.CreateClientEncryptionKeyAsync(clientEncryptionKeyProperties);
}
catch(CosmosException ex)
{
if (ex.StatusCode == HttpStatusCode.Conflict &&
ex.Message.Contains("Resource with specified id, name, or unique index already exists."))
{
return;
}

throw;
}
}

internal static Database CreateOrGetDatabase(DocumentClient client)
{
IList<Database> databases = TestCommon.ListAll<Database>(
Expand Down
Loading