From af580e90c4855caeb4701b31f072163c310e2740 Mon Sep 17 00:00:00 2001 From: Nalu Tripician Date: Fri, 19 May 2023 17:03:07 -0400 Subject: [PATCH 1/5] Initial Commit DO NOT REVIEW --- .../CosmosMultiHashTest.cs | 40 +++++++++++++++++ .../FeedToken/ChangeFeedIteratorCoreTests.cs | 42 ++++++++++++++++++ .../Utils/ToDoActivity.cs | 44 +++++++++++++++++++ 3 files changed, 126 insertions(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosMultiHashTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosMultiHashTest.cs index 5307145dfc..e9370c24c0 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosMultiHashTest.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosMultiHashTest.cs @@ -35,6 +35,46 @@ public async Task Cleanup() this.client.Dispose(); } + [TestMethod] + public async Task REMOVEME() + { + string ConnectionString = ""; + string DatabaseName = "LeaderboardsData"; + string ContainerId = "DataContainer4"; + + Console.WriteLine($"{DateTime.Now}:Starting ChangeFeedPullReader"); + + //Container container = CreateDatabaseAndContainerAsync(ConnectionString, DatabaseName, AutoscaleMaxThroughput, ContainerId).Result; + CosmosClient cosmosClient = new CosmosClient(ConnectionString); + Cosmos.Database database = cosmosClient.GetDatabase(DatabaseName); + Cosmos.Container container = database.GetContainer(ContainerId); + + Cosmos.PartitionKey shardPartitionKey = new PartitionKeyBuilder().Add($"shard0").Build(); + FeedRange shardFeedRange = FeedRange.FromPartitionKey(shardPartitionKey); + + FeedIterator iteratorForShard = container.GetChangeFeedIterator(ChangeFeedStartFrom.Beginning(shardFeedRange), ChangeFeedMode.Incremental, new ChangeFeedRequestOptions() { PageSizeHint = 10 }); + int abc = 0; + // Read all changes for documents for the shard. + while (iteratorForShard.HasMoreResults) + { + Console.Out.WriteLine("Before read" + abc.ToString()); + // Consume all changes for documents for the shard. + FeedResponse response = await iteratorForShard.ReadNextAsync(); // BUG: Code gets stuck here! + Console.Out.WriteLine("After read" + abc.ToString()); + + Console.WriteLine($"Response Status: {response.StatusCode} Request Charge: {response.RequestCharge}"); + + if (response.StatusCode == HttpStatusCode.NotModified) + { + Console.WriteLine($"No new changes" + abc.ToString()); + await Task.Delay(TimeSpan.FromSeconds(5)); + } + abc++; + } + + Console.WriteLine($"{DateTime.Now}:Exiting ChangeFeedPullReader."); + } + [TestMethod] public async Task MultiHashCreateDocumentTest() { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs index 4a1d92c59c..02d76391ef 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs @@ -26,6 +26,7 @@ namespace Microsoft.Azure.Cosmos.EmulatorTests.FeedRanges public class ChangeFeedIteratorCoreTests : BaseCosmosClientHelper { private static readonly string PartitionKey = "/pk"; + private static readonly List PartitionKeyPaths = new List { "/pk0", "/pk1", "/pk2" }; [TestInitialize] public async Task TestInitialize() @@ -49,6 +50,16 @@ private async Task InitializeLargeContainerAsync() return (ContainerInternal)response; } + private async Task InitializeLargeMultiHashedContainerAsync() + { + ContainerResponse response = await this.database.CreateContainerAsync( + new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPath: ChangeFeedIteratorCoreTests.PartitionKeyPaths), + throughput: 20000, + cancellationToken: this.cancellationToken); + + return (ContainerInternal)response; + } + private async Task InitializeContainerAsync() { ContainerResponse response = await this.database.CreateContainerAsync( @@ -1202,7 +1213,38 @@ private static void AssertGatewayMode(FeedResponse feedResponse) Assert.IsNotNull(jsonToken["Summary"]["GatewayCalls"], "'GatewayCalls' is not found in diagnostics. UseGateMode is set to false."); } + private async Task> CreateRandomItemsForMultihashedContainer(ContainerInternal container, int pkCount, int perPKItemCount = 1, bool randomPartitionKey = true) + { + Assert.IsFalse(!randomPartitionKey && perPKItemCount > 1); + + List createdList = new List(); + for (int i = 0; i < pkCount; i++) + { + string pk0 = "TBD-0"; + string pk1 = "TBD-1"; + string pk2 = "TBD-2"; + + if (randomPartitionKey) + { + pk0 += Guid.NewGuid().ToString(); + pk0 += Guid.NewGuid().ToString(); + pk0 += Guid.NewGuid().ToString(); + } + + List pks = new List { pk0, pk1, pk2 }; + for (int j = 0; j < perPKItemCount; j++) + { + ToDoActivity temp = ToDoActivity.CreateRandomToDoActivityMultiHashed(pks: pks); + + createdList.Add(temp); + + await container.CreateItemAsync(item: temp); + } + } + + return createdList; + } private async Task> CreateRandomItems(ContainerInternal container, int pkCount, int perPKItemCount = 1, bool randomPartitionKey = true) { Assert.IsFalse(!randomPartitionKey && perPKItemCount > 1); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/ToDoActivity.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/ToDoActivity.cs index c2800c02d2..76a7136524 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/ToDoActivity.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/ToDoActivity.cs @@ -16,6 +16,10 @@ public class ToDoActivity public double cost { get; set; } public string description { get; set; } public string pk { get; set; } + + public string pk0 { get; set; } + public string pk1 { get; set; } + public string pk2 { get; set; } public string CamelCase { get; set; } public int? nullableInt { get; set; } @@ -76,6 +80,46 @@ public static async Task> CreateRandomItems( return createdList; } + public static ToDoActivity CreateRandomToDoActivityMultiHashed( + List pks = null, + string id = null, + bool randomTaskNumber = false) + { + if(pks.Count == 0) + { + pks.Add("TBD-0-" + Guid.NewGuid().ToString()); + pks.Add("TBD-1-" + Guid.NewGuid().ToString()); + pks.Add("TBD-2-" + Guid.NewGuid().ToString()); + } + if (id == null) + { + id = Guid.NewGuid().ToString(); + } + + int taskNum = 42; + if (randomTaskNumber) + { + taskNum = Random.Shared.Next(); + } + + return new ToDoActivity() + { + id = id, + description = "CreateRandomToDoActivityMultiHashed", + pk0 = pks[0], + pk1 = pks[1], + pk2 = pks[2], + taskNum = taskNum, + cost = double.MaxValue, + CamelCase = "camelCase", + children = new ToDoActivity[] + { new ToDoActivity { id = "child1", taskNum = 30 }, + new ToDoActivity { id = "child2", taskNum = 40} + }, + valid = true, + nullableInt = null + }; + } public static ToDoActivity CreateRandomToDoActivity( string pk = null, From ef246ba8594ad78b1c0c0074cdbee5fc806fb937 Mon Sep 17 00:00:00 2001 From: Nalu Tripician Date: Thu, 25 May 2023 13:21:41 -0400 Subject: [PATCH 2/5] bug fix, needs Direct Package Changes --- .../FeedRanges/FeedRangePartitionKey.cs | 36 ++- .../NetworkAttachedDocumentContainer.cs | 43 +++ .../CosmosQueryExecutionContextFactory.cs | 74 +++++- .../CosmosMultiHashTest.cs | 42 +-- .../FeedToken/ChangeFeedIteratorCoreTests.cs | 44 +-- .../Utils/ToDoActivity.cs | 40 --- .../Routing/PartitionRoutingHelperTest.cs | 251 ++++++++++++++++++ 7 files changed, 399 insertions(+), 131 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/FeedRange/FeedRanges/FeedRangePartitionKey.cs b/Microsoft.Azure.Cosmos/src/FeedRange/FeedRanges/FeedRangePartitionKey.cs index 47f84fd280..1c643bc4fb 100644 --- a/Microsoft.Azure.Cosmos/src/FeedRange/FeedRanges/FeedRangePartitionKey.cs +++ b/Microsoft.Azure.Cosmos/src/FeedRange/FeedRanges/FeedRangePartitionKey.cs @@ -4,11 +4,13 @@ namespace Microsoft.Azure.Cosmos { + using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Routing; using Microsoft.Azure.Cosmos.Tracing; + using Microsoft.Azure.Documents; /// /// FeedRange that represents an exact Partition Key value. @@ -22,18 +24,40 @@ public FeedRangePartitionKey(PartitionKey partitionKey) this.PartitionKey = partitionKey; } - internal override Task>> GetEffectiveRangesAsync( + internal override async Task>> GetEffectiveRangesAsync( IRoutingMapProvider routingMapProvider, string containerRid, Documents.PartitionKeyDefinition partitionKeyDefinition, ITrace trace) { - return Task.FromResult( - new List> + if (partitionKeyDefinition.Kind == PartitionKind.MultiHash) + { + Documents.Routing.Range range = this.PartitionKey.InternalKey.GetEPKRangeForPrefixPartitionKey(partitionKeyDefinition); + IReadOnlyList overlappingRanges = await routingMapProvider.TryGetOverlappingRangesAsync( + containerRid, + range, + trace, + forceRefresh: false); + List> effectiveRanges = new List>(); + + foreach (PartitionKeyRange overlappingRange in overlappingRanges) { - Documents.Routing.Range.GetPointRange( - this.PartitionKey.InternalKey.GetEffectivePartitionKeyString(partitionKeyDefinition)) - }); + effectiveRanges.Add(new Documents.Routing.Range(overlappingRange.MinInclusive, overlappingRange.MaxExclusive, true, false)); + } + + return effectiveRanges; + } + + return new List> + { + Documents.Routing.Range.GetPointRange( + this.PartitionKey.InternalKey.GetEffectivePartitionKeyString(partitionKeyDefinition)) + }; + + //return new List> + //{ + // PartitionKey.InternalKey.GetEffectivePartitionKeyRange(partitionKeyDefinition, new Documents.Routing.Range(this.PartitionKey.InternalKey, this.PartitionKey.InternalKey)) + //}; } internal override async Task> GetPartitionKeyRangesAsync( diff --git a/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs b/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs index 1132a40a35..380d43da49 100644 --- a/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs +++ b/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs @@ -14,6 +14,7 @@ namespace Microsoft.Azure.Cosmos.Pagination using System.Text; using System.Threading; using System.Threading.Tasks; + using global::Azure; using Microsoft.Azure.Cosmos.ChangeFeed.Pagination; using Microsoft.Azure.Cosmos.CosmosElements; using Microsoft.Azure.Cosmos.Diagnostics; @@ -126,6 +127,48 @@ await this.container.GetCachedRIDAsync(forceRefresh: false, trace, cancellationT feedRange, forceRefresh: false, trace); + + if (containerProperties.PartitionKey.Kind == PartitionKind.MultiHash && feedRange.GetType() == typeof(FeedRangePartitionKey)) + { + FeedRangePartitionKey frpk = (FeedRangePartitionKey)feedRange; + Documents.Routing.Range range = frpk.PartitionKey.InternalKey.GetEPKRangeForPrefixPartitionKey(containerProperties.PartitionKey); + + List feedRanges = new List(); + int count = 0; + foreach (PartitionKeyRange partitionKeyRange in overlappingRanges) + { + //First and last range must incude the min and max value for the original EPK range + if (count == 0) + { + feedRanges.Add(new FeedRangeEpk( + new Documents.Routing.Range( + min: range.Min, + max: partitionKeyRange.MaxExclusive, + isMinInclusive: true, + isMaxInclusive: false))); + } + else if (count == overlappingRanges.Count - 1) + { + feedRanges.Add(new FeedRangeEpk( + new Documents.Routing.Range( + min: partitionKeyRange.MinInclusive, + max: range.Max, + isMinInclusive: true, + isMaxInclusive: false))); + } + else + { + feedRanges.Add(new FeedRangeEpk( + new Documents.Routing.Range( + min: partitionKeyRange.MinInclusive, + max: partitionKeyRange.MaxExclusive, + isMinInclusive: true, + isMaxInclusive: false))); + } + count++; + } + return TryCatch>.FromResult(feedRanges); + } return TryCatch>.FromResult( overlappingRanges.Select(range => new FeedRangeEpk( new Documents.Routing.Range( diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs index e276466f9d..5fb4341123 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs @@ -311,11 +311,63 @@ private static async Task> TryCreateFromPartitione { SetTestInjectionPipelineType(inputParameters, Passthrough); - tryCreatePipelineStage = CosmosQueryExecutionContextFactory.TryCreatePassthroughQueryExecutionContext( + if (containerQueryProperties.PartitionKeyDefinition.Kind == Documents.PartitionKind.MultiHash + && inputParameters.InitialFeedRange.GetType() == typeof(FeedRangePartitionKey)) + { + FeedRangePartitionKey frpk = (FeedRangePartitionKey)inputParameters.InitialFeedRange; + Documents.Routing.Range range = frpk.PartitionKey.InternalKey.GetEPKRangeForPrefixPartitionKey(containerQueryProperties.PartitionKeyDefinition); + + List feedRanges = new List(); + int count = 0; + foreach (Documents.PartitionKeyRange partitionKeyRange in targetRanges) + { + if (count == 0) + { + feedRanges.Add(new FeedRangeEpk( + new Documents.Routing.Range( + min: range.Min, + max: partitionKeyRange.MaxExclusive, + isMinInclusive: true, + isMaxInclusive: false))); + } + else if (count == targetRanges.Count - 1) + { + feedRanges.Add(new FeedRangeEpk( + new Documents.Routing.Range( + min: partitionKeyRange.MinInclusive, + max: range.Max, + isMinInclusive: true, + isMaxInclusive: false))); + } + else + { + feedRanges.Add(new FeedRangeEpk( + new Documents.Routing.Range( + min: partitionKeyRange.MinInclusive, + max: partitionKeyRange.MaxExclusive, + isMinInclusive: true, + isMaxInclusive: false))); + } + + count++; + } + + tryCreatePipelineStage = CosmosQueryExecutionContextFactory.TryCreatePassthroughQueryExecutionContext( + documentContainer, + inputParameters, + feedRanges, + cancellationToken); + + } + else + { + tryCreatePipelineStage = CosmosQueryExecutionContextFactory.TryCreatePassthroughQueryExecutionContext( documentContainer, inputParameters, targetRanges, cancellationToken); + } + } else { @@ -522,6 +574,26 @@ private static TryCatch TryCreatePassthroughQueryExecutionC continuationToken: inputParameters.InitialUserContinuationToken); } + private static TryCatch TryCreatePassthroughQueryExecutionContext( + DocumentContainer documentContainer, + InputParameters inputParameters, + List targetRanges, + CancellationToken cancellationToken) + { + // Return a parallel context, since we still want to be able to handle splits and concurrency / buffering. + return ParallelCrossPartitionQueryPipelineStage.MonadicCreate( + documentContainer: documentContainer, + sqlQuerySpec: inputParameters.SqlQuerySpec, + targetRanges: targetRanges, + queryPaginationOptions: new QueryPaginationOptions( + pageSizeHint: inputParameters.MaxItemCount), + partitionKey: inputParameters.PartitionKey, + prefetchPolicy: PrefetchPolicy.PrefetchSinglePage, + maxConcurrency: inputParameters.MaxConcurrency, + cancellationToken: cancellationToken, + continuationToken: inputParameters.InitialUserContinuationToken); + } + private static TryCatch TryCreateSpecializedDocumentQueryExecutionContext( DocumentContainer documentContainer, CosmosQueryContext cosmosQueryContext, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosMultiHashTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosMultiHashTest.cs index e9370c24c0..dbdc326ea7 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosMultiHashTest.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosMultiHashTest.cs @@ -33,47 +33,7 @@ public async Task Cleanup() { await this.database.DeleteAsync(); this.client.Dispose(); - } - - [TestMethod] - public async Task REMOVEME() - { - string ConnectionString = ""; - string DatabaseName = "LeaderboardsData"; - string ContainerId = "DataContainer4"; - - Console.WriteLine($"{DateTime.Now}:Starting ChangeFeedPullReader"); - - //Container container = CreateDatabaseAndContainerAsync(ConnectionString, DatabaseName, AutoscaleMaxThroughput, ContainerId).Result; - CosmosClient cosmosClient = new CosmosClient(ConnectionString); - Cosmos.Database database = cosmosClient.GetDatabase(DatabaseName); - Cosmos.Container container = database.GetContainer(ContainerId); - - Cosmos.PartitionKey shardPartitionKey = new PartitionKeyBuilder().Add($"shard0").Build(); - FeedRange shardFeedRange = FeedRange.FromPartitionKey(shardPartitionKey); - - FeedIterator iteratorForShard = container.GetChangeFeedIterator(ChangeFeedStartFrom.Beginning(shardFeedRange), ChangeFeedMode.Incremental, new ChangeFeedRequestOptions() { PageSizeHint = 10 }); - int abc = 0; - // Read all changes for documents for the shard. - while (iteratorForShard.HasMoreResults) - { - Console.Out.WriteLine("Before read" + abc.ToString()); - // Consume all changes for documents for the shard. - FeedResponse response = await iteratorForShard.ReadNextAsync(); // BUG: Code gets stuck here! - Console.Out.WriteLine("After read" + abc.ToString()); - - Console.WriteLine($"Response Status: {response.StatusCode} Request Charge: {response.RequestCharge}"); - - if (response.StatusCode == HttpStatusCode.NotModified) - { - Console.WriteLine($"No new changes" + abc.ToString()); - await Task.Delay(TimeSpan.FromSeconds(5)); - } - abc++; - } - - Console.WriteLine($"{DateTime.Now}:Exiting ChangeFeedPullReader."); - } + } [TestMethod] public async Task MultiHashCreateDocumentTest() diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs index 02d76391ef..7cbe16aa10 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs @@ -26,7 +26,6 @@ namespace Microsoft.Azure.Cosmos.EmulatorTests.FeedRanges public class ChangeFeedIteratorCoreTests : BaseCosmosClientHelper { private static readonly string PartitionKey = "/pk"; - private static readonly List PartitionKeyPaths = new List { "/pk0", "/pk1", "/pk2" }; [TestInitialize] public async Task TestInitialize() @@ -50,16 +49,6 @@ private async Task InitializeLargeContainerAsync() return (ContainerInternal)response; } - private async Task InitializeLargeMultiHashedContainerAsync() - { - ContainerResponse response = await this.database.CreateContainerAsync( - new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPath: ChangeFeedIteratorCoreTests.PartitionKeyPaths), - throughput: 20000, - cancellationToken: this.cancellationToken); - - return (ContainerInternal)response; - } - private async Task InitializeContainerAsync() { ContainerResponse response = await this.database.CreateContainerAsync( @@ -84,7 +73,7 @@ private async Task InitializeFFCFContainerAsync(TimeSpan time return (ContainerInternal)response; } - + /// /// Test to verify that StartFromBeginning works as expected by inserting 25 items, reading them all, then taking the last continuationtoken, /// inserting another 25, and verifying that the iterator continues from the saved token and reads the second 25 for a total of 50 documents. @@ -1213,38 +1202,7 @@ private static void AssertGatewayMode(FeedResponse feedResponse) Assert.IsNotNull(jsonToken["Summary"]["GatewayCalls"], "'GatewayCalls' is not found in diagnostics. UseGateMode is set to false."); } - private async Task> CreateRandomItemsForMultihashedContainer(ContainerInternal container, int pkCount, int perPKItemCount = 1, bool randomPartitionKey = true) - { - Assert.IsFalse(!randomPartitionKey && perPKItemCount > 1); - - List createdList = new List(); - for (int i = 0; i < pkCount; i++) - { - string pk0 = "TBD-0"; - string pk1 = "TBD-1"; - string pk2 = "TBD-2"; - if (randomPartitionKey) - { - pk0 += Guid.NewGuid().ToString(); - pk0 += Guid.NewGuid().ToString(); - pk0 += Guid.NewGuid().ToString(); - } - - List pks = new List { pk0, pk1, pk2 }; - - for (int j = 0; j < perPKItemCount; j++) - { - ToDoActivity temp = ToDoActivity.CreateRandomToDoActivityMultiHashed(pks: pks); - - createdList.Add(temp); - - await container.CreateItemAsync(item: temp); - } - } - - return createdList; - } private async Task> CreateRandomItems(ContainerInternal container, int pkCount, int perPKItemCount = 1, bool randomPartitionKey = true) { Assert.IsFalse(!randomPartitionKey && perPKItemCount > 1); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/ToDoActivity.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/ToDoActivity.cs index 76a7136524..a19a160883 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/ToDoActivity.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/ToDoActivity.cs @@ -80,46 +80,6 @@ public static async Task> CreateRandomItems( return createdList; } - public static ToDoActivity CreateRandomToDoActivityMultiHashed( - List pks = null, - string id = null, - bool randomTaskNumber = false) - { - if(pks.Count == 0) - { - pks.Add("TBD-0-" + Guid.NewGuid().ToString()); - pks.Add("TBD-1-" + Guid.NewGuid().ToString()); - pks.Add("TBD-2-" + Guid.NewGuid().ToString()); - } - if (id == null) - { - id = Guid.NewGuid().ToString(); - } - - int taskNum = 42; - if (randomTaskNumber) - { - taskNum = Random.Shared.Next(); - } - - return new ToDoActivity() - { - id = id, - description = "CreateRandomToDoActivityMultiHashed", - pk0 = pks[0], - pk1 = pks[1], - pk2 = pks[2], - taskNum = taskNum, - cost = double.MaxValue, - CamelCase = "camelCase", - children = new ToDoActivity[] - { new ToDoActivity { id = "child1", taskNum = 30 }, - new ToDoActivity { id = "child2", taskNum = 40} - }, - valid = true, - nullableInt = null - }; - } public static ToDoActivity CreateRandomToDoActivity( string pk = null, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Routing/PartitionRoutingHelperTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Routing/PartitionRoutingHelperTest.cs index 35ffaa65bb..fdf3112523 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Routing/PartitionRoutingHelperTest.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Routing/PartitionRoutingHelperTest.cs @@ -247,6 +247,257 @@ public async Task TestGetPartitionRoutingInfo() } } + [TestMethod] + public async Task TestRoutingForPrefixedPartitionKeyQueriesAsync() + { + PartitionKeyDefinition partitionKeyDefinition = new PartitionKeyDefinition() + { + Kind = PartitionKind.MultiHash, + Paths = new Collection() { "/path1", "/path2", "/path3" }, + Version = PartitionKeyDefinitionVersion.V2 + }; + + // Case 1: Query with 1 prefix path, split at 1st level. Should route to only one partition. + { + List partitionKeyRanges = new List() + { + new PartitionKeyRange() + { + Id = "0", + MinInclusive = string.Empty, + MaxExclusive = "07E4D14180A45153F00B44907886F856" //Seattle + }, + new PartitionKeyRange() + { + Id = "1", + MinInclusive = "07E4D14180A45153F00B44907886F856", //Seattle + MaxExclusive = "FF" + }, + }; + await PrefixPartitionKeyTestRunnerAsync( + partitionKeyDefinition, + new PartitionKeyBuilder().Add("Microsoft").Build(), + ///$"SELECT VALUE r.id from r where r.path1 = \"Microsoft\"", + epkRanges => + { + return epkRanges.Count == 1; //Routes to only one pkRange. + }, + partitionKeyRanges); + } + + //Case 2: Query with 1 prefix path value which is split at 2nd level. Should route to two partitions. + //Case 3: Query with 2 prefix path values which is split at 2nd level. Should route to one partition. + { + List partitionKeyRanges = new List() + { + new PartitionKeyRange() + { + Id = "0", + MinInclusive = string.Empty, + MaxExclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A5963" //[Seattle, Redmond] + }, + new PartitionKeyRange() + { + Id = "1", + MinInclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A5963", //[Seattle, Redmond] + MaxExclusive = "FF" + }, + }; + await PrefixPartitionKeyTestRunnerAsync( + partitionKeyDefinition, + new PartitionKeyBuilder().Add("seattle").Build(), + //$"SELECT VALUE r.id from r where r.path1 = \"seattle\"", + epkRanges => + { + return epkRanges.Count == 2; //Since data is split at pkey [seattle, redmond], it should route to two pkRange. + }, + partitionKeyRanges); + await PrefixPartitionKeyTestRunnerAsync( + partitionKeyDefinition, + new PartitionKeyBuilder().Add("seattle").Add("redmond").Build(), + //$"SELECT VALUE r.id from r where r.path1 = \"seattle\" and r.path2 =\"redmond\"", + epkRanges => + { + return epkRanges.Count == 1; //Since data is split at pkey [seattle, redmond], this query should route to one pkRange + }, + partitionKeyRanges); + } + + //Case 4: Query with 2 prefix path values split at the 3rd level. Should route to 2 paritions. + //Case 5: Query with 1 prefix path value split at 3rd level. Should route to 2 partitions. + { + List partitionKeyRanges = new List() + { + new PartitionKeyRange() + { + Id = "0", + MinInclusive = string.Empty, + MaxExclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A59630EF2E2D82460884AF0F6440BE4F726A8"//[seattle, redmond, 5.12312419050912359123] + }, + new PartitionKeyRange() + { + Id = "1", + MinInclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A59630EF2E2D82460884AF0F6440BE4F726A8", //[seattle, redmond, 5.12312419050912359123] + MaxExclusive = "FF" + }, + }; + await PrefixPartitionKeyTestRunnerAsync( + partitionKeyDefinition, + new PartitionKeyBuilder().Add("seattle").Add("redmond").Build(), + //$"SELECT VALUE r.id from r where r.path1 = \"seattle\" and r.path2 =\"redmond\"", + epkRanges => + { + return epkRanges.Count == 2; //Since data is split at pkey [seattle, redmond, 5.12312419050912359123], it should route to two pkRange. + }, + partitionKeyRanges); + await PrefixPartitionKeyTestRunnerAsync( + partitionKeyDefinition, + new PartitionKeyBuilder().Add("seattle").Add("redmond").Add(5.12312419050912359123).Build(), + //$"SELECT VALUE r.id from r where r.path1 = \"seattle\" and r.path2 =\"redmond\" and r.path3 =5.12312419050912359123", + epkRanges => + { + return epkRanges.Count == 1; + }, + partitionKeyRanges); + await PrefixPartitionKeyTestRunnerAsync( + partitionKeyDefinition, + new PartitionKeyBuilder().Add("seattle").Build(), + //$"SELECT VALUE r.id from r where r.path1 = \"seattle\"", + epkRanges => + { + return epkRanges.Count == 2; + }, + partitionKeyRanges); + } + + //Case 6: Query with 1 prefix path value split succesively at 2nd and then at the 3rd level. Should route to 3 partitions. + { + List partitionKeyRanges = new List() + { + new PartitionKeyRange() + { + Id = "0", + MinInclusive = string.Empty, + MaxExclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A5963" //[seattle, redmond] + }, + new PartitionKeyRange() + { + Id = "1", + MinInclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A5963", //[seattle, redmond] + MaxExclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A59630EF2E2D82460884AF0F6440BE4F726A8" //[seattle, redmond, 5.12312419050912359123] + }, + new PartitionKeyRange() + { + Id = "2", + MinInclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A59630EF2E2D82460884AF0F6440BE4F726A8", //[seattle, redmond, 5.12312419050912359123] + MaxExclusive = "FF" + }, + }; + await PrefixPartitionKeyTestRunnerAsync( + partitionKeyDefinition, + new PartitionKeyBuilder().Add("seattle").Build(), + //$"SELECT VALUE r.id from r where r.path1 = \"seattle\"", + epkRanges => + { + return epkRanges.Count == 3; + }, + partitionKeyRanges); + } + + //Case 7: Query with 1 prefix path value split succesively at 1st, 2nd and then at the 3rd level. Should route to 3 partitions. + //Case 8: Query with 2 prefix path value split succesively at 1st, 2nd and then at the 3rd level. Should route to 2 partitions. + //Case 9: Query with fully specfied pkey, split succesively at 1st, 2nd and then at the 3rd level. Should route to 1 partitions. + { + List partitionKeyRanges = new List() + { + new PartitionKeyRange() + { + Id = "0", + MinInclusive = string.Empty, + MaxExclusive = "07E4D14180A45153F00B44907886F856" //[seattle] + }, + new PartitionKeyRange() + { + Id = "1", + MinInclusive = "07E4D14180A45153F00B44907886F856", //[seattle] + MaxExclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A5963" //[seattle, redmond] + }, + new PartitionKeyRange() + { + Id = "2", + MinInclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A5963", //[seattle, redmond] + MaxExclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A59630EF2E2D82460884AF0F6440BE4F726A8"//[seattle, redmond, 5.12312419050912359123] + }, + new PartitionKeyRange() + { + Id = "3", + MinInclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A59630EF2E2D82460884AF0F6440BE4F726A8", //[seattle, redmond, 5.12312419050912359123] + MaxExclusive = "FF" + }, + }; + await PrefixPartitionKeyTestRunnerAsync( + partitionKeyDefinition, + new PartitionKeyBuilder().Add("seattle").Build(), + //$"SELECT VALUE r.id from r where r.path1 = \"seattle\"", + epkRanges => + { + return epkRanges.Count == 3; //Routes tp three pkRanges + }, + partitionKeyRanges); + await PrefixPartitionKeyTestRunnerAsync( + partitionKeyDefinition, + new PartitionKeyBuilder().Add("seattle").Add("redmond").Build(), + //$"SELECT VALUE r.id from r where r.path1 = \"seattle\" and r.path2 = \"redmond\"", + epkRanges => + { + return epkRanges.Count == 2; //Routes to two pkRanges. + }, + partitionKeyRanges); + await PrefixPartitionKeyTestRunnerAsync( + partitionKeyDefinition, + new PartitionKeyBuilder().Add("seattle").Add("redmond").Add("98052").Build(), + //$"SELECT VALUE r.id from r where r.path1 = \"seattle\" and r.path2 = \"redmond\" and r.path3 = \"98052\"", + epkRanges => + { + Assert.AreEqual(epkRanges.Count, 1); + + return epkRanges.Count == 1; //Routes to only one pkRanges. + }, + partitionKeyRanges); + } + } + + private static async Task PrefixPartitionKeyTestRunnerAsync( + PartitionKeyDefinition partitionKeyDefinition, + Cosmos.PartitionKey partitionKey, + Predicate> validator, + List partitionKeyRanges) + { + IEnumerable> rangesAndServiceIdentity = partitionKeyRanges + .Select(range => Tuple.Create(range, (ServiceIdentity)null)); + string collectionRid = string.Empty; + CollectionRoutingMap routingMap = + CollectionRoutingMap.TryCreateCompleteRoutingMap( + rangesAndServiceIdentity, + collectionRid); + + RoutingMapProvider routingMapProvider = new RoutingMapProvider(routingMap); + + HashSet resolvedPKRanges = new HashSet(); + FeedRangePartitionKey feedRangePartitionKey = new FeedRangePartitionKey(partitionKey); + List> effectiveRanges = await feedRangePartitionKey.GetEffectiveRangesAsync(routingMapProvider, null, partitionKeyDefinition, null); + foreach (Range range in effectiveRanges) + { + resolvedPKRanges.UnionWith(await routingMapProvider.TryGetOverlappingRangesAsync( + collectionRid, + range, + NoOpTrace.Singleton)); + + } + + Assert.IsTrue(validator(resolvedPKRanges)); + } + [TestMethod] public void TestCrossPartitionAggregateQueries() { From afa3228c0aec66e15098795051622fdc7b941d2a Mon Sep 17 00:00:00 2001 From: nalutripician Date: Tue, 30 May 2023 17:08:04 -0400 Subject: [PATCH 3/5] fix for change feed and query plus tests --- .../FeedRanges/FeedRangePartitionKey.cs | 39 ++++---------- .../NetworkAttachedDocumentContainer.cs | 41 -------------- .../CosmosQueryExecutionContextFactory.cs | 54 +------------------ .../CosmosMultiHashTest.cs | 2 +- 4 files changed, 13 insertions(+), 123 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/FeedRange/FeedRanges/FeedRangePartitionKey.cs b/Microsoft.Azure.Cosmos/src/FeedRange/FeedRanges/FeedRangePartitionKey.cs index 1c643bc4fb..2534f7e371 100644 --- a/Microsoft.Azure.Cosmos/src/FeedRange/FeedRanges/FeedRangePartitionKey.cs +++ b/Microsoft.Azure.Cosmos/src/FeedRange/FeedRanges/FeedRangePartitionKey.cs @@ -24,40 +24,23 @@ public FeedRangePartitionKey(PartitionKey partitionKey) this.PartitionKey = partitionKey; } - internal override async Task>> GetEffectiveRangesAsync( + internal override Task>> GetEffectiveRangesAsync( IRoutingMapProvider routingMapProvider, string containerRid, Documents.PartitionKeyDefinition partitionKeyDefinition, ITrace trace) { - if (partitionKeyDefinition.Kind == PartitionKind.MultiHash) - { - Documents.Routing.Range range = this.PartitionKey.InternalKey.GetEPKRangeForPrefixPartitionKey(partitionKeyDefinition); - IReadOnlyList overlappingRanges = await routingMapProvider.TryGetOverlappingRangesAsync( - containerRid, - range, - trace, - forceRefresh: false); - List> effectiveRanges = new List>(); - - foreach (PartitionKeyRange overlappingRange in overlappingRanges) + return Task.FromResult( + new List> { - effectiveRanges.Add(new Documents.Routing.Range(overlappingRange.MinInclusive, overlappingRange.MaxExclusive, true, false)); - } - - return effectiveRanges; - } - - return new List> - { - Documents.Routing.Range.GetPointRange( - this.PartitionKey.InternalKey.GetEffectivePartitionKeyString(partitionKeyDefinition)) - }; - - //return new List> - //{ - // PartitionKey.InternalKey.GetEffectivePartitionKeyRange(partitionKeyDefinition, new Documents.Routing.Range(this.PartitionKey.InternalKey, this.PartitionKey.InternalKey)) - //}; + Documents.Routing.PartitionKeyInternal.GetEffectivePartitionKeyRange( + partitionKeyDefinition, + new Documents.Routing.Range( + this.PartitionKey.InternalKey, + this.PartitionKey.InternalKey, + true, + true)) + }); } internal override async Task> GetPartitionKeyRangesAsync( diff --git a/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs b/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs index 380d43da49..6d89fe135e 100644 --- a/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs +++ b/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs @@ -128,47 +128,6 @@ await this.container.GetCachedRIDAsync(forceRefresh: false, trace, cancellationT forceRefresh: false, trace); - if (containerProperties.PartitionKey.Kind == PartitionKind.MultiHash && feedRange.GetType() == typeof(FeedRangePartitionKey)) - { - FeedRangePartitionKey frpk = (FeedRangePartitionKey)feedRange; - Documents.Routing.Range range = frpk.PartitionKey.InternalKey.GetEPKRangeForPrefixPartitionKey(containerProperties.PartitionKey); - - List feedRanges = new List(); - int count = 0; - foreach (PartitionKeyRange partitionKeyRange in overlappingRanges) - { - //First and last range must incude the min and max value for the original EPK range - if (count == 0) - { - feedRanges.Add(new FeedRangeEpk( - new Documents.Routing.Range( - min: range.Min, - max: partitionKeyRange.MaxExclusive, - isMinInclusive: true, - isMaxInclusive: false))); - } - else if (count == overlappingRanges.Count - 1) - { - feedRanges.Add(new FeedRangeEpk( - new Documents.Routing.Range( - min: partitionKeyRange.MinInclusive, - max: range.Max, - isMinInclusive: true, - isMaxInclusive: false))); - } - else - { - feedRanges.Add(new FeedRangeEpk( - new Documents.Routing.Range( - min: partitionKeyRange.MinInclusive, - max: partitionKeyRange.MaxExclusive, - isMinInclusive: true, - isMaxInclusive: false))); - } - count++; - } - return TryCatch>.FromResult(feedRanges); - } return TryCatch>.FromResult( overlappingRanges.Select(range => new FeedRangeEpk( new Documents.Routing.Range( diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs index 5fb4341123..c1655439ef 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs @@ -311,63 +311,11 @@ private static async Task> TryCreateFromPartitione { SetTestInjectionPipelineType(inputParameters, Passthrough); - if (containerQueryProperties.PartitionKeyDefinition.Kind == Documents.PartitionKind.MultiHash - && inputParameters.InitialFeedRange.GetType() == typeof(FeedRangePartitionKey)) - { - FeedRangePartitionKey frpk = (FeedRangePartitionKey)inputParameters.InitialFeedRange; - Documents.Routing.Range range = frpk.PartitionKey.InternalKey.GetEPKRangeForPrefixPartitionKey(containerQueryProperties.PartitionKeyDefinition); - - List feedRanges = new List(); - int count = 0; - foreach (Documents.PartitionKeyRange partitionKeyRange in targetRanges) - { - if (count == 0) - { - feedRanges.Add(new FeedRangeEpk( - new Documents.Routing.Range( - min: range.Min, - max: partitionKeyRange.MaxExclusive, - isMinInclusive: true, - isMaxInclusive: false))); - } - else if (count == targetRanges.Count - 1) - { - feedRanges.Add(new FeedRangeEpk( - new Documents.Routing.Range( - min: partitionKeyRange.MinInclusive, - max: range.Max, - isMinInclusive: true, - isMaxInclusive: false))); - } - else - { - feedRanges.Add(new FeedRangeEpk( - new Documents.Routing.Range( - min: partitionKeyRange.MinInclusive, - max: partitionKeyRange.MaxExclusive, - isMinInclusive: true, - isMaxInclusive: false))); - } - - count++; - } - - tryCreatePipelineStage = CosmosQueryExecutionContextFactory.TryCreatePassthroughQueryExecutionContext( - documentContainer, - inputParameters, - feedRanges, - cancellationToken); - - } - else - { - tryCreatePipelineStage = CosmosQueryExecutionContextFactory.TryCreatePassthroughQueryExecutionContext( + tryCreatePipelineStage = CosmosQueryExecutionContextFactory.TryCreatePassthroughQueryExecutionContext( documentContainer, inputParameters, targetRanges, cancellationToken); - } - } else { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosMultiHashTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosMultiHashTest.cs index dbdc326ea7..5307145dfc 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosMultiHashTest.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosMultiHashTest.cs @@ -33,7 +33,7 @@ public async Task Cleanup() { await this.database.DeleteAsync(); this.client.Dispose(); - } + } [TestMethod] public async Task MultiHashCreateDocumentTest() From 4eb9c14c6ec6fc046ba8fb128707ce2827cd0b84 Mon Sep 17 00:00:00 2001 From: nalutripician Date: Tue, 30 May 2023 17:13:48 -0400 Subject: [PATCH 4/5] clean up --- .../NetworkAttachedDocumentContainer.cs | 2 -- .../CosmosQueryExecutionContextFactory.cs | 20 ------------------- .../FeedToken/ChangeFeedIteratorCoreTests.cs | 2 +- .../Utils/ToDoActivity.cs | 4 ---- 4 files changed, 1 insertion(+), 27 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs b/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs index 6d89fe135e..1132a40a35 100644 --- a/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs +++ b/Microsoft.Azure.Cosmos/src/Pagination/NetworkAttachedDocumentContainer.cs @@ -14,7 +14,6 @@ namespace Microsoft.Azure.Cosmos.Pagination using System.Text; using System.Threading; using System.Threading.Tasks; - using global::Azure; using Microsoft.Azure.Cosmos.ChangeFeed.Pagination; using Microsoft.Azure.Cosmos.CosmosElements; using Microsoft.Azure.Cosmos.Diagnostics; @@ -127,7 +126,6 @@ await this.container.GetCachedRIDAsync(forceRefresh: false, trace, cancellationT feedRange, forceRefresh: false, trace); - return TryCatch>.FromResult( overlappingRanges.Select(range => new FeedRangeEpk( new Documents.Routing.Range( diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs index c1655439ef..e276466f9d 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs @@ -522,26 +522,6 @@ private static TryCatch TryCreatePassthroughQueryExecutionC continuationToken: inputParameters.InitialUserContinuationToken); } - private static TryCatch TryCreatePassthroughQueryExecutionContext( - DocumentContainer documentContainer, - InputParameters inputParameters, - List targetRanges, - CancellationToken cancellationToken) - { - // Return a parallel context, since we still want to be able to handle splits and concurrency / buffering. - return ParallelCrossPartitionQueryPipelineStage.MonadicCreate( - documentContainer: documentContainer, - sqlQuerySpec: inputParameters.SqlQuerySpec, - targetRanges: targetRanges, - queryPaginationOptions: new QueryPaginationOptions( - pageSizeHint: inputParameters.MaxItemCount), - partitionKey: inputParameters.PartitionKey, - prefetchPolicy: PrefetchPolicy.PrefetchSinglePage, - maxConcurrency: inputParameters.MaxConcurrency, - cancellationToken: cancellationToken, - continuationToken: inputParameters.InitialUserContinuationToken); - } - private static TryCatch TryCreateSpecializedDocumentQueryExecutionContext( DocumentContainer documentContainer, CosmosQueryContext cosmosQueryContext, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs index 7cbe16aa10..4a1d92c59c 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs @@ -73,7 +73,7 @@ private async Task InitializeFFCFContainerAsync(TimeSpan time return (ContainerInternal)response; } - + /// /// Test to verify that StartFromBeginning works as expected by inserting 25 items, reading them all, then taking the last continuationtoken, /// inserting another 25, and verifying that the iterator continues from the saved token and reads the second 25 for a total of 50 documents. diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/ToDoActivity.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/ToDoActivity.cs index a19a160883..c2800c02d2 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/ToDoActivity.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Utils/ToDoActivity.cs @@ -16,10 +16,6 @@ public class ToDoActivity public double cost { get; set; } public string description { get; set; } public string pk { get; set; } - - public string pk0 { get; set; } - public string pk1 { get; set; } - public string pk2 { get; set; } public string CamelCase { get; set; } public int? nullableInt { get; set; } From cc823222cc9b016395bbc7b70f8250ec3366644e Mon Sep 17 00:00:00 2001 From: nalutripician Date: Thu, 8 Jun 2023 09:59:10 -0700 Subject: [PATCH 5/5] query + clean up --- .../FeedRanges/FeedRangePartitionKey.cs | 8 +- .../Core/QueryPlan/QueryPartitionProvider.cs | 6 +- .../Routing/PartitionRoutingHelperTest.cs | 318 ++++++++++++++++-- 3 files changed, 297 insertions(+), 35 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/FeedRange/FeedRanges/FeedRangePartitionKey.cs b/Microsoft.Azure.Cosmos/src/FeedRange/FeedRanges/FeedRangePartitionKey.cs index 2534f7e371..1a04800b50 100644 --- a/Microsoft.Azure.Cosmos/src/FeedRange/FeedRanges/FeedRangePartitionKey.cs +++ b/Microsoft.Azure.Cosmos/src/FeedRange/FeedRanges/FeedRangePartitionKey.cs @@ -36,10 +36,10 @@ public FeedRangePartitionKey(PartitionKey partitionKey) Documents.Routing.PartitionKeyInternal.GetEffectivePartitionKeyRange( partitionKeyDefinition, new Documents.Routing.Range( - this.PartitionKey.InternalKey, - this.PartitionKey.InternalKey, - true, - true)) + min: this.PartitionKey.InternalKey, + max: this.PartitionKey.InternalKey, + isMinInclusive: true, + isMaxInclusive: true)) }); } diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/QueryPlan/QueryPartitionProvider.cs b/Microsoft.Azure.Cosmos/src/Query/Core/QueryPlan/QueryPartitionProvider.cs index 6e7b0b4a8f..c62a65b27a 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/QueryPlan/QueryPartitionProvider.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/QueryPlan/QueryPartitionProvider.cs @@ -147,11 +147,7 @@ internal PartitionedQueryExecutionInfo ConvertPartitionedQueryExecutionInfo( List> effectiveRanges = new List>(queryInfoInternal.QueryRanges.Count); foreach (Documents.Routing.Range internalRange in queryInfoInternal.QueryRanges) { - effectiveRanges.Add(new Documents.Routing.Range( - internalRange.Min.GetEffectivePartitionKeyString(partitionKeyDefinition, false), - internalRange.Max.GetEffectivePartitionKeyString(partitionKeyDefinition, false), - internalRange.IsMinInclusive, - internalRange.IsMaxInclusive)); + effectiveRanges.Add(PartitionKeyInternal.GetEffectivePartitionKeyRange(partitionKeyDefinition, internalRange)); } effectiveRanges.Sort(Documents.Routing.Range.MinComparer.Instance); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Routing/PartitionRoutingHelperTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Routing/PartitionRoutingHelperTest.cs index fdf3112523..792621f54a 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Routing/PartitionRoutingHelperTest.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Routing/PartitionRoutingHelperTest.cs @@ -19,6 +19,8 @@ namespace Microsoft.Azure.Cosmos.Tests.Routing using Microsoft.Azure.Cosmos.Query.Core.QueryPlan; using System.Collections.ObjectModel; using System.Net; + using Microsoft.Azure.Cosmos.Query.Core; + using Microsoft.Azure.Cosmos.Query.Core.Monads; /// /// Tests for class. @@ -246,7 +248,6 @@ public async Task TestGetPartitionRoutingInfo() } } } - [TestMethod] public async Task TestRoutingForPrefixedPartitionKeyQueriesAsync() { @@ -276,8 +277,7 @@ public async Task TestRoutingForPrefixedPartitionKeyQueriesAsync() }; await PrefixPartitionKeyTestRunnerAsync( partitionKeyDefinition, - new PartitionKeyBuilder().Add("Microsoft").Build(), - ///$"SELECT VALUE r.id from r where r.path1 = \"Microsoft\"", + $"SELECT VALUE r.id from r where r.path1 = \"Microsoft\"", epkRanges => { return epkRanges.Count == 1; //Routes to only one pkRange. @@ -299,14 +299,13 @@ await PrefixPartitionKeyTestRunnerAsync( new PartitionKeyRange() { Id = "1", - MinInclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A5963", //[Seattle, Redmond] + MinInclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A5963",//[Seattle, Redmond] MaxExclusive = "FF" }, }; await PrefixPartitionKeyTestRunnerAsync( partitionKeyDefinition, - new PartitionKeyBuilder().Add("seattle").Build(), - //$"SELECT VALUE r.id from r where r.path1 = \"seattle\"", + $"SELECT VALUE r.id from r where r.path1 = \"seattle\" or (r.path1 = \"seattle\" and r.path2 = \"bellevue\")", epkRanges => { return epkRanges.Count == 2; //Since data is split at pkey [seattle, redmond], it should route to two pkRange. @@ -314,8 +313,7 @@ await PrefixPartitionKeyTestRunnerAsync( partitionKeyRanges); await PrefixPartitionKeyTestRunnerAsync( partitionKeyDefinition, - new PartitionKeyBuilder().Add("seattle").Add("redmond").Build(), - //$"SELECT VALUE r.id from r where r.path1 = \"seattle\" and r.path2 =\"redmond\"", + $"SELECT VALUE r.id from r where r.path1 = \"seattle\" and r.path2 =\"redmond\"", epkRanges => { return epkRanges.Count == 1; //Since data is split at pkey [seattle, redmond], this query should route to one pkRange @@ -337,14 +335,13 @@ await PrefixPartitionKeyTestRunnerAsync( new PartitionKeyRange() { Id = "1", - MinInclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A59630EF2E2D82460884AF0F6440BE4F726A8", //[seattle, redmond, 5.12312419050912359123] + MinInclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A59630EF2E2D82460884AF0F6440BE4F726A8",//[seattle, redmond, 5.12312419050912359123] MaxExclusive = "FF" }, }; await PrefixPartitionKeyTestRunnerAsync( partitionKeyDefinition, - new PartitionKeyBuilder().Add("seattle").Add("redmond").Build(), - //$"SELECT VALUE r.id from r where r.path1 = \"seattle\" and r.path2 =\"redmond\"", + $"SELECT VALUE r.id from r where r.path1 = \"seattle\" and r.path2 =\"redmond\"", epkRanges => { return epkRanges.Count == 2; //Since data is split at pkey [seattle, redmond, 5.12312419050912359123], it should route to two pkRange. @@ -352,8 +349,7 @@ await PrefixPartitionKeyTestRunnerAsync( partitionKeyRanges); await PrefixPartitionKeyTestRunnerAsync( partitionKeyDefinition, - new PartitionKeyBuilder().Add("seattle").Add("redmond").Add(5.12312419050912359123).Build(), - //$"SELECT VALUE r.id from r where r.path1 = \"seattle\" and r.path2 =\"redmond\" and r.path3 =5.12312419050912359123", + $"SELECT VALUE r.id from r where r.path1 = \"seattle\" and r.path2 =\"redmond\" and r.path3=5.12312419050912359123", epkRanges => { return epkRanges.Count == 1; @@ -361,8 +357,7 @@ await PrefixPartitionKeyTestRunnerAsync( partitionKeyRanges); await PrefixPartitionKeyTestRunnerAsync( partitionKeyDefinition, - new PartitionKeyBuilder().Add("seattle").Build(), - //$"SELECT VALUE r.id from r where r.path1 = \"seattle\"", + $"SELECT VALUE r.id from r where r.path1 = \"seattle\"", epkRanges => { return epkRanges.Count == 2; @@ -371,6 +366,216 @@ await PrefixPartitionKeyTestRunnerAsync( } //Case 6: Query with 1 prefix path value split succesively at 2nd and then at the 3rd level. Should route to 3 partitions. + { + List partitionKeyRanges = new List() + { + new PartitionKeyRange() + { + Id = "0", + MinInclusive = string.Empty, + MaxExclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A5963"//[seattle, redmond] + }, + new PartitionKeyRange() + { + Id = "1", + MinInclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A5963",//[seattle, redmond] + MaxExclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A59630EF2E2D82460884AF0F6440BE4F726A8"//[seattle, redmond, 5.12312419050912359123] + }, + new PartitionKeyRange() + { + Id = "2", + MinInclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A59630EF2E2D82460884AF0F6440BE4F726A8",//[seattle, redmond, 5.12312419050912359123] + MaxExclusive = "FF" + }, + }; + await PrefixPartitionKeyTestRunnerAsync( + partitionKeyDefinition, + $"SELECT VALUE r.id from r where r.path1 = \"seattle\"", + epkRanges => + { + return epkRanges.Count == 3; + }, + partitionKeyRanges); + } + + //Case 7: Query with 1 prefix path value split succesively at 1st, 2nd and then at the 3rd level. Should route to 3 partitions. + //Case 8: Query with 2 prefix path value split succesively at 1st, 2nd and then at the 3rd level. Should route to 2 partitions. + //Case 9: Query with fully specfied pkey, split succesively at 1st, 2nd and then at the 3rd level. Should route to 1 partitions. + { + List partitionKeyRanges = new List() + { + new PartitionKeyRange() + { + Id = "0", + MinInclusive = string.Empty, + MaxExclusive = "07E4D14180A45153F00B44907886F856" //[seattle] + }, + new PartitionKeyRange() + { + Id = "1", + MinInclusive = "07E4D14180A45153F00B44907886F856", //[seattle] + MaxExclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A5963" //[seattle, redmond] + }, + new PartitionKeyRange() + { + Id = "2", + MinInclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A5963", //[seattle, redmond] + MaxExclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A59630EF2E2D82460884AF0F6440BE4F726A8"//[seattle, redmond, 5.12312419050912359123] + }, + new PartitionKeyRange() + { + Id = "3", + MinInclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A59630EF2E2D82460884AF0F6440BE4F726A8",//[seattle, redmond, 5.12312419050912359123] + MaxExclusive = "FF" + }, + }; + await PrefixPartitionKeyTestRunnerAsync( + partitionKeyDefinition, + $"SELECT VALUE r.id from r where r.path1 = \"seattle\"", + epkRanges => + { + return epkRanges.Count == 3; //Routes tp three pkRanges + }, + partitionKeyRanges); + await PrefixPartitionKeyTestRunnerAsync( + partitionKeyDefinition, + $"SELECT VALUE r.id from r where r.path1 = \"seattle\" and r.path2 = \"redmond\"", + epkRanges => + { + return epkRanges.Count == 2; //Routes to two pkRanges. + }, + partitionKeyRanges); + await PrefixPartitionKeyTestRunnerAsync( + partitionKeyDefinition, + $"SELECT VALUE r.id from r where r.path1 = \"seattle\" and r.path2 = \"redmond\" and r.path3 = \"98052\"", + epkRanges => + { + Assert.AreEqual(epkRanges.Count, 1); + + return epkRanges.Count == 1; //Routes to only one pkRanges. + }, + partitionKeyRanges); + } + } + + [TestMethod] + public async Task TestRoutingForPrefixedPartitionKeyChangeFeedAsync() + { + PartitionKeyDefinition partitionKeyDefinition = new PartitionKeyDefinition() + { + Kind = PartitionKind.MultiHash, + Paths = new Collection() { "/path1", "/path2", "/path3" }, + Version = PartitionKeyDefinitionVersion.V2 + }; + + // Case 1: ChangeFeed with 1 prefix path, split at 1st level. Should route to only one partition. + { + List partitionKeyRanges = new List() + { + new PartitionKeyRange() + { + Id = "0", + MinInclusive = string.Empty, + MaxExclusive = "07E4D14180A45153F00B44907886F856" //Seattle + }, + new PartitionKeyRange() + { + Id = "1", + MinInclusive = "07E4D14180A45153F00B44907886F856", //Seattle + MaxExclusive = "FF" + }, + }; + await PrefixPartitionKeyChangeFeedTestRunnerAsync( + partitionKeyDefinition, + new PartitionKeyBuilder().Add("Microsoft").Build(), + epkRanges => + { + return epkRanges.Count == 1; //Routes to only one pkRange. + }, + partitionKeyRanges); + } + + //Case 2: ChangeFeed with 1 prefix path value which is split at 2nd level. Should route to two partitions. + //Case 3: ChangeFeed with 2 prefix path values which is split at 2nd level. Should route to one partition. + { + List partitionKeyRanges = new List() + { + new PartitionKeyRange() + { + Id = "0", + MinInclusive = string.Empty, + MaxExclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A5963" //[Seattle, Redmond] + }, + new PartitionKeyRange() + { + Id = "1", + MinInclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A5963", //[Seattle, Redmond] + MaxExclusive = "FF" + }, + }; + await PrefixPartitionKeyChangeFeedTestRunnerAsync( + partitionKeyDefinition, + new PartitionKeyBuilder().Add("seattle").Build(), + epkRanges => + { + return epkRanges.Count == 2; //Since data is split at pkey [seattle, redmond], it should route to two pkRange. + }, + partitionKeyRanges); + await PrefixPartitionKeyChangeFeedTestRunnerAsync( + partitionKeyDefinition, + new PartitionKeyBuilder().Add("seattle").Add("redmond").Build(), + epkRanges => + { + return epkRanges.Count == 1; //Since data is split at pkey [seattle, redmond], this query should route to one pkRange + }, + partitionKeyRanges); + } + + //Case 4: ChangeFeed with 2 prefix path values split at the 3rd level. Should route to 2 paritions. + //Case 5: ChangeFeed with 1 prefix path value split at 3rd level. Should route to 2 partitions. + { + List partitionKeyRanges = new List() + { + new PartitionKeyRange() + { + Id = "0", + MinInclusive = string.Empty, + MaxExclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A59630EF2E2D82460884AF0F6440BE4F726A8"//[seattle, redmond, 5.12312419050912359123] + }, + new PartitionKeyRange() + { + Id = "1", + MinInclusive = "07E4D14180A45153F00B44907886F85622E342F38A486A088463DFF7838A59630EF2E2D82460884AF0F6440BE4F726A8", //[seattle, redmond, 5.12312419050912359123] + MaxExclusive = "FF" + }, + }; + await PrefixPartitionKeyChangeFeedTestRunnerAsync( + partitionKeyDefinition, + new PartitionKeyBuilder().Add("seattle").Add("redmond").Build(), + epkRanges => + { + return epkRanges.Count == 2; //Since data is split at pkey [seattle, redmond, 5.12312419050912359123], it should route to two pkRange. + }, + partitionKeyRanges); + await PrefixPartitionKeyChangeFeedTestRunnerAsync( + partitionKeyDefinition, + new PartitionKeyBuilder().Add("seattle").Add("redmond").Add(5.12312419050912359123).Build(), + epkRanges => + { + return epkRanges.Count == 1; + }, + partitionKeyRanges); + await PrefixPartitionKeyChangeFeedTestRunnerAsync( + partitionKeyDefinition, + new PartitionKeyBuilder().Add("seattle").Build(), + epkRanges => + { + return epkRanges.Count == 2; + }, + partitionKeyRanges); + } + + //Case 6: ChangeFeed with 1 prefix path value split succesively at 2nd and then at the 3rd level. Should route to 3 partitions. { List partitionKeyRanges = new List() { @@ -393,10 +598,9 @@ await PrefixPartitionKeyTestRunnerAsync( MaxExclusive = "FF" }, }; - await PrefixPartitionKeyTestRunnerAsync( + await PrefixPartitionKeyChangeFeedTestRunnerAsync( partitionKeyDefinition, new PartitionKeyBuilder().Add("seattle").Build(), - //$"SELECT VALUE r.id from r where r.path1 = \"seattle\"", epkRanges => { return epkRanges.Count == 3; @@ -404,9 +608,9 @@ await PrefixPartitionKeyTestRunnerAsync( partitionKeyRanges); } - //Case 7: Query with 1 prefix path value split succesively at 1st, 2nd and then at the 3rd level. Should route to 3 partitions. - //Case 8: Query with 2 prefix path value split succesively at 1st, 2nd and then at the 3rd level. Should route to 2 partitions. - //Case 9: Query with fully specfied pkey, split succesively at 1st, 2nd and then at the 3rd level. Should route to 1 partitions. + //Case 7: ChangeFeed with 1 prefix path value split succesively at 1st, 2nd and then at the 3rd level. Should route to 3 partitions. + //Case 8: ChangeFeed with 2 prefix path value split succesively at 1st, 2nd and then at the 3rd level. Should route to 2 partitions. + //Case 9: ChangeFeed with fully specfied pkey, split succesively at 1st, 2nd and then at the 3rd level. Should route to 1 partitions. { List partitionKeyRanges = new List() { @@ -435,28 +639,25 @@ await PrefixPartitionKeyTestRunnerAsync( MaxExclusive = "FF" }, }; - await PrefixPartitionKeyTestRunnerAsync( + await PrefixPartitionKeyChangeFeedTestRunnerAsync( partitionKeyDefinition, new PartitionKeyBuilder().Add("seattle").Build(), - //$"SELECT VALUE r.id from r where r.path1 = \"seattle\"", epkRanges => { return epkRanges.Count == 3; //Routes tp three pkRanges }, partitionKeyRanges); - await PrefixPartitionKeyTestRunnerAsync( + await PrefixPartitionKeyChangeFeedTestRunnerAsync( partitionKeyDefinition, new PartitionKeyBuilder().Add("seattle").Add("redmond").Build(), - //$"SELECT VALUE r.id from r where r.path1 = \"seattle\" and r.path2 = \"redmond\"", epkRanges => { return epkRanges.Count == 2; //Routes to two pkRanges. }, partitionKeyRanges); - await PrefixPartitionKeyTestRunnerAsync( + await PrefixPartitionKeyChangeFeedTestRunnerAsync( partitionKeyDefinition, new PartitionKeyBuilder().Add("seattle").Add("redmond").Add("98052").Build(), - //$"SELECT VALUE r.id from r where r.path1 = \"seattle\" and r.path2 = \"redmond\" and r.path3 = \"98052\"", epkRanges => { Assert.AreEqual(epkRanges.Count, 1); @@ -468,6 +669,71 @@ await PrefixPartitionKeyTestRunnerAsync( } private static async Task PrefixPartitionKeyTestRunnerAsync( + PartitionKeyDefinition partitionKeyDefinition, + string queryText, + Predicate> validator, + List partitionKeyRanges) + { + IDictionary DefaultQueryengineConfiguration = new Dictionary() + { + {"maxSqlQueryInputLength", 30720}, + {"maxJoinsPerSqlQuery", 5}, + {"maxLogicalAndPerSqlQuery", 200}, + {"maxLogicalOrPerSqlQuery", 200}, + {"maxUdfRefPerSqlQuery", 2}, + {"maxInExpressionItemsCount", 8000}, + {"queryMaxInMemorySortDocumentCount", 500}, + {"maxQueryRequestTimeoutFraction", 0.90}, + {"sqlAllowNonFiniteNumbers", false}, + {"sqlAllowAggregateFunctions", true}, + {"sqlAllowSubQuery", true}, + {"sqlAllowScalarSubQuery", false}, + {"allowNewKeywords", true}, + {"sqlAllowLike", true}, + {"sqlAllowGroupByClause", false}, + {"queryEnableMongoNativeRegex", true}, + {"maxSpatialQueryCells", 12}, + {"spatialMaxGeometryPointCount", 256}, + {"sqlDisableOptimizationFlags", 0}, + {"sqlEnableParameterExpansionCheck", true} + }; + + QueryPartitionProvider QueryPartitionProvider = new QueryPartitionProvider(DefaultQueryengineConfiguration); + + IEnumerable> rangesAndServiceIdentity = partitionKeyRanges + .Select(range => Tuple.Create(range, (ServiceIdentity)null)); + string collectionRid = string.Empty; + CollectionRoutingMap routingMap = + CollectionRoutingMap.TryCreateCompleteRoutingMap( + rangesAndServiceIdentity, + collectionRid); + + RoutingMapProvider routingMapProvider = new RoutingMapProvider(routingMap); + TryCatch tryGetQueryPlan = + QueryPartitionProvider.TryGetPartitionedQueryExecutionInfo( + querySpecJsonString: JsonConvert.SerializeObject(new SqlQuerySpec(queryText)), + partitionKeyDefinition: partitionKeyDefinition, + requireFormattableOrderByQuery: true, + isContinuationExpected: true, + allowNonValueAggregateQuery: false, + hasLogicalPartitionKey: false, + allowDCount: true, + useSystemPrefix: false, + geospatialType: Cosmos.GeospatialType.Geography); + + HashSet resolvedPKRanges = new HashSet(); + foreach (Range range in tryGetQueryPlan.Result.QueryRanges) + { + resolvedPKRanges.UnionWith(await routingMapProvider.TryGetOverlappingRangesAsync( + collectionRid, + range, + NoOpTrace.Singleton)); + } + + Assert.IsTrue(validator(resolvedPKRanges)); + } + + private static async Task PrefixPartitionKeyChangeFeedTestRunnerAsync( PartitionKeyDefinition partitionKeyDefinition, Cosmos.PartitionKey partitionKey, Predicate> validator,