From 4044aa9b48c0afb8606fd7a038ab05d1c9413866 Mon Sep 17 00:00:00 2001 From: Brandon Chong Date: Thu, 12 Mar 2020 10:06:49 -0700 Subject: [PATCH 1/3] Query Test Refactor (#1271) * broke up tests into subclasses * renamed files for shorter names * renamed * wired through CosmosElement * broke up some tests * fixed filename --- .../src/CosmosElements/CosmosElement.cs | 11 + .../CosmosObject.LazyCosmosObject.cs | 23 +- .../CosmosPermissionTests.cs | 5 +- .../CrossPartitionQueryTests.cs | 5091 ----------------- ...icrosoft.Azure.Cosmos.EmulatorTests.csproj | 6 +- ...eryTests.AggregateMixedTypes_baseline.xml} | 0 .../Query/AggregateQueryTests.cs | 877 +++ .../Query/CosmosElementTestExtensions.cs | 18 + .../Query/DistinctQueryTests.cs | 285 + .../Query/GroupByQueryTests.cs | 393 ++ .../Query/MockCosmosQueryClient.cs | 91 + .../Query/NegativeQueryTests.cs | 81 + .../Query/OrderByQueryTests.cs | 1195 ++++ .../Query/PartitioningQueryTests.cs | 497 ++ .../Query/PersonGenerator.cs | 182 + .../Query/QueryTestsBase.cs | 837 +++ .../Query/SanityQueryTests.cs | 434 ++ .../Query/SkipTakeQueryTests.cs | 186 + .../Utils/QueryOracle.cs | 40 +- .../Utils/QueryOracleUtil.cs | 288 +- 20 files changed, 5258 insertions(+), 5282 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.cs rename Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/{CrossPartitionQueryTests.AggregateMixedTypes_baseline.xml => Query/AggregateQueryTests.AggregateMixedTypes_baseline.xml} (100%) create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/AggregateQueryTests.cs create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/CosmosElementTestExtensions.cs create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/DistinctQueryTests.cs create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/GroupByQueryTests.cs create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/MockCosmosQueryClient.cs create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/NegativeQueryTests.cs create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/OrderByQueryTests.cs create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/PartitioningQueryTests.cs create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/PersonGenerator.cs create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/QueryTestsBase.cs create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SanityQueryTests.cs create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/SkipTakeQueryTests.cs diff --git a/Microsoft.Azure.Cosmos/src/CosmosElements/CosmosElement.cs b/Microsoft.Azure.Cosmos/src/CosmosElements/CosmosElement.cs index 07bf9b2b4f..eb0d3eca47 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosElements/CosmosElement.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosElements/CosmosElement.cs @@ -224,6 +224,17 @@ public static CosmosElement Parse(string json) return cosmosElement; } + + public static TCosmosElement Parse(string json) + where TCosmosElement : CosmosElement + { + if (!CosmosElement.TryParse(json, out TCosmosElement cosmosElement)) + { + throw new ArgumentException($"Failed to parse json: {json}."); + } + + return cosmosElement; + } } #if INTERNAL #pragma warning restore SA1600 // Elements should be documented diff --git a/Microsoft.Azure.Cosmos/src/CosmosElements/CosmosObject.LazyCosmosObject.cs b/Microsoft.Azure.Cosmos/src/CosmosElements/CosmosObject.LazyCosmosObject.cs index 93fd2e1b26..8d3c30ae94 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosElements/CosmosObject.LazyCosmosObject.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosElements/CosmosObject.LazyCosmosObject.cs @@ -4,6 +4,7 @@ namespace Microsoft.Azure.Cosmos.CosmosElements { using System; + using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using Microsoft.Azure.Cosmos.Json; @@ -21,7 +22,7 @@ private class LazyCosmosObject : CosmosObject { private readonly IJsonNavigator jsonNavigator; private readonly IJsonNavigatorNode jsonNavigatorNode; - private readonly Dictionary cachedElements; + private readonly ConcurrentDictionary cachedElements; private readonly Lazy lazyCount; public LazyCosmosObject(IJsonNavigator jsonNavigator, IJsonNavigatorNode jsonNavigatorNode) @@ -44,7 +45,7 @@ public LazyCosmosObject(IJsonNavigator jsonNavigator, IJsonNavigatorNode jsonNav this.jsonNavigator = jsonNavigator; this.jsonNavigatorNode = jsonNavigatorNode; - this.cachedElements = new Dictionary(); + this.cachedElements = new ConcurrentDictionary(); this.lazyCount = new Lazy(() => this.jsonNavigator.GetObjectPropertyCount(this.jsonNavigatorNode)); } @@ -92,31 +93,27 @@ public override IEnumerator> GetEnumerator() public override bool TryGetValue(string key, out CosmosElement value) { - value = default; - bool gotValue; if (this.cachedElements.TryGetValue( key, out CosmosElement cosmosElemet)) { value = cosmosElemet; - gotValue = true; + return true; } - else if (this.jsonNavigator.TryGetObjectProperty( + + if (this.jsonNavigator.TryGetObjectProperty( this.jsonNavigatorNode, key, out ObjectProperty objectProperty)) { value = CosmosElement.Dispatch(this.jsonNavigator, objectProperty.ValueNode); - gotValue = true; this.cachedElements[key] = value; - } - else - { - value = null; - gotValue = false; + + return true; } - return gotValue; + value = default; + return false; } public override void WriteTo(IJsonWriter jsonWriter) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosPermissionTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosPermissionTests.cs index 451a5f2262..471aa4a86d 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosPermissionTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosPermissionTests.cs @@ -8,8 +8,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests using System.Collections.Generic; using System.Net; using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Fluent; - using Microsoft.Azure.Cosmos.Scripts; + using Microsoft.Azure.Cosmos.EmulatorTests.Query; using Microsoft.Azure.Cosmos.Utils; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -258,7 +257,7 @@ await container.CreateItemAsync( // Test query with no service interop via gateway query plan to replicate x32 app ContainerCore containerCore = (ContainerInlineCore)tokenContainer; - CrossPartitionQueryTests.MockCosmosQueryClient mock = new CrossPartitionQueryTests.MockCosmosQueryClient( + MockCosmosQueryClient mock = new MockCosmosQueryClient( clientContext: containerCore.ClientContext, cosmosContainerCore: containerCore, forceQueryPlanGatewayElseServiceInterop: true); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.cs deleted file mode 100644 index 195947d506..0000000000 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.cs +++ /dev/null @@ -1,5091 +0,0 @@ -//----------------------------------------------------------------------- -// -// Copyright (c) Microsoft Corporation. All rights reserved. -// -//----------------------------------------------------------------------- -namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests -{ - using System; - using System.Collections.Generic; - using System.Collections.ObjectModel; - using System.Diagnostics; - using System.Globalization; - using System.IO; - using System.Linq; - using System.Net; - using System.Runtime.ExceptionServices; - using System.Text; - using System.Text.RegularExpressions; - using System.Threading; - using System.Threading.Tasks; - using System.Xml; - using Microsoft.Azure.Cosmos.CosmosElements; - using Microsoft.Azure.Cosmos.CosmosElements.Numbers; - using Microsoft.Azure.Cosmos.Query.Core; - using Microsoft.Azure.Cosmos.Query.Core.ExecutionContext.ItemProducers; - using Microsoft.Azure.Cosmos.Query.Core.Metrics; - using Microsoft.Azure.Cosmos.Query.Core.QueryClient; - using Microsoft.Azure.Cosmos.Query.Core.QueryPlan; - using Microsoft.Azure.Cosmos.Resource.CosmosExceptions; - using Microsoft.Azure.Cosmos.Routing; - using Microsoft.Azure.Documents; - using Microsoft.Azure.Documents.Routing; - using Microsoft.VisualStudio.TestTools.UnitTesting; - using Newtonsoft.Json; - using Newtonsoft.Json.Converters; - using Newtonsoft.Json.Linq; - - /// - /// Tests for CrossPartitionQueryTests. - /// - [TestClass] - [TestCategory("Query")] - public class CrossPartitionQueryTests - { - private static readonly string[] NoDocuments = new string[] { }; - private CosmosClient GatewayClient = TestCommon.CreateCosmosClient(true); - private CosmosClient Client = TestCommon.CreateCosmosClient(false); - private Cosmos.Database database; - // private readonly AsyncLocal responseLengthBytes = new AsyncLocal(); - private readonly AsyncLocal outerCosmosQueryResponseActivityId = new AsyncLocal(); - - [FlagsAttribute] - private enum ConnectionModes - { - None = 0, - Direct = 0x1, - Gateway = 0x2, - } - - [FlagsAttribute] - private enum CollectionTypes - { - None = 0, - NonPartitioned = 0x1, - SinglePartition = 0x2, - MultiPartition = 0x4, - } - - [ClassInitialize] - [ClassCleanup] - public static void ClassSetup(TestContext testContext = null) - { - CosmosClient client = TestCommon.CreateCosmosClient(false); - CrossPartitionQueryTests.CleanUp(client).Wait(); - } - - [TestInitialize] - public async Task Initialize() - { - this.database = await this.Client.CreateDatabaseAsync(Guid.NewGuid().ToString() + "db"); - } - - [TestCleanup] - public async Task Cleanup() - { - await this.database.DeleteStreamAsync(); - } - - [TestMethod] - public void ServiceInteropUsedByDefault() - { - // Test initialie does load CosmosClient - Assert.IsFalse(CustomTypeExtensions.ByPassQueryParsing()); - } - - private static string GetApiVersion() - { - return HttpConstants.Versions.CurrentVersion; - } - - private static void SetApiVersion(string apiVersion) - { - HttpConstants.Versions.CurrentVersion = apiVersion; - HttpConstants.Versions.CurrentVersionUTF8 = Encoding.UTF8.GetBytes(apiVersion); - } - - private async Task> GetPartitionKeyRanges(ContainerProperties container) - { - Range fullRange = new Range( - PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey, - PartitionKeyInternal.MaximumExclusiveEffectivePartitionKey, - true, - false); - IRoutingMapProvider routingMapProvider = await this.Client.DocumentClient.GetPartitionKeyRangeCacheAsync(); - Assert.IsNotNull(routingMapProvider); - - IReadOnlyList ranges = await routingMapProvider.TryGetOverlappingRangesAsync(container.ResourceId, fullRange); - return ranges; - } - - private async Task CreateMultiPartitionContainer( - string partitionKey = "/id", - Microsoft.Azure.Cosmos.IndexingPolicy indexingPolicy = null) - { - ContainerResponse containerResponse = await this.CreatePartitionedContainer( - throughput: 25000, - partitionKey: partitionKey, - indexingPolicy: indexingPolicy); - - IReadOnlyList ranges = await this.GetPartitionKeyRanges(containerResponse); - Assert.IsTrue( - ranges.Count() > 1, - $"{nameof(CreateMultiPartitionContainer)} failed to create a container with more than 1 physical partition."); - - return containerResponse; - } - - private async Task CreateSinglePartitionContainer( - string partitionKey = "/id", - Microsoft.Azure.Cosmos.IndexingPolicy indexingPolicy = null) - { - ContainerResponse containerResponse = await this.CreatePartitionedContainer( - throughput: 4000, - partitionKey: partitionKey, - indexingPolicy: indexingPolicy); - - Assert.IsNotNull(containerResponse); - Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode); - Assert.IsNotNull(containerResponse.Resource); - Assert.IsNotNull(containerResponse.Resource.ResourceId); - - IReadOnlyList ranges = await this.GetPartitionKeyRanges(containerResponse); - Assert.AreEqual(1, ranges.Count()); - - return containerResponse; - } - - private async Task CreateNonPartitionedContainer( - Microsoft.Azure.Cosmos.IndexingPolicy indexingPolicy = null) - { - string containerName = Guid.NewGuid().ToString() + "container"; - await NonPartitionedContainerHelper.CreateNonPartitionedContainer( - this.database, - containerName, - indexingPolicy == null ? null : JsonConvert.SerializeObject(indexingPolicy)); - - return this.database.GetContainer(containerName); - } - - private async Task CreatePartitionedContainer( - int throughput, - string partitionKey = "/id", - Microsoft.Azure.Cosmos.IndexingPolicy indexingPolicy = null) - { - // Assert that database exists (race deletes are possible when used concurrently) - ResponseMessage responseMessage = await this.database.ReadStreamAsync(); - Assert.AreEqual(HttpStatusCode.OK, responseMessage.StatusCode); - - ContainerResponse containerResponse = await this.database.CreateContainerAsync( - new ContainerProperties - { - Id = Guid.NewGuid().ToString() + "container", - IndexingPolicy = indexingPolicy ?? new Cosmos.IndexingPolicy - { - IncludedPaths = new Collection - { - new Cosmos.IncludedPath - { - Path = "/*", - Indexes = new Collection - { - Cosmos.Index.Range(Cosmos.DataType.Number), - Cosmos.Index.Range(Cosmos.DataType.String), - } - } - } - }, - PartitionKey = partitionKey == null ? null : new PartitionKeyDefinition - { - Paths = new Collection { partitionKey }, - Kind = PartitionKind.Hash - } - }, - // This throughput needs to be about half the max with multi master - // otherwise it will create about twice as many partitions. - throughput); - - Assert.IsNotNull(containerResponse); - Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode); - Assert.IsNotNull(containerResponse.Resource); - Assert.IsNotNull(containerResponse.Resource.ResourceId); - - return containerResponse; - } - - private async Task>> CreateNonPartitionedContainerAndIngestDocuments( - IEnumerable documents, - Cosmos.IndexingPolicy indexingPolicy = null) - { - return await this.CreateContainerAndIngestDocuments( - CollectionTypes.NonPartitioned, - documents, - partitionKey: null, - indexingPolicy: indexingPolicy); - } - - private async Task>> CreateSinglePartitionContainerAndIngestDocuments( - IEnumerable documents, - string partitionKey = "/id", - Cosmos.IndexingPolicy indexingPolicy = null) - { - return await this.CreateContainerAndIngestDocuments( - CollectionTypes.SinglePartition, - documents, - partitionKey, - indexingPolicy); - } - - private async Task>> CreateMultiPartitionContainerAndIngestDocuments( - IEnumerable documents, - string partitionKey = "/id", - Cosmos.IndexingPolicy indexingPolicy = null) - { - return await this.CreateContainerAndIngestDocuments( - CollectionTypes.MultiPartition, - documents, - partitionKey, - indexingPolicy); - } - - private async Task>> CreateContainerAndIngestDocuments( - CollectionTypes collectionType, - IEnumerable documents, - string partitionKey = "/id", - Cosmos.IndexingPolicy indexingPolicy = null) - { - Container container; - switch (collectionType) - { - case CollectionTypes.NonPartitioned: - container = await this.CreateNonPartitionedContainer(indexingPolicy); - break; - - case CollectionTypes.SinglePartition: - container = await this.CreateSinglePartitionContainer(partitionKey, indexingPolicy); - break; - - case CollectionTypes.MultiPartition: - container = await this.CreateMultiPartitionContainer(partitionKey, indexingPolicy); - break; - - default: - throw new ArgumentException($"Unknown {nameof(CollectionTypes)} : {collectionType}"); - } - - List insertedDocuments = new List(); - foreach (string document in documents) - { - JObject documentObject = JsonConvert.DeserializeObject(document); - // Add an id - if (documentObject["id"] == null) - { - documentObject["id"] = Guid.NewGuid().ToString(); - } - - // Get partition key value. - Cosmos.PartitionKey pkValue; - if (partitionKey != null) - { - string jObjectPartitionKey = partitionKey.Remove(0, 1); - JValue pkToken = (JValue)documentObject[jObjectPartitionKey]; - if (pkToken == null) - { - pkValue = Cosmos.PartitionKey.None; - } - else - { - switch (pkToken.Type) - { - case JTokenType.Integer: - case JTokenType.Float: - pkValue = new Cosmos.PartitionKey(pkToken.Value()); - break; - case JTokenType.String: - pkValue = new Cosmos.PartitionKey(pkToken.Value()); - break; - case JTokenType.Boolean: - pkValue = new Cosmos.PartitionKey(pkToken.Value()); - break; - case JTokenType.Null: - pkValue = Cosmos.PartitionKey.Null; - break; - default: - throw new ArgumentException("Unknown partition key type"); - } - } - } - else - { - pkValue = Cosmos.PartitionKey.None; - } - - JObject createdDocument = await container.CreateItemAsync(documentObject, pkValue); - Document insertedDocument = Document.FromObject(createdDocument); - insertedDocuments.Add(insertedDocument); - } - - return new Tuple>(container, insertedDocuments); - } - - private static async Task CleanUp(CosmosClient client) - { - FeedIterator allDatabases = client.GetDatabaseQueryIterator(); - - while (allDatabases.HasMoreResults) - { - foreach (DatabaseProperties db in await allDatabases.ReadNextAsync()) - { - await client.GetDatabase(db.Id).DeleteAsync(); - } - } - } - - private async Task RunWithApiVersion(string apiVersion, Func function) - { - string originalApiVersion = GetApiVersion(); - CosmosClient originalCosmosClient = this.Client; - CosmosClient originalGatewayClient = this.GatewayClient; - Cosmos.Database originalDatabase = this.database; - - try - { - SetApiVersion(apiVersion); - if (apiVersion != originalApiVersion) - { - this.Client = TestCommon.CreateCosmosClient(false); - this.GatewayClient = TestCommon.CreateCosmosClient(true); - this.database = this.Client.GetDatabase(this.database.Id); - } - - await function(); - } - finally - { - this.Client = originalCosmosClient; - this.GatewayClient = originalGatewayClient; - this.database = originalDatabase; - SetApiVersion(originalApiVersion); - } - } - - internal delegate Task Query( - Container container, - IEnumerable documents); - - internal delegate Task Query( - Container container, - IEnumerable documents, - T testArgs); - - internal delegate CosmosClient CosmosClientFactory(ConnectionMode connectionMode); - - private async Task CreateIngestQueryDelete( - ConnectionModes connectionModes, - CollectionTypes collectionTypes, - IEnumerable documents, - Query query, - string partitionKey = "/id", - Cosmos.IndexingPolicy indexingPolicy = null, - CosmosClientFactory cosmosClientFactory = null) - { - Task queryWrapper(Container container, IEnumerable inputDocuments, object throwaway) - { - return query(container, inputDocuments); - } - - await this.CreateIngestQueryDelete( - connectionModes, - collectionTypes, - documents, - queryWrapper, - null, - partitionKey, - indexingPolicy, - cosmosClientFactory); - } - - private async Task CreateIngestQueryDelete( - ConnectionModes connectionModes, - CollectionTypes collectionTypes, - IEnumerable documents, - Query query, - T testArgs, - string partitionKey = "/id", - Cosmos.IndexingPolicy indexingPolicy = null, - CosmosClientFactory cosmosClientFactory = null) - { - await this.CreateIngestQueryDelete( - connectionModes, - collectionTypes, - documents, - query, - cosmosClientFactory ?? this.CreateDefaultCosmosClient, - testArgs, - partitionKey, - indexingPolicy); - } - - /// - /// Task that wraps boiler plate code for query tests (container create -> ingest documents -> query documents -> delete collections). - /// Note that this function will take the cross product connectionModes - /// - /// The connection modes to use. - /// The documents to ingest - /// - /// The callback for the queries. - /// All the standard arguments will be passed in. - /// Please make sure that this function is idempotent, since a container will be reused for each connection mode. - /// - /// - /// The callback for the create CosmosClient. This is invoked for the different ConnectionModes that the query is targeting. - /// If CosmosClient instantiated by this does not apply the expected ConnectionMode, an assert is thrown. - /// - /// The partition key for the partition container. - /// The optional args that you want passed in to the query. - /// A task to await on. - private async Task CreateIngestQueryDelete( - ConnectionModes connectionModes, - CollectionTypes collectionTypes, - IEnumerable documents, - Query query, - CosmosClientFactory cosmosClientFactory, - T testArgs, - string partitionKey = "/id", - Cosmos.IndexingPolicy indexingPolicy = null) - { - try - { - List>> collectionsAndDocuments = new List>>(); - foreach (CollectionTypes collectionType in Enum.GetValues(collectionTypes.GetType()).Cast().Where(collectionTypes.HasFlag)) - { - if (collectionType == CollectionTypes.None) - { - continue; - } - - Task>> createContainerTask; - switch (collectionType) - { - case CollectionTypes.NonPartitioned: - createContainerTask = this.CreateNonPartitionedContainerAndIngestDocuments( - documents, - indexingPolicy); - break; - - case CollectionTypes.SinglePartition: - createContainerTask = this.CreateSinglePartitionContainerAndIngestDocuments( - documents, - partitionKey, - indexingPolicy); - break; - - case CollectionTypes.MultiPartition: - createContainerTask = this.CreateMultiPartitionContainerAndIngestDocuments( - documents, - partitionKey, - indexingPolicy); - break; - - default: - throw new ArgumentException($"Unknown {nameof(CollectionTypes)} : {collectionType}"); - } - - collectionsAndDocuments.Add(await createContainerTask); - } - - List cosmosClients = new List(); - foreach (ConnectionModes connectionMode in Enum.GetValues(connectionModes.GetType()).Cast().Where(connectionModes.HasFlag)) - { - if (connectionMode == ConnectionModes.None) - { - continue; - } - - ConnectionMode targetConnectionMode = GetTargetConnectionMode(connectionMode); - CosmosClient cosmosClient = cosmosClientFactory(targetConnectionMode); - - Assert.AreEqual( - targetConnectionMode, - cosmosClient.ClientOptions.ConnectionMode, - "Test setup: Invalid connection policy applied to CosmosClient"); - cosmosClients.Add(cosmosClient); - } - - List queryTasks = new List(); - foreach (CosmosClient cosmosClient in cosmosClients) - { - foreach (Tuple> containerAndDocuments in collectionsAndDocuments) - { - Container container = cosmosClient.GetContainer(((ContainerCore)(ContainerInlineCore)containerAndDocuments.Item1).Database.Id, containerAndDocuments.Item1.Id); - Task queryTask = Task.Run(() => query(container, containerAndDocuments.Item2, testArgs)); - queryTasks.Add(queryTask); - } - } - - await Task.WhenAll(queryTasks); - - List> deleteContainerTasks = new List>(); - foreach (Container container in collectionsAndDocuments.Select(tuple => tuple.Item1)) - { - deleteContainerTasks.Add(container.DeleteContainerAsync()); - } - - await Task.WhenAll(deleteContainerTasks); - } - catch (Exception ex) when (ex.GetType() != typeof(AssertFailedException)) - { - while (ex.InnerException != null) ex = ex.InnerException; - - ExceptionDispatchInfo.Capture(ex).Throw(); - } - } - - private static ConnectionMode GetTargetConnectionMode(ConnectionModes connectionMode) - { - ConnectionMode targetConnectionMode = ConnectionMode.Gateway; - switch (connectionMode) - { - case ConnectionModes.Gateway: - targetConnectionMode = ConnectionMode.Gateway; - break; - - case ConnectionModes.Direct: - targetConnectionMode = ConnectionMode.Direct; - break; - - default: - throw new ArgumentException($"Unexpected connection mode: {connectionMode}"); - } - - return targetConnectionMode; - } - - private CosmosClient CreateDefaultCosmosClient(ConnectionMode connectionMode) - { - switch (connectionMode) - { - case ConnectionMode.Gateway: - return this.GatewayClient; - case ConnectionMode.Direct: - return this.Client; - default: - throw new ArgumentException($"Unexpected connection mode: {connectionMode}"); - } - } - - private CosmosClient CreateNewCosmosClient(ConnectionMode connectionMode) - { - switch (connectionMode) - { - case ConnectionMode.Gateway: - return TestCommon.CreateCosmosClient(true); - case ConnectionMode.Direct: - return TestCommon.CreateCosmosClient(false); - default: - throw new ArgumentException($"Unexpected connection mode: {connectionMode}"); - } - } - - private static async Task> QueryWithCosmosElementContinuationTokenAsync( - Container container, - string query, - QueryRequestOptions queryRequestOptions = null) - { - if (queryRequestOptions == null) - { - queryRequestOptions = new QueryRequestOptions(); - } - - List resultsFromCosmosElementContinuationToken = new List(); - CosmosElement continuationToken = null; - do - { - QueryRequestOptions computeRequestOptions = queryRequestOptions.Clone(); - computeRequestOptions.ExecutionEnvironment = Cosmos.Query.Core.ExecutionContext.ExecutionEnvironment.Compute; - computeRequestOptions.CosmosElementContinuationToken = continuationToken; - - FeedIteratorInternal itemQuery = (FeedIteratorInternal)container.GetItemQueryIterator( - queryText: query, - requestOptions: computeRequestOptions); - try - { - FeedResponse cosmosQueryResponse = await itemQuery.ReadNextAsync(); - if (queryRequestOptions.MaxItemCount.HasValue) - { - Assert.IsTrue( - cosmosQueryResponse.Count <= queryRequestOptions.MaxItemCount.Value, - "Max Item Count is not being honored"); - } - - resultsFromCosmosElementContinuationToken.AddRange(cosmosQueryResponse); - continuationToken = itemQuery.GetCosmosElementContinuationToken(); - } - catch (CosmosException cosmosException) when (cosmosException.StatusCode == (HttpStatusCode)429) - { - itemQuery = (FeedIteratorInternal)container.GetItemQueryIterator( - queryText: query, - requestOptions: queryRequestOptions); - } - } while (continuationToken != null); - - return resultsFromCosmosElementContinuationToken; - } - - private static async Task> QueryWithContinuationTokensAsync( - Container container, - string query, - QueryRequestOptions queryRequestOptions = null) - { - if (queryRequestOptions == null) - { - queryRequestOptions = new QueryRequestOptions(); - } - - List resultsFromContinuationToken = new List(); - string continuationToken = null; - do - { - FeedIterator itemQuery = container.GetItemQueryIterator( - queryText: query, - requestOptions: queryRequestOptions, - continuationToken: continuationToken); - - while (true) - { - try - { - FeedResponse cosmosQueryResponse = await itemQuery.ReadNextAsync(); - if (queryRequestOptions.MaxItemCount.HasValue) - { - Assert.IsTrue( - cosmosQueryResponse.Count <= queryRequestOptions.MaxItemCount.Value, - "Max Item Count is not being honored"); - } - - resultsFromContinuationToken.AddRange(cosmosQueryResponse); - continuationToken = cosmosQueryResponse.ContinuationToken; - break; - } - catch (CosmosException cosmosException) when (cosmosException.StatusCode == (HttpStatusCode)429) - { - itemQuery = container.GetItemQueryIterator( - queryText: query, - requestOptions: queryRequestOptions, - continuationToken: continuationToken); - } - } - } while (continuationToken != null); - - return resultsFromContinuationToken; - } - - private static async Task> QueryWithoutContinuationTokensAsync( - Container container, - string query, - QueryRequestOptions queryRequestOptions = null) - { - if (queryRequestOptions == null) - { - queryRequestOptions = new QueryRequestOptions(); - } - - List results = new List(); - FeedIterator itemQuery = container.GetItemQueryIterator( - queryText: query, - requestOptions: queryRequestOptions); - - string continuationTokenForRetries = null; - while (itemQuery.HasMoreResults) - { - try - { - FeedResponse page = await itemQuery.ReadNextAsync(); - results.AddRange(page); - - if (queryRequestOptions.MaxItemCount.HasValue) - { - Assert.IsTrue( - page.Count <= queryRequestOptions.MaxItemCount.Value, - "Max Item Count is not being honored"); - } - - try - { - continuationTokenForRetries = page.ContinuationToken; - } - catch (Exception) - { - // Grabbing a continuation token is not supported on all queries. - } - } - catch (CosmosException cosmosException) when (cosmosException.StatusCode == (HttpStatusCode)429) - { - itemQuery = container.GetItemQueryIterator( - queryText: query, - requestOptions: queryRequestOptions, - continuationToken: continuationTokenForRetries); - - if (continuationTokenForRetries == null) - { - // The query failed and we don't have a save point, so just restart the whole thing. - results = new List(); - } - } - } - - return results; - } - - private static async Task NoOp() - { - await Task.Delay(0); - } - - private async Task RandomlyThrowException(Exception exception = null) - { - await CrossPartitionQueryTests.NoOp(); - Random random = new Random(); - if (random.Next(0, 2) == 0) - { - throw exception; - } - } - - [TestMethod] - public async Task TestBadQueriesOverMultiplePartitions() - { - await this.CreateIngestQueryDelete( - ConnectionModes.Direct | ConnectionModes.Gateway, - CollectionTypes.MultiPartition, - CrossPartitionQueryTests.NoDocuments, - this.TestBadQueriesOverMultiplePartitionsHelper); - } - - private async Task TestBadQueriesOverMultiplePartitionsHelper(Container container, IEnumerable documents) - { - await CrossPartitionQueryTests.NoOp(); - try - { - FeedIterator resultSetIterator = container.GetItemQueryIterator( - @"SELECT * FROM Root r WHERE a = 1", - requestOptions: new QueryRequestOptions() { MaxConcurrency = 2 }); - - await resultSetIterator.ReadNextAsync(); - - Assert.Fail($"Expected {nameof(CosmosException)}"); - } - catch (CosmosException exception) when (exception.StatusCode == HttpStatusCode.BadRequest) - { - Assert.IsTrue(exception.Message.Contains(@"Identifier 'a' could not be resolved."), - exception.Message); - } - } - - /// - //"SELECT c._ts, c.id, c.TicketNumber, c.PosCustomerNumber, c.CustomerId, c.CustomerUserId, c.ContactEmail, c.ContactPhone, c.StoreCode, c.StoreUid, c.PoNumber, c.OrderPlacedOn, c.OrderType, c.OrderStatus, c.Customer.UserFirstName, c.Customer.UserLastName, c.Customer.Name, c.UpdatedBy, c.UpdatedOn, c.ExpirationDate, c.TotalAmountFROM c ORDER BY c._ts"' created an ArgumentOutofRangeException since ServiceInterop was returning DISP_E_BUFFERTOOSMALL in the case of an invalid query that is also really long. - /// This test case just double checks that you get the appropriate document client exception instead of just failing. - /// - [TestMethod] - public async Task TestQueryCrossParitionPartitionProviderInvalid() - { - await this.CreateIngestQueryDelete( - ConnectionModes.Direct | ConnectionModes.Gateway, - CollectionTypes.MultiPartition, - CrossPartitionQueryTests.NoDocuments, - this.TestQueryCrossParitionPartitionProviderInvalidHelper); - } - - private async Task TestQueryCrossParitionPartitionProviderInvalidHelper(Container container, IEnumerable documents) - { - await CrossPartitionQueryTests.NoOp(); - try - { - /// note that there is no space before the from clause thus this query should fail - /// '"code":"SC2001","message":"Identifier 'c' could not be resolved."' - string query = "SELECT c._ts, c.id, c.TicketNumber, c.PosCustomerNumber, c.CustomerId, c.CustomerUserId, c.ContactEmail, c.ContactPhone, c.StoreCode, c.StoreUid, c.PoNumber, c.OrderPlacedOn, c.OrderType, c.OrderStatus, c.Customer.UserFirstName, c.Customer.UserLastName, c.Customer.Name, c.UpdatedBy, c.UpdatedOn, c.ExpirationDate, c.TotalAmountFROM c ORDER BY c._ts"; - List expectedValues = new List(); - FeedIterator resultSetIterator = container.GetItemQueryIterator( - query, - requestOptions: new QueryRequestOptions() { MaxConcurrency = 0 }); - - while (resultSetIterator.HasMoreResults) - { - expectedValues.AddRange(await resultSetIterator.ReadNextAsync()); - } - - Assert.Fail("Expected to get an exception for this query."); - } - catch (CosmosException e) when (e.StatusCode == HttpStatusCode.BadRequest) - { - } - } - - [TestMethod] - public async Task TestQueryWithPartitionKey() - { - string[] documents = new[] - { - @"{""id"":""documentId1"",""key"":""A"",""prop"":3,""shortArray"":[{""a"":5}]}", - @"{""id"":""documentId2"",""key"":""A"",""prop"":2,""shortArray"":[{""a"":6}]}", - @"{""id"":""documentId3"",""key"":""A"",""prop"":1,""shortArray"":[{""a"":7}]}", - @"{""id"":""documentId4"",""key"":5,""prop"":3,""shortArray"":[{""a"":5}]}", - @"{""id"":""documentId5"",""key"":5,""prop"":2,""shortArray"":[{""a"":6}]}", - @"{""id"":""documentId6"",""key"":5,""prop"":1,""shortArray"":[{""a"":7}]}", - @"{""id"":""documentId10"",""prop"":3,""shortArray"":[{""a"":5}]}", - @"{""id"":""documentId11"",""prop"":2,""shortArray"":[{""a"":6}]}", - @"{""id"":""documentId12"",""prop"":1,""shortArray"":[{""a"":7}]}", - }; - - await this.CreateIngestQueryDelete( - ConnectionModes.Direct | ConnectionModes.Gateway, - CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, - documents, - this.TestQueryWithPartitionKeyHelper, - "/key"); - } - - private async Task TestQueryWithPartitionKeyHelper( - Container container, - IEnumerable documents) - { - Assert.AreEqual(0, (await CrossPartitionQueryTests.RunQueryAsync( - container, - @"SELECT * FROM Root r WHERE false", - new QueryRequestOptions() - { - MaxConcurrency = 1, - })).Count); - - object[] keys = new object[] { "A", 5, Undefined.Value }; - for (int i = 0; i < keys.Length; ++i) - { - List expected = documents.Skip(i * 3).Take(3).Select(doc => doc.Id).ToList(); - string expectedResult = string.Join(",", expected); - // Order-by - expected.Reverse(); - string expectedOrderByResult = string.Join(",", expected); - - List<(string, string)> queries = new List<(string, string)>() - { - ($@"SELECT * FROM Root r WHERE r.id IN (""{expected[0]}"", ""{expected[1]}"", ""{expected[2]}"")", expectedResult), - (@"SELECT * FROM Root r WHERE r.prop BETWEEN 1 AND 3", expectedResult), - (@"SELECT VALUE r FROM Root r JOIN c IN r.shortArray WHERE c.a BETWEEN 5 and 7", expectedResult), - ($@"SELECT TOP 10 * FROM Root r WHERE r.id IN (""{expected[0]}"", ""{expected[1]}"", ""{expected[2]}"")", expectedResult), - (@"SELECT TOP 10 * FROM Root r WHERE r.prop BETWEEN 1 AND 3", expectedResult), - (@"SELECT TOP 10 VALUE r FROM Root r JOIN c IN r.shortArray WHERE c.a BETWEEN 5 and 7", expectedResult), - ($@"SELECT * FROM Root r WHERE r.id IN (""{expected[0]}"", ""{expected[1]}"", ""{expected[2]}"") ORDER BY r.prop", expectedOrderByResult), - (@"SELECT * FROM Root r WHERE r.prop BETWEEN 1 AND 3 ORDER BY r.prop", expectedOrderByResult), - (@"SELECT VALUE r FROM Root r JOIN c IN r.shortArray WHERE c.a BETWEEN 5 and 7 ORDER BY r.prop", expectedOrderByResult), - }; - - - - if (i < keys.Length - 1) - { - string key; - if (keys[i] is string) - { - key = "'" + keys[i].ToString() + "'"; - } - else - { - key = keys[i].ToString(); - } - - queries.Add((string.Format(CultureInfo.InvariantCulture, @"SELECT * FROM Root r WHERE r.key = {0} ORDER BY r.prop", key), expectedOrderByResult)); - } - - foreach ((string, string) queryAndExpectedResult in queries) - { - FeedIterator resultSetIterator = container.GetItemQueryIterator( - queryText: queryAndExpectedResult.Item1, - requestOptions: new QueryRequestOptions() - { - MaxItemCount = 1, - PartitionKey = new Cosmos.PartitionKey(keys[i]), - }); - - List result = new List(); - while (resultSetIterator.HasMoreResults) - { - result.AddRange(await resultSetIterator.ReadNextAsync()); - } - - string resultDocIds = string.Join(",", result.Select(doc => doc.Id)); - Assert.AreEqual(queryAndExpectedResult.Item2, resultDocIds); - } - } - } - - [TestMethod] - public async Task TestQuerySinglePartitionKey() - { - string[] documents = new[] - { - @"{""pk"":""doc1""}", - @"{""pk"":""doc2""}", - @"{""pk"":""doc3""}", - @"{""pk"":""doc4""}", - @"{""pk"":""doc5""}", - @"{""pk"":""doc6""}", - }; - - await this.CreateIngestQueryDelete( - ConnectionModes.Direct | ConnectionModes.Gateway, - CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, - documents, - this.TestQuerySinglePartitionKeyHelper, - "/pk"); - } - - private async Task TestQuerySinglePartitionKeyHelper( - Container container, - IEnumerable documents) - { - // Query with partition key should be done in one round trip. - FeedIterator resultSetIterator = container.GetItemQueryIterator( - "SELECT * FROM c WHERE c.pk = 'doc5'"); - - FeedResponse response = await resultSetIterator.ReadNextAsync(); - Assert.AreEqual(1, response.Count()); - Assert.IsNull(response.ContinuationToken); - - resultSetIterator = container.GetItemQueryIterator( - "SELECT * FROM c WHERE c.pk = 'doc10'"); - - response = await resultSetIterator.ReadNextAsync(); - Assert.AreEqual(0, response.Count()); - Assert.IsNull(response.ContinuationToken); - } - - private struct QueryWithSpecialPartitionKeysArgs - { - public string Name; - public object Value; - public Func ValueToPartitionKey; - } - - // V3 only supports Numeric, string, bool, null, undefined - [TestMethod] - [Ignore] - public async Task TestQueryWithSpecialPartitionKeys() - { - await CrossPartitionQueryTests.NoOp(); - QueryWithSpecialPartitionKeysArgs[] queryWithSpecialPartitionKeyArgsList = new QueryWithSpecialPartitionKeysArgs[] - { - new QueryWithSpecialPartitionKeysArgs() - { - Name = "Guid", - Value = Guid.NewGuid(), - ValueToPartitionKey = val => val.ToString(), - }, - //new QueryWithSpecialPartitionKeysArgs() - //{ - // Name = "DateTime", - // Value = DateTime.Now, - // ValueToPartitionKey = val => - // { - // string str = JsonConvert.SerializeObject( - // val, - // new JsonSerializerSettings() - // { - // Converters = new List { new IsoDateTimeConverter() } - // }); - // return str.Substring(1, str.Length - 2); - // }, - //}, - new QueryWithSpecialPartitionKeysArgs() - { - Name = "Enum", - Value = HttpStatusCode.OK, - ValueToPartitionKey = val => (int)val, - }, - new QueryWithSpecialPartitionKeysArgs() - { - Name = "CustomEnum", - Value = HttpStatusCode.OK, - ValueToPartitionKey = val => val.ToString(), - }, - new QueryWithSpecialPartitionKeysArgs() - { - Name = "ResourceId", - Value = "testid", - ValueToPartitionKey = val => val, - }, - new QueryWithSpecialPartitionKeysArgs() - { - Name = "CustomDateTime", - Value = new DateTime(2016, 11, 12), - ValueToPartitionKey = val => EpochDateTimeConverter.DateTimeToEpoch((DateTime)val), - }, - }; - - foreach (QueryWithSpecialPartitionKeysArgs testArg in queryWithSpecialPartitionKeyArgsList) - { - // For this test we need to split direct and gateway runs into separate collections, - // since the query callback inserts some documents (thus has side effects). - await this.CreateIngestQueryDelete( - ConnectionModes.Direct, - CollectionTypes.SinglePartition, - CrossPartitionQueryTests.NoDocuments, - this.TestQueryWithSpecialPartitionKeysHelper, - testArg, - "/" + testArg.Name); - - await this.CreateIngestQueryDelete( - ConnectionModes.Direct, - CollectionTypes.MultiPartition, - CrossPartitionQueryTests.NoDocuments, - this.TestQueryWithSpecialPartitionKeysHelper, - testArg, - "/" + testArg.Name); - - await this.CreateIngestQueryDelete( - ConnectionModes.Gateway, - CollectionTypes.SinglePartition, - CrossPartitionQueryTests.NoDocuments, - this.TestQueryWithSpecialPartitionKeysHelper, - testArg, - "/" + testArg.Name); - - await this.CreateIngestQueryDelete( - ConnectionModes.Gateway, - CollectionTypes.MultiPartition, - CrossPartitionQueryTests.NoDocuments, - this.TestQueryWithSpecialPartitionKeysHelper, - testArg, - "/" + testArg.Name); - } - } - - private async Task TestQueryWithSpecialPartitionKeysHelper(Container container, IEnumerable documents, QueryWithSpecialPartitionKeysArgs testArgs) - { - QueryWithSpecialPartitionKeysArgs args = testArgs; - - SpecialPropertyDocument specialPropertyDocument = new SpecialPropertyDocument - { - Id = Guid.NewGuid().ToString() - }; - - specialPropertyDocument.GetType().GetProperty(args.Name).SetValue(specialPropertyDocument, args.Value); - object getPropertyValueFunction(SpecialPropertyDocument d) => d.GetType().GetProperty(args.Name).GetValue(d); - - ItemResponse response = await container.CreateItemAsync(specialPropertyDocument); - dynamic returnedDoc = response.Resource; - Assert.AreEqual(args.Value, getPropertyValueFunction((SpecialPropertyDocument)returnedDoc)); - - PartitionKey key = new PartitionKey(args.ValueToPartitionKey(args.Value)); - response = await container.ReadItemAsync(response.Resource.Id, new Cosmos.PartitionKey(key)); - returnedDoc = response.Resource; - Assert.AreEqual(args.Value, getPropertyValueFunction((SpecialPropertyDocument)returnedDoc)); - - returnedDoc = (await this.RunSinglePartitionQuery( - container, - "SELECT * FROM t")).Single(); - - Assert.AreEqual(args.Value, getPropertyValueFunction(returnedDoc)); - - string query; - switch (args.Name) - { - case "Guid": - query = $"SELECT * FROM T WHERE T.Guid = '{(Guid)args.Value}'"; - break; - case "Enum": - query = $"SELECT * FROM T WHERE T.Enum = '{(HttpStatusCode)args.Value}'"; - break; - case "DateTime": - query = $"SELECT * FROM T WHERE T.DateTime = '{(DateTime)args.Value}'"; - break; - case "CustomEnum": - query = $"SELECT * FROM T WHERE T.CustomEnum = '{(HttpStatusCode)args.Value}'"; - break; - case "ResourceId": - query = $"SELECT * FROM T WHERE T.ResourceId = '{(string)args.Value}'"; - break; - case "CustomDateTime": - query = $"SELECT * FROM T WHERE T.CustomDateTime = '{(DateTime)args.Value}'"; - break; - default: - query = null; - break; - } - - returnedDoc = (await container.GetItemQueryIterator( - query, - requestOptions: new QueryRequestOptions() - { - MaxItemCount = 1, - PartitionKey = new Cosmos.PartitionKey(args.ValueToPartitionKey), - }).ReadNextAsync()).First(); - - Assert.AreEqual(args.Value, getPropertyValueFunction(returnedDoc)); - } - - private sealed class SpecialPropertyDocument - { - public string Id - { - get; - set; - } - - public Guid Guid - { - get; - set; - } - - [JsonConverter(typeof(IsoDateTimeConverter))] - public DateTime DateTime - { - get; - set; - } - - [JsonConverter(typeof(EpochDateTimeConverter))] - public DateTime CustomDateTime - { - get; - set; - } - - - public HttpStatusCode Enum - { - get; - set; - } - - [JsonConverter(typeof(StringEnumConverter))] - public HttpStatusCode CustomEnum - { - get; - set; - } - - public string ResourceId - { - get; - set; - } - } - - private sealed class EpochDateTimeConverter : JsonConverter - { - public static int DateTimeToEpoch(DateTime dt) - { - if (!dt.Equals(DateTime.MinValue)) - { - DateTime epoch = new DateTime(1970, 1, 1); - TimeSpan epochTimeSpan = dt - epoch; - return (int)epochTimeSpan.TotalSeconds; - } - else - { - return int.MinValue; - } - } - - public override bool CanConvert(Type objectType) - { - return true; - } - - public override object ReadJson(Newtonsoft.Json.JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) - { - if (reader.TokenType == JsonToken.None || reader.TokenType == JsonToken.Null) - { - return null; - } - - - if (reader.TokenType != JsonToken.Integer) - { - throw new Exception( - string.Format( - CultureInfo.InvariantCulture, - "Unexpected token parsing date. Expected Integer, got {0}.", - reader.TokenType)); - } - - int seconds = Convert.ToInt32(reader.Value, CultureInfo.InvariantCulture); - return new DateTime(1970, 1, 1).AddSeconds(seconds); - } - - public override void WriteJson(Newtonsoft.Json.JsonWriter writer, object value, JsonSerializer serializer) - { - int seconds; - if (value is DateTime) - { - seconds = DateTimeToEpoch((DateTime)value); - } - else - { - throw new Exception("Expected date object value."); - } - - writer.WriteValue(seconds); - } - } - - private struct QueryCrossPartitionWithLargeNumberOfKeysArgs - { - public int NumberOfDocuments; - public string PartitionKey; - public HashSet ExpectedPartitionKeyValues; - } - - [TestMethod] - public async Task TestQueryCrossPartitionWithLargeNumberOfKeys() - { - int numberOfDocuments = 1000; - string partitionKey = "key"; - HashSet expectedPartitionKeyValues = new HashSet(); - List documents = new List(); - for (int i = 0; i < numberOfDocuments; i++) - { - Document doc = new Document(); - doc.SetPropertyValue(partitionKey, i); - documents.Add(doc.ToString()); - - expectedPartitionKeyValues.Add(i); - } - - Assert.AreEqual(numberOfDocuments, expectedPartitionKeyValues.Count); - - QueryCrossPartitionWithLargeNumberOfKeysArgs args = new QueryCrossPartitionWithLargeNumberOfKeysArgs() - { - NumberOfDocuments = numberOfDocuments, - PartitionKey = partitionKey, - ExpectedPartitionKeyValues = expectedPartitionKeyValues, - }; - - await this.CreateIngestQueryDelete( - ConnectionModes.Direct | ConnectionModes.Gateway, - CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, - documents, - this.TestQueryCrossPartitionWithLargeNumberOfKeysHelper, - args, - "/" + partitionKey); - } - - private async Task TestQueryCrossPartitionWithLargeNumberOfKeysHelper( - Container container, - IEnumerable documents, - QueryCrossPartitionWithLargeNumberOfKeysArgs args) - { - QueryDefinition query = new QueryDefinition( - $"SELECT VALUE r.{args.PartitionKey} FROM r WHERE ARRAY_CONTAINS(@keys, r.{args.PartitionKey})").WithParameter("@keys", args.ExpectedPartitionKeyValues); - - HashSet actualPartitionKeyValues = new HashSet(); - FeedIterator documentQuery = container.GetItemQueryIterator( - queryDefinition: query, - requestOptions: new QueryRequestOptions() { MaxItemCount = -1, MaxConcurrency = 100 }); - - while (documentQuery.HasMoreResults) - { - FeedResponse response = await documentQuery.ReadNextAsync(); - foreach (int item in response) - { - actualPartitionKeyValues.Add(item); - } - } - - Assert.IsTrue(actualPartitionKeyValues.SetEquals(args.ExpectedPartitionKeyValues)); - } - - [TestMethod] - public async Task TestBasicCrossPartitionQuery() - { - int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; - uint numberOfDocuments = 100; - QueryOracle.QueryOracleUtil util = new QueryOracle.QueryOracle2(seed); - IEnumerable documents = util.GetDocuments(numberOfDocuments); - - await this.CreateIngestQueryDelete( - ConnectionModes.Direct, - CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, - documents, - this.TestBasicCrossPartitionQueryHelper); - } - - private async Task TestBasicCrossPartitionQueryHelper( - Container container, - IEnumerable documents) - { - foreach (int maxDegreeOfParallelism in new int[] { 1, 100 }) - { - foreach (int maxItemCount in new int[] { 10, 100 }) - { - foreach (string query in new string[] { "SELECT c.id FROM c", "SELECT c._ts, c.id FROM c ORDER BY c._ts" }) - { - QueryRequestOptions feedOptions = new QueryRequestOptions - { - MaxBufferedItemCount = 7000, - MaxConcurrency = maxDegreeOfParallelism, - MaxItemCount = maxItemCount, - ReturnResultsInDeterministicOrder = true, - }; - - List queryResults = await CrossPartitionQueryTests.RunQueryAsync( - container, - query, - feedOptions); - - Assert.AreEqual( - documents.Count(), - queryResults.Count, - $"query: {query} failed with {nameof(maxDegreeOfParallelism)}: {maxDegreeOfParallelism}, {nameof(maxItemCount)}: {maxItemCount}"); - } - } - } - } - - [TestMethod] - public async Task TestNonDeterministicQueryResults() - { - int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; - uint numberOfDocuments = 100; - QueryOracle.QueryOracleUtil util = new QueryOracle.QueryOracle2(seed); - IEnumerable documents = util.GetDocuments(numberOfDocuments); - - async Task Implementation(Container container, IEnumerable inputDocuments) - { - foreach (int maxDegreeOfParallelism in new int[] { 1, 100 }) - { - foreach (int maxItemCount in new int[] { 10, 100 }) - { - foreach (bool useOrderBy in new bool[] { false, true }) - { - string query; - if (useOrderBy) - { - query = "SELECT c._ts, c.id FROM c ORDER BY c._ts"; - } - else - { - query = "SELECT c.id FROM c"; - } - - QueryRequestOptions queryRequestOptions = new QueryRequestOptions - { - MaxBufferedItemCount = 7000, - MaxConcurrency = maxDegreeOfParallelism, - MaxItemCount = maxItemCount, - ReturnResultsInDeterministicOrder = false, - }; - - async Task ValidateNonDeterministicQuery(Func>> queryFunc, bool hasOrderBy) - { - List queryResults = await queryFunc(container, query, queryRequestOptions); - HashSet expectedIds = new HashSet(inputDocuments.Select(document => document.Id)); - HashSet actualIds = new HashSet(queryResults.Select(queryResult => queryResult["id"].Value())); - Assert.IsTrue(expectedIds.SetEquals(actualIds), $"query: {query} failed with {nameof(maxDegreeOfParallelism)}: {maxDegreeOfParallelism}, {nameof(maxItemCount)}: {maxItemCount}"); - - if (hasOrderBy) - { - IEnumerable timestamps = queryResults.Select(token => token["_ts"].Value()); - IEnumerable sortedTimestamps = timestamps.OrderBy(x => x); - Assert.IsTrue(timestamps.SequenceEqual(sortedTimestamps), "Items were not sorted."); - } - } - - await ValidateNonDeterministicQuery(CrossPartitionQueryTests.QueryWithoutContinuationTokensAsync, useOrderBy); - await ValidateNonDeterministicQuery(CrossPartitionQueryTests.QueryWithContinuationTokensAsync, useOrderBy); - await ValidateNonDeterministicQuery(CrossPartitionQueryTests.QueryWithCosmosElementContinuationTokenAsync, useOrderBy); - } - } - } - } - - await this.CreateIngestQueryDelete( - ConnectionModes.Direct, - CollectionTypes.MultiPartition, - documents, - Implementation); - } - - [TestMethod] - public async Task TestExceptionlessFailures() - { - int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; - uint numberOfDocuments = 100; - QueryOracle.QueryOracleUtil util = new QueryOracle.QueryOracle2(seed); - IEnumerable documents = util.GetDocuments(numberOfDocuments); - - await this.CreateIngestQueryDelete( - ConnectionModes.Direct, - CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, - documents, - this.TestExceptionlessFailuresHelper); - } - - private async Task TestExceptionlessFailuresHelper( - Container container, - IEnumerable documents) - { - foreach (int maxItemCount in new int[] { 10, 100 }) - { - foreach (string query in new string[] { "SELECT c.id FROM c", "SELECT c._ts, c.id FROM c ORDER BY c._ts" }) - { - QueryRequestOptions feedOptions = new QueryRequestOptions - { - MaxBufferedItemCount = 7000, - MaxConcurrency = 2, - MaxItemCount = maxItemCount, - TestSettings = new TestInjections(simulate429s: true, simulateEmptyPages: false) - }; - - List queryResults = await CrossPartitionQueryTests.RunQueryAsync( - container, - query, - feedOptions); - - Assert.AreEqual( - documents.Count(), - queryResults.Count, - $"query: {query} failed with {nameof(maxItemCount)}: {maxItemCount}"); - } - } - } - - [TestMethod] - public async Task TestEmptyPages() - { - int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; - uint numberOfDocuments = 100; - QueryOracle.QueryOracleUtil util = new QueryOracle.QueryOracle2(seed); - IEnumerable documents = util.GetDocuments(numberOfDocuments); - - await this.CreateIngestQueryDelete( - ConnectionModes.Direct, - CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, - documents, - this.TestEmptyPagesHelper); - } - - private async Task TestEmptyPagesHelper( - Container container, - IEnumerable documents) - { - foreach (int maxItemCount in new int[] { 10, 100 }) - { - foreach (string query in new string[] { "SELECT c.id FROM c", "SELECT c._ts, c.id FROM c ORDER BY c._ts" }) - { - QueryRequestOptions feedOptions = new QueryRequestOptions - { - MaxBufferedItemCount = 7000, - MaxConcurrency = 2, - MaxItemCount = maxItemCount, - TestSettings = new TestInjections(simulate429s: false, simulateEmptyPages: true) - }; - - List queryResults = await CrossPartitionQueryTests.RunQueryAsync( - container, - query, - feedOptions); - - Assert.AreEqual( - documents.Count(), - queryResults.Count, - $"query: {query} failed with {nameof(maxItemCount)}: {maxItemCount}"); - } - } - } - - [TestMethod] - public async Task TestQueryPlanGatewayAndServiceInterop() - { - int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; - uint numberOfDocuments = 100; - QueryOracle.QueryOracleUtil util = new QueryOracle.QueryOracle2(seed); - IEnumerable documents = util.GetDocuments(numberOfDocuments); - - await this.CreateIngestQueryDelete( - ConnectionModes.Direct, - CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, - documents, - this.TestQueryPlanGatewayAndServiceInteropHelper); - } - - private async Task TestQueryPlanGatewayAndServiceInteropHelper( - Container container, - IEnumerable documents) - { - ContainerCore containerCore = (ContainerInlineCore)container; - - foreach (bool isGatewayQueryPlan in new bool[] { true, false }) - { - MockCosmosQueryClient cosmosQueryClientCore = new MockCosmosQueryClient( - containerCore.ClientContext, - containerCore, - isGatewayQueryPlan); - - ContainerCore containerWithForcedPlan = new ContainerCore( - containerCore.ClientContext, - (DatabaseCore)containerCore.Database, - containerCore.Id, - cosmosQueryClientCore); - - int numOfQueries = 0; - foreach (int maxDegreeOfParallelism in new int[] { 1, 100 }) - { - foreach (int maxItemCount in new int[] { 10, 100 }) - { - numOfQueries++; - QueryRequestOptions feedOptions = new QueryRequestOptions - { - MaxBufferedItemCount = 7000, - MaxConcurrency = maxDegreeOfParallelism, - MaxItemCount = maxItemCount, - }; - - List queryResults = await CrossPartitionQueryTests.RunQueryAsync( - containerWithForcedPlan, - "SELECT * FROM c ORDER BY c._ts", - feedOptions); - - Assert.AreEqual(documents.Count(), queryResults.Count); - } - } - - if (isGatewayQueryPlan) - { - Assert.IsTrue(cosmosQueryClientCore.QueryPlanCalls > numOfQueries); - } - else - { - Assert.AreEqual(0, cosmosQueryClientCore.QueryPlanCalls, "ServiceInterop mode should not be calling gateway plan retriever"); - } - } - } - - [TestMethod] - public async Task TestUnsupportedQueries() - { - await this.CreateIngestQueryDelete( - ConnectionModes.Direct | ConnectionModes.Gateway, - CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, - NoDocuments, - this.TestUnsupportedQueriesHelper); - } - - private async Task TestUnsupportedQueriesHelper( - Container container, - IEnumerable documents) - { - QueryRequestOptions feedOptions = new QueryRequestOptions - { - MaxBufferedItemCount = 7000, - MaxConcurrency = 10, - MaxItemCount = 10, - }; - - string compositeAggregate = "SELECT COUNT(1) + 5 FROM c"; - - string[] unsupportedQueries = new string[] - { - compositeAggregate, - }; - - foreach (string unsupportedQuery in unsupportedQueries) - { - try - { - await CrossPartitionQueryTests.RunQueryAsync( - container, - unsupportedQuery, - queryRequestOptions: feedOptions); - Assert.Fail("Expected query to fail due it not being supported."); - } - catch (Exception e) - { - Assert.IsTrue(e.Message.Contains("Compositions of aggregates and other expressions are not allowed."), - e.Message); - } - } - } - - [TestMethod] - public async Task TestQueryCrossPartitionAggregateFunctions() - { - AggregateTestArgs aggregateTestArgs = new AggregateTestArgs() - { - NumberOfDocsWithSamePartitionKey = 37, - NumberOfDocumentsDifferentPartitionKey = 43, - PartitionKey = "key", - UniquePartitionKey = "uniquePartitionKey", - Field = "field", - Values = new object[] { false, true, "abc", "cdfg", "opqrs", "ttttttt", "xyz" }, - }; - - List documents = new List(aggregateTestArgs.NumberOfDocumentsDifferentPartitionKey + aggregateTestArgs.NumberOfDocsWithSamePartitionKey); - foreach (object val in aggregateTestArgs.Values) - { - Document doc; - doc = new Document(); - doc.SetPropertyValue(aggregateTestArgs.PartitionKey, val); - doc.SetPropertyValue("id", Guid.NewGuid().ToString()); - - documents.Add(doc.ToString()); - } - - for (int i = 0; i < aggregateTestArgs.NumberOfDocsWithSamePartitionKey; ++i) - { - Document doc = new Document(); - doc.SetPropertyValue(aggregateTestArgs.PartitionKey, aggregateTestArgs.UniquePartitionKey); - documents.Add(doc.ToString()); - } - - Random random = new Random(); - for (int i = 0; i < aggregateTestArgs.NumberOfDocumentsDifferentPartitionKey; ++i) - { - Document doc = new Document(); - doc.SetPropertyValue(aggregateTestArgs.PartitionKey, random.NextDouble()); - documents.Add(doc.ToString()); - } - - await this.CreateIngestQueryDelete( - ConnectionModes.Direct | ConnectionModes.Gateway, - CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, - documents, - this.TestQueryCrossPartitionAggregateFunctionsAsync, - aggregateTestArgs, - "/" + aggregateTestArgs.PartitionKey); - } - - private struct AggregateTestArgs - { - public int NumberOfDocumentsDifferentPartitionKey; - public int NumberOfDocsWithSamePartitionKey; - public string PartitionKey; - public string UniquePartitionKey; - public string Field; - public object[] Values; - } - - private struct AggregateQueryArguments - { - public string AggregateOperator; - public CosmosElement ExpectedValue; - public string Predicate; - } - - private async Task TestQueryCrossPartitionAggregateFunctionsAsync( - Container container, - IEnumerable documents, - AggregateTestArgs aggregateTestArgs) - { - IEnumerable documentsWherePkIsANumber = documents - .Where(doc => - { - return double.TryParse( - JObject.Parse(doc.ToString())[aggregateTestArgs.PartitionKey].ToString(), - out double result); - }); - double numberSum = documentsWherePkIsANumber - .Sum(doc => - { - return double.Parse(JObject.Parse(doc.ToString())[aggregateTestArgs.PartitionKey].ToString()); - }); - double count = documentsWherePkIsANumber.Count(); - AggregateQueryArguments[] aggregateQueryArgumentsList = new AggregateQueryArguments[] - { - new AggregateQueryArguments() - { - AggregateOperator = "AVG", - ExpectedValue = CosmosNumber64.Create(numberSum / count), - Predicate = $"IS_NUMBER(r.{aggregateTestArgs.PartitionKey})", - }, - new AggregateQueryArguments() - { - AggregateOperator = "AVG", - ExpectedValue = null, - Predicate = "true", - }, - new AggregateQueryArguments() - { - AggregateOperator = "COUNT", - ExpectedValue = CosmosNumber64.Create(documents.Count()), - Predicate = "true", - }, - new AggregateQueryArguments() - { - AggregateOperator = "MAX", - ExpectedValue = CosmosString.Create("xyz"), - Predicate = "true", - }, - new AggregateQueryArguments() - { - AggregateOperator = "MIN", - ExpectedValue = CosmosBoolean.Create(false), - Predicate = "true", - }, - new AggregateQueryArguments() - { - AggregateOperator = "SUM", - ExpectedValue = CosmosNumber64.Create(numberSum), - Predicate = $"IS_NUMBER(r.{aggregateTestArgs.PartitionKey})", - }, - new AggregateQueryArguments() - { - AggregateOperator = "SUM", - ExpectedValue = null, - Predicate = $"true", - }, - }; - - foreach (int maxDoP in new[] { 0, 10 }) - { - foreach (AggregateQueryArguments argument in aggregateQueryArgumentsList) - { - string[] queryFormats = new[] - { - "SELECT VALUE {0}(r.{1}) FROM r WHERE {2}", - "SELECT VALUE {0}(r.{1}) FROM r WHERE {2} ORDER BY r.{1}" - }; - - foreach (string queryFormat in queryFormats) - { - string query = string.Format( - CultureInfo.InvariantCulture, - queryFormat, - argument.AggregateOperator, - aggregateTestArgs.PartitionKey, - argument.Predicate); - string message = string.Format( - CultureInfo.InvariantCulture, - "query: {0}, data: {1}", - query, - JsonConvert.SerializeObject(argument)); - - List items = await CrossPartitionQueryTests.RunQueryAsync( - container, - query, - new QueryRequestOptions() - { - MaxConcurrency = maxDoP, - }); - - if (argument.ExpectedValue == null) - { - Assert.AreEqual(0, items.Count, message); - } - else - { - Assert.AreEqual(1, items.Count, message); - CosmosElement expected = argument.ExpectedValue; - CosmosElement actual = items.Single(); - - if ((expected is CosmosNumber expectedNumber) && (actual is CosmosNumber actualNumber)) - { - Assert.AreEqual(Number64.ToDouble(expectedNumber.Value), Number64.ToDouble(actualNumber.Value), .01); - } - else - { - Assert.AreEqual(expected, actual, message); - } - } - } - } - } - } - - [TestMethod] - public async Task TestQueryCrossPartitionAggregateFunctionsEmptyPartitions() - { - AggregateQueryEmptyPartitionsArgs args = new AggregateQueryEmptyPartitionsArgs() - { - NumDocuments = 100, - PartitionKey = "key", - UniqueField = "UniqueField", - }; - - List documents = new List(args.NumDocuments); - for (int i = 0; i < args.NumDocuments; ++i) - { - Document doc = new Document(); - doc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); - doc.SetPropertyValue(args.UniqueField, i); - documents.Add(doc.ToString()); - } - - await this.CreateIngestQueryDelete( - ConnectionModes.Direct | ConnectionModes.Gateway, - CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, - documents, - this.TestQueryCrossPartitionAggregateFunctionsEmptyPartitionsHelper, - args, - "/" + args.PartitionKey); - } - - private struct AggregateQueryEmptyPartitionsArgs - { - public int NumDocuments; - public string PartitionKey; - public string UniqueField; - } - - private async Task TestQueryCrossPartitionAggregateFunctionsEmptyPartitionsHelper(Container container, IEnumerable documents, AggregateQueryEmptyPartitionsArgs args) - { - await CrossPartitionQueryTests.NoOp(); - int numDocuments = args.NumDocuments; - string partitionKey = args.PartitionKey; - string uniqueField = args.UniqueField; - - // Perform full fanouts but only match a single value that isn't the partition key. - // This leads to all other partitions returning { "" = UNDEFINDED, "count" = 0 } - // which should be ignored from the aggregation. - int valueOfInterest = args.NumDocuments / 2; - string[] queries = new string[] - { - $"SELECT VALUE AVG(c.{uniqueField}) FROM c WHERE c.{uniqueField} = {valueOfInterest}", - $"SELECT VALUE MIN(c.{uniqueField}) FROM c WHERE c.{uniqueField} = {valueOfInterest}", - $"SELECT VALUE MAX(c.{uniqueField}) FROM c WHERE c.{uniqueField} = {valueOfInterest}", - $"SELECT VALUE SUM(c.{uniqueField}) FROM c WHERE c.{uniqueField} = {valueOfInterest}", - }; - - foreach (string query in queries) - { - try - { - List items = await CrossPartitionQueryTests.RunQueryAsync( - container, - query, - new QueryRequestOptions() - { - MaxConcurrency = 10, - }); - - Assert.AreEqual(valueOfInterest, items.Single()); - } - catch (Exception ex) - { - Assert.Fail($"Something went wrong with query: {query}, ex: {ex}"); - } - } - } - - [TestMethod] - public async Task TestQueryCrossPartitionAggregateFunctionsWithMixedTypes() - { - AggregateQueryMixedTypes args = new AggregateQueryMixedTypes() - { - PartitionKey = "key", - Field = "field", - DoubleOnlyKey = "doubleOnly", - StringOnlyKey = "stringOnly", - BoolOnlyKey = "boolOnly", - NullOnlyKey = "nullOnly", - ObjectOnlyKey = "objectOnlyKey", - ArrayOnlyKey = "arrayOnlyKey", - OneObjectKey = "oneObjectKey", - OneArrayKey = "oneArrayKey", - UndefinedKey = "undefinedKey", - }; - - List documents = new List(); - Random random = new Random(1234); - for (int i = 0; i < 20; ++i) - { - Document doubleDoc = new Document(); - doubleDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); - doubleDoc.SetPropertyValue(args.Field, random.Next(1, 100000)); - documents.Add(doubleDoc.ToString()); - doubleDoc.SetPropertyValue(args.PartitionKey, args.DoubleOnlyKey); - documents.Add(doubleDoc.ToString()); - - Document stringDoc = new Document(); - stringDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); - stringDoc.SetPropertyValue(args.Field, random.NextDouble().ToString()); - documents.Add(stringDoc.ToString()); - stringDoc.SetPropertyValue(args.PartitionKey, args.StringOnlyKey); - documents.Add(stringDoc.ToString()); - - Document boolDoc = new Document(); - boolDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); - boolDoc.SetPropertyValue(args.Field, random.Next() % 2 == 0); - documents.Add(boolDoc.ToString()); - boolDoc.SetPropertyValue(args.PartitionKey, args.BoolOnlyKey); - documents.Add(boolDoc.ToString()); - - Document nullDoc = new Document(); - nullDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); - nullDoc.propertyBag.Add(args.Field, null); - documents.Add(nullDoc.ToString()); - nullDoc.SetPropertyValue(args.PartitionKey, args.NullOnlyKey); - documents.Add(nullDoc.ToString()); - - Document objectDoc = new Document(); - objectDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); - objectDoc.SetPropertyValue(args.Field, new object { }); - documents.Add(objectDoc.ToString()); - objectDoc.SetPropertyValue(args.PartitionKey, args.ObjectOnlyKey); - documents.Add(objectDoc.ToString()); - - Document arrayDoc = new Document(); - arrayDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); - arrayDoc.SetPropertyValue(args.Field, new object[] { }); - documents.Add(arrayDoc.ToString()); - arrayDoc.SetPropertyValue(args.PartitionKey, args.ArrayOnlyKey); - documents.Add(arrayDoc.ToString()); - } - - Document oneObjectDoc = new Document(); - oneObjectDoc.SetPropertyValue(args.PartitionKey, args.OneObjectKey); - oneObjectDoc.SetPropertyValue(args.Field, new object { }); - documents.Add(oneObjectDoc.ToString()); - - Document oneArrayDoc = new Document(); - oneArrayDoc.SetPropertyValue(args.PartitionKey, args.OneArrayKey); - oneArrayDoc.SetPropertyValue(args.Field, new object[] { }); - documents.Add(oneArrayDoc.ToString()); - - Document undefinedDoc = new Document(); - undefinedDoc.SetPropertyValue(args.PartitionKey, args.UndefinedKey); - // This doc does not have the field key set - documents.Add(undefinedDoc.ToString()); - - await this.CreateIngestQueryDelete( - ConnectionModes.Direct, - CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, - documents, - this.TestQueryCrossPartitionAggregateFunctionsWithMixedTypesHelper, - args, - "/" + args.PartitionKey); - } - - private struct AggregateQueryMixedTypes - { - public string PartitionKey; - public string Field; - public string DoubleOnlyKey; - public string StringOnlyKey; - public string BoolOnlyKey; - public string NullOnlyKey; - public string ObjectOnlyKey; - public string ArrayOnlyKey; - public string OneObjectKey; - public string OneArrayKey; - public string UndefinedKey; - } - - private async Task TestQueryCrossPartitionAggregateFunctionsWithMixedTypesHelper( - Container container, - IEnumerable documents, - AggregateQueryMixedTypes args) - { - await CrossPartitionQueryTests.NoOp(); - string partitionKey = args.PartitionKey; - string field = args.Field; - string[] typeOnlyPartitionKeys = new string[] - { - args.DoubleOnlyKey, - args.StringOnlyKey, - args.BoolOnlyKey, - args.NullOnlyKey, - args.ObjectOnlyKey, - args.ArrayOnlyKey, - args.OneArrayKey, - args.OneObjectKey, - args.UndefinedKey - }; - - string[] aggregateOperators = new string[] { "AVG", "MIN", "MAX", "SUM", "COUNT" }; - string[] typeCheckFunctions = new string[] { "IS_ARRAY", "IS_BOOL", "IS_NULL", "IS_NUMBER", "IS_OBJECT", "IS_STRING", "IS_DEFINED", "IS_PRIMITIVE" }; - List queries = new List(); - foreach (string aggregateOperator in aggregateOperators) - { - foreach (string typeCheckFunction in typeCheckFunctions) - { - queries.Add( - $@" - SELECT VALUE {aggregateOperator} (c.{field}) - FROM c - WHERE {typeCheckFunction}(c.{field}) - "); - } - - foreach (string typeOnlyPartitionKey in typeOnlyPartitionKeys) - { - queries.Add( - $@" - SELECT VALUE {aggregateOperator} (c.{field}) - FROM c - WHERE c.{partitionKey} = ""{typeOnlyPartitionKey}"" - "); - } - }; - - // mixing primitive and non primitives - foreach (string minmaxop in new string[] { "MIN", "MAX" }) - { - foreach (string key in new string[] { args.OneObjectKey, args.OneArrayKey }) - { - queries.Add( - $@" - SELECT VALUE {minmaxop} (c.{field}) - FROM c - WHERE c.{partitionKey} IN (""{key}"", ""{args.DoubleOnlyKey}"") - "); - } - } - - string filename = $"CrossPartitionQueryTests.AggregateMixedTypes"; - string baselinePath = $"{filename}_baseline.xml"; - - XmlWriterSettings settings = new XmlWriterSettings() - { - OmitXmlDeclaration = true, - Indent = true, - NewLineOnAttributes = true, - }; - - StringBuilder builder = new StringBuilder(); - using (XmlWriter writer = XmlWriter.Create(builder, settings)) - { - writer.WriteStartDocument(); - writer.WriteStartElement("Results"); - foreach (string query in queries) - { - string formattedQuery = string.Join( - Environment.NewLine, - query.Trim().Split( - new[] { Environment.NewLine }, - StringSplitOptions.None) - .Select(x => x.Trim())); - - List items = await CrossPartitionQueryTests.RunQueryAsync( - container, - query, - new QueryRequestOptions() - { - MaxItemCount = 10, - }); - - writer.WriteStartElement("Result"); - writer.WriteStartElement("Query"); - writer.WriteCData(formattedQuery); - writer.WriteEndElement(); - writer.WriteStartElement("Aggregation"); - if (items.Count > 0) - { - writer.WriteCData(JsonConvert.SerializeObject(items.Single())); - } - writer.WriteEndElement(); - writer.WriteEndElement(); - } - writer.WriteEndElement(); - writer.WriteEndDocument(); - } - - Regex r = new Regex(">\\s+"); - string normalizedBaseline = r.Replace(File.ReadAllText(baselinePath), ">"); - string normalizedOutput = r.Replace(builder.ToString(), ">"); - - Assert.AreEqual(normalizedBaseline, normalizedOutput); - } - - [TestMethod] - [Owner("brchon")] - public async Task TestNonValueAggregates() - { - string[] documents = new string[] - { - @"{""first"":""Good"",""last"":""Trevino"",""age"":23,""height"":61,""income"":59848}", - @"{""first"":""Charles"",""last"":""Decker"",""age"":31,""height"":64,""income"":55970}", - @"{""first"":""Holden"",""last"":""Cotton"",""age"":30,""height"":66,""income"":57075}", - @"{""first"":""Carlene"",""last"":""Cabrera"",""age"":26,""height"":72,""income"":98018}", - @"{""first"":""Gates"",""last"":""Spence"",""age"":38,""height"":53,""income"":12338}", - @"{""first"":""Camacho"",""last"":""Singleton"",""age"":40,""height"":52,""income"":76973}", - @"{""first"":""Rachel"",""last"":""Tucker"",""age"":27,""height"":68,""income"":28116}", - @"{""first"":""Kristi"",""last"":""Robertson"",""age"":32,""height"":53,""income"":61687}", - @"{""first"":""Poole"",""last"":""Petty"",""age"":22,""height"":75,""income"":53381}", - @"{""first"":""Lacey"",""last"":""Carlson"",""age"":38,""height"":78,""income"":63989}", - @"{""first"":""Rosario"",""last"":""Mendez"",""age"":21,""height"":64,""income"":20300}", - @"{""first"":""Estrada"",""last"":""Collins"",""age"":28,""height"":74,""income"":6926}", - @"{""first"":""Ursula"",""last"":""Burton"",""age"":26,""height"":66,""income"":32870}", - @"{""first"":""Rochelle"",""last"":""Sanders"",""age"":24,""height"":56,""income"":47564}", - @"{""first"":""Darcy"",""last"":""Herring"",""age"":27,""height"":52,""income"":67436}", - @"{""first"":""Carole"",""last"":""Booth"",""age"":34,""height"":60,""income"":50177}", - @"{""first"":""Cruz"",""last"":""Russell"",""age"":25,""height"":52,""income"":95072}", - @"{""first"":""Wilma"",""last"":""Robbins"",""age"":36,""height"":50,""income"":53008}", - @"{""first"":""Mcdaniel"",""last"":""Barlow"",""age"":21,""height"":78,""income"":85441}", - @"{""first"":""Leann"",""last"":""Blackwell"",""age"":40,""height"":79,""income"":900}", - @"{""first"":""Hoffman"",""last"":""Hoffman"",""age"":31,""height"":76,""income"":1208}", - @"{""first"":""Pittman"",""last"":""Shepherd"",""age"":35,""height"":61,""income"":26887}", - @"{""first"":""Wright"",""last"":""Rojas"",""age"":35,""height"":73,""income"":76487}", - @"{""first"":""Lynne"",""last"":""Waters"",""age"":27,""height"":60,""income"":22926}", - @"{""first"":""Corina"",""last"":""Shelton"",""age"":29,""height"":78,""income"":67379}", - @"{""first"":""Alvarez"",""last"":""Barr"",""age"":29,""height"":59,""income"":34698}", - @"{""first"":""Melinda"",""last"":""Mccoy"",""age"":24,""height"":63,""income"":69811}", - @"{""first"":""Chelsea"",""last"":""Bolton"",""age"":20,""height"":63,""income"":47698}", - @"{""first"":""English"",""last"":""Ingram"",""age"":28,""height"":50,""income"":94977}", - @"{""first"":""Vance"",""last"":""Thomas"",""age"":30,""height"":49,""income"":67638}", - @"{""first"":""Howell"",""last"":""Joyner"",""age"":34,""height"":78,""income"":65547}", - @"{""first"":""Ofelia"",""last"":""Chapman"",""age"":23,""height"":82,""income"":85049}", - @"{""first"":""Downs"",""last"":""Adams"",""age"":28,""height"":76,""income"":19373}", - @"{""first"":""Terrie"",""last"":""Bryant"",""age"":32,""height"":55,""income"":79024}", - @"{""first"":""Jeanie"",""last"":""Carson"",""age"":26,""height"":52,""income"":68293}", - @"{""first"":""Hazel"",""last"":""Bean"",""age"":40,""height"":70,""income"":46028}", - @"{""first"":""Dominique"",""last"":""Norman"",""age"":25,""height"":50,""income"":59445}", - @"{""first"":""Lyons"",""last"":""Patterson"",""age"":36,""height"":64,""income"":71748}", - @"{""first"":""Catalina"",""last"":""Cantrell"",""age"":30,""height"":78,""income"":16999}", - @"{""first"":""Craft"",""last"":""Head"",""age"":30,""height"":49,""income"":10542}", - @"{""first"":""Suzanne"",""last"":""Gilliam"",""age"":36,""height"":77,""income"":7511}", - @"{""first"":""Pamela"",""last"":""Merritt"",""age"":30,""height"":81,""income"":80653}", - @"{""first"":""Haynes"",""last"":""Ayala"",""age"":38,""height"":65,""income"":85832}", - @"{""first"":""Teri"",""last"":""Martin"",""age"":40,""height"":83,""income"":27839}", - @"{""first"":""Susanne"",""last"":""Short"",""age"":25,""height"":57,""income"":48957}", - @"{""first"":""Rosalie"",""last"":""Camacho"",""age"":24,""height"":83,""income"":30313}", - @"{""first"":""Walls"",""last"":""Bray"",""age"":28,""height"":74,""income"":21616}", - @"{""first"":""Norris"",""last"":""Bates"",""age"":23,""height"":59,""income"":13631}", - @"{""first"":""Wendy"",""last"":""King"",""age"":38,""height"":48,""income"":19845}", - @"{""first"":""Deena"",""last"":""Ramsey"",""age"":20,""height"":66,""income"":49665}", - @"{""first"":""Richmond"",""last"":""Meadows"",""age"":36,""height"":59,""income"":43244}", - @"{""first"":""Burks"",""last"":""Whitley"",""age"":25,""height"":55,""income"":39974}", - @"{""first"":""Gilliam"",""last"":""George"",""age"":37,""height"":82,""income"":47114}", - @"{""first"":""Marcy"",""last"":""Harding"",""age"":33,""height"":80,""income"":20316}", - @"{""first"":""Curtis"",""last"":""Gomez"",""age"":31,""height"":50,""income"":69085}", - @"{""first"":""Lopez"",""last"":""Burt"",""age"":34,""height"":79,""income"":37577}", - @"{""first"":""Nell"",""last"":""Nixon"",""age"":37,""height"":58,""income"":67999}", - @"{""first"":""Sonja"",""last"":""Lamb"",""age"":37,""height"":53,""income"":92553}", - @"{""first"":""Owens"",""last"":""Fischer"",""age"":40,""height"":48,""income"":75199}", - @"{""first"":""Ortega"",""last"":""Padilla"",""age"":28,""height"":55,""income"":29126}", - @"{""first"":""Stacie"",""last"":""Velez"",""age"":20,""height"":56,""income"":45292}", - @"{""first"":""Brennan"",""last"":""Craig"",""age"":38,""height"":65,""income"":37445}" - }; - - await this.CreateIngestQueryDelete( - ConnectionModes.Direct, - /*CollectionTypes.SinglePartition |*/ CollectionTypes.MultiPartition, - documents, - this.TestNonValueAggregates); - } - - private async Task TestNonValueAggregates( - Container container, - IEnumerable documents) - { - IEnumerable documentsAsJTokens = documents.Select(document => JToken.FromObject(document)); - - // ------------------------------------------ - // Positive - // ------------------------------------------ - - List> queryAndExpectedAggregation = new List>() - { - // ------------------------------------------ - // Simple Aggregates without a value - // ------------------------------------------ - - new Tuple( - "SELECT SUM(c.age) FROM c", - new JObject - { - { - "$1", - documentsAsJTokens.Sum(document => document["age"].Value()) - } - }), - - new Tuple( - "SELECT COUNT(c.age) FROM c", - new JObject - { - { - "$1", - documentsAsJTokens.Where(document => document["age"] != null).Count() - } - }), - - new Tuple( - "SELECT MIN(c.age) FROM c", - new JObject - { - { - "$1", - documentsAsJTokens.Min(document => document["age"].Value()) - } - }), - - new Tuple( - "SELECT MAX(c.age) FROM c", - new JObject - { - { - "$1", - documentsAsJTokens.Max(document => document["age"].Value()) - } - }), - - new Tuple( - "SELECT AVG(c.age) FROM c", - new JObject - { - { - "$1", - documentsAsJTokens.Average(document => document["age"].Value()) - } - }), - - // ------------------------------------------ - // Simple aggregates with alias - // ------------------------------------------ - - new Tuple( - "SELECT SUM(c.age) as sum_age FROM c", - new JObject - { - { - "sum_age", - documentsAsJTokens.Sum(document => document["age"].Value()) - } - }), - - new Tuple( - "SELECT COUNT(c.age) as count_age FROM c", - new JObject - { - { - "count_age", - documentsAsJTokens.Where(document => document["age"] != null).Count() - } - }), - - new Tuple( - "SELECT MIN(c.age) as min_age FROM c", - new JObject - { - { - "min_age", - documentsAsJTokens.Min(document => document["age"].Value()) - } - }), - - new Tuple( - "SELECT MAX(c.age) as max_age FROM c", - new JObject - { - { - "max_age", - documentsAsJTokens.Max(document => document["age"].Value()) - } - }), - - new Tuple( - "SELECT AVG(c.age) as avg_age FROM c", - new JObject - { - { - "avg_age", - documentsAsJTokens.Average(document => document["age"].Value()) - } - }), - - // ------------------------------------------ - // Multiple Aggregates without alias - // ------------------------------------------ - - new Tuple( - "SELECT MIN(c.age), MAX(c.age) FROM c", - new JObject - { - { - "$1", - documentsAsJTokens.Min(document => document["age"].Value()) - }, - { - "$2", - documentsAsJTokens.Max(document => document["age"].Value()) - } - }), - - // ------------------------------------------ - // Multiple Aggregates with alias - // ------------------------------------------ - - new Tuple( - "SELECT MIN(c.age) as min_age, MAX(c.age) as max_age FROM c", - new JObject - { - { - "min_age", - documentsAsJTokens.Min(document => document["age"].Value()) - }, - { - "max_age", - documentsAsJTokens.Max(document => document["age"].Value()) - } - }), - - // ------------------------------------------ - // Multiple Aggregates with and without alias - // ------------------------------------------ - - new Tuple( - "SELECT MIN(c.age), MAX(c.age) as max_age FROM c", - new JObject - { - { - "$1", - documentsAsJTokens.Min(document => document["age"].Value()) - }, - { - "max_age", - documentsAsJTokens.Max(document => document["age"].Value()) - } - }), - - new Tuple( - "SELECT MIN(c.age) as min_age, MAX(c.age) FROM c", - new JObject - { - { - "min_age", - documentsAsJTokens.Min(document => document["age"].Value()) - }, - { - "$1", - documentsAsJTokens.Max(document => document["age"].Value()) - } - }), - }; - - // Test query correctness. - foreach ((string query, JToken expectedAggregation) in queryAndExpectedAggregation) - { - foreach (int maxItemCount in new int[] { 1, 5, 10 }) - { - List actual = await RunQueryAsync( - container: container, - query: query, - queryRequestOptions: new QueryRequestOptions() - { - MaxBufferedItemCount = 100, - MaxConcurrency = 100, - MaxItemCount = maxItemCount, - }); - - Assert.AreEqual(1, actual.Count()); - - Assert.IsTrue( - JsonTokenEqualityComparer.Value.Equals(actual.First(), expectedAggregation), - $"Results did not match for query: {query} with maxItemCount: {maxItemCount}" + - $"Actual: {JsonConvert.SerializeObject(actual.First())}" + - $"Expected: {JsonConvert.SerializeObject(expectedAggregation)}"); - } - } - - // ------------------------------------------ - // Negative - // ------------------------------------------ - - List notSupportedQueries = new List() - { - "SELECT MIN(c.age) + MAX(c.age) FROM c", - "SELECT MIN(c.age) / 2 FROM c", - }; - - foreach (string query in notSupportedQueries) - { - try - { - List actual = await QueryWithoutContinuationTokensAsync( - container: container, - query: query, - queryRequestOptions: new QueryRequestOptions() - { - MaxBufferedItemCount = 100, - MaxConcurrency = 100, - }); - - Assert.Fail("Expected Query To Fail"); - } - catch (Exception) - { - // Do Nothing - } - } - } - - [TestMethod] - public async Task TestQueryDistinct() - { - int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; - uint numberOfDocuments = 100; - - Random rand = new Random(seed); - List people = new List(); - - for (int i = 0; i < numberOfDocuments; i++) - { - // Generate random people - Person person = CrossPartitionQueryTests.GetRandomPerson(rand); - for (int j = 0; j < rand.Next(0, 4); j++) - { - // Force an exact duplicate - people.Add(person); - } - } - - List documents = new List(); - // Shuffle them so they end up in different pages - people = people.OrderBy((person) => Guid.NewGuid()).ToList(); - foreach (Person person in people) - { - documents.Add(JsonConvert.SerializeObject(person)); - } - - await this.CreateIngestQueryDelete( - ConnectionModes.Direct, - CollectionTypes.MultiPartition, - documents, - this.TestQueryDistinct, - "/id"); - } - - private async Task TestQueryDistinct(Container container, IEnumerable documents, dynamic testArgs = null) - { - #region Queries - // To verify distint queries you can run it once without the distinct clause and run it through a hash set - // then compare to the query with the distinct clause. - List queries = new List() - { - // basic distinct queries - "SELECT {0} VALUE null", - - // number value distinct queries - "SELECT {0} VALUE c.income from c", - - // string value distinct queries - "SELECT {0} VALUE c.name from c", - - // array value distinct queries - "SELECT {0} VALUE c.children from c", - - // object value distinct queries - "SELECT {0} VALUE c.pet from c", - - // scalar expressions distinct query - "SELECT {0} VALUE c.age % 2 FROM c", - - // distinct queries with order by - "SELECT {0} VALUE c.age FROM c ORDER BY c.age", - - // distinct queries with top and no matching order by - "SELECT {0} TOP 2147483647 VALUE c.age FROM c", - - // distinct queries with top and matching order by - "SELECT {0} TOP 2147483647 VALUE c.age FROM c ORDER BY c.age", - - // distinct queries with aggregates - "SELECT {0} VALUE MAX(c.age) FROM c", - - // distinct queries with joins - "SELECT {0} VALUE c.age FROM p JOIN c IN p.children", - - // distinct queries in subqueries - "SELECT {0} r.age, s FROM r JOIN (SELECT DISTINCT VALUE c FROM (SELECT 1 a) c) s WHERE r.age > 25", - - // distinct queries in scalar subqeries - "SELECT {0} p.name, (SELECT DISTINCT VALUE p.age) AS Age FROM p", - - // select * - "SELECT {0} * FROM c", - }; - #endregion - #region ExecuteNextAsync API - // run the query with distinct and without + MockDistinctMap - // Should receive same results - // PageSize = 1 guarantees that the backend will return some duplicates. - foreach (string query in queries) - { - string queryWithoutDistinct = string.Format(query, ""); - - QueryRequestOptions requestOptions = new QueryRequestOptions() { MaxItemCount = 100, MaxConcurrency = 100 }; - FeedIterator documentQueryWithoutDistinct = container.GetItemQueryIterator( - queryWithoutDistinct, - requestOptions: requestOptions); - - MockDistinctMap documentsSeen = new MockDistinctMap(); - List documentsFromWithoutDistinct = new List(); - while (documentQueryWithoutDistinct.HasMoreResults) - { - FeedResponse cosmosQueryResponse = await documentQueryWithoutDistinct.ReadNextAsync(); - foreach (JToken document in cosmosQueryResponse) - { - if (documentsSeen.Add(document, out Cosmos.Query.Core.UInt128 hash)) - { - documentsFromWithoutDistinct.Add(document); - } - else - { - // No Op for debugging purposes. - } - } - } - - foreach (int pageSize in new int[] { 1, 10, 100 }) - { - string queryWithDistinct = string.Format(query, "DISTINCT"); - List documentsFromWithDistinct = new List(); - FeedIterator documentQueryWithDistinct = container.GetItemQueryIterator( - queryWithDistinct, - requestOptions: requestOptions); - - while (documentQueryWithDistinct.HasMoreResults) - { - FeedResponse cosmosQueryResponse = await documentQueryWithDistinct.ReadNextAsync(); - documentsFromWithDistinct.AddRange(cosmosQueryResponse); - } - - Assert.AreEqual(documentsFromWithDistinct.Count, documentsFromWithoutDistinct.Count()); - for (int i = 0; i < documentsFromWithDistinct.Count; i++) - { - JToken documentFromWithDistinct = documentsFromWithDistinct.ElementAt(i); - JToken documentFromWithoutDistinct = documentsFromWithoutDistinct.ElementAt(i); - Assert.IsTrue( - JsonTokenEqualityComparer.Value.Equals(documentFromWithDistinct, documentFromWithoutDistinct), - $"{documentFromWithDistinct} did not match {documentFromWithoutDistinct} at index {i} for {queryWithDistinct}, with page size: {pageSize} on a container"); - } - } - } - #endregion - #region Continuation Token Support - // Run the ordered distinct query through the continuation api, should result in the same set - // since the previous hash is passed in the continuation token. - foreach (string query in new string[] - { - "SELECT {0} VALUE c.age FROM c ORDER BY c.age", - "SELECT {0} VALUE c.name FROM c ORDER BY c.name", - }) - { - string queryWithoutDistinct = string.Format(query, ""); - MockDistinctMap documentsSeen = new MockDistinctMap(); - List documentsFromWithoutDistinct = await CrossPartitionQueryTests.RunQueryCombinationsAsync( - container, - queryWithoutDistinct, - new QueryRequestOptions() - { - MaxConcurrency = 10, - MaxItemCount = 100, - }, - QueryDrainingMode.ContinuationToken | QueryDrainingMode.HoldState); - documentsFromWithoutDistinct = documentsFromWithoutDistinct - .Where(document => documentsSeen.Add(document, out Cosmos.Query.Core.UInt128 hash)) - .ToList(); - - foreach (int pageSize in new int[] { 1, 10, 100 }) - { - string queryWithDistinct = string.Format(query, "DISTINCT"); - List documentsFromWithDistinct = await CrossPartitionQueryTests.RunQueryCombinationsAsync( - container, - queryWithDistinct, - new QueryRequestOptions() - { - MaxConcurrency = 10, - MaxItemCount = pageSize - }, - QueryDrainingMode.ContinuationToken | QueryDrainingMode.HoldState); - - Assert.IsTrue( - documentsFromWithDistinct.SequenceEqual(documentsFromWithoutDistinct, JsonTokenEqualityComparer.Value), - $"Documents didn't match for {queryWithDistinct} on a Partitioned container"); - } - } - #endregion - #region TryGetContinuationToken Support - // Run the ordered distinct query through the continuation api, should result in the same set - // since the previous hash is passed in the continuation token. - foreach (string query in new string[] - { - "SELECT {0} VALUE c.age FROM c ORDER BY c.age", - "SELECT {0} VALUE c.name FROM c ORDER BY c.name", - "SELECT {0} VALUE c.name from c", - "SELECT {0} VALUE c.age from c", - "SELECT {0} VALUE c.mixedTypeField from c", - "SELECT {0} TOP 2147483647 VALUE c.city from c", - "SELECT {0} VALUE c.age from c ORDER BY c.name", - }) - { - string queryWithoutDistinct = string.Format(query, ""); - MockDistinctMap documentsSeen = new MockDistinctMap(); - List documentsFromWithoutDistinct = await CrossPartitionQueryTests.RunQueryCombinationsAsync( - container, - queryWithoutDistinct, - new QueryRequestOptions() - { - MaxConcurrency = 10, - MaxItemCount = 100, - }, - QueryDrainingMode.HoldState | QueryDrainingMode.CosmosElementContinuationToken); - documentsFromWithoutDistinct = documentsFromWithoutDistinct - .Where(document => documentsSeen.Add(document, out Cosmos.Query.Core.UInt128 hash)) - .ToList(); - - foreach (int pageSize in new int[] { 1, 10, 100 }) - { - string queryWithDistinct = string.Format(query, "DISTINCT"); - List documentsFromWithDistinct = await CrossPartitionQueryTests.RunQueryCombinationsAsync( - container, - queryWithDistinct, - new QueryRequestOptions() - { - MaxConcurrency = 10, - MaxItemCount = pageSize - }, - QueryDrainingMode.HoldState | QueryDrainingMode.CosmosElementContinuationToken); - - Assert.IsTrue( - documentsFromWithDistinct.SequenceEqual(documentsFromWithoutDistinct, JsonTokenEqualityComparer.Value), - $"Documents didn't match for {queryWithDistinct} on a Partitioned container"); - } - } - #endregion - } - - [TestMethod] - public async Task TestQueryCrossPartitionTopOrderByDifferentDimension() - { - string[] documents = new[] - { - @"{""id"":""documentId1"",""key"":""A""}", - @"{""id"":""documentId2"",""key"":""A"",""prop"":3}", - @"{""id"":""documentId3"",""key"":""A""}", - @"{""id"":""documentId4"",""key"":5}", - @"{""id"":""documentId5"",""key"":5,""prop"":2}", - @"{""id"":""documentId6"",""key"":5}", - @"{""id"":""documentId7"",""key"":2}", - @"{""id"":""documentId8"",""key"":2,""prop"":1}", - @"{""id"":""documentId9"",""key"":2}", - }; - - await this.CreateIngestQueryDelete( - ConnectionModes.Direct | ConnectionModes.Gateway, - CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, - documents, - this.TestQueryCrossPartitionTopOrderByDifferentDimensionHelper, - "/key"); - } - - private async Task TestQueryCrossPartitionTopOrderByDifferentDimensionHelper(Container container, IEnumerable documents) - { - await CrossPartitionQueryTests.NoOp(); - - string[] expected = new[] { "documentId2", "documentId5", "documentId8" }; - List query = await CrossPartitionQueryTests.RunQueryAsync( - container, - "SELECT r.id FROM r ORDER BY r.prop DESC", - new QueryRequestOptions() - { - MaxItemCount = 1, - MaxConcurrency = 1, - }); - - Assert.AreEqual(string.Join(", ", expected), string.Join(", ", query.Select(doc => doc.Id))); - } - - [TestMethod] - public async Task TestMixedTypeOrderBy() - { - int numberOfDocuments = 1 << 4; - int numberOfDuplicates = 1 << 2; - - List documents = new List(numberOfDocuments * numberOfDuplicates); - Random random = new Random(1234); - for (int i = 0; i < numberOfDocuments; ++i) - { - MixedTypedDocument mixedTypeDocument = CrossPartitionQueryTests.GenerateMixedTypeDocument(random); - for (int j = 0; j < numberOfDuplicates; j++) - { - if (mixedTypeDocument.MixedTypeField != null) - { - documents.Add(JsonConvert.SerializeObject(mixedTypeDocument)); - } - else - { - documents.Add("{}"); - } - } - } - - // Add a composite index to force an index v2 container to be made. - Cosmos.IndexingPolicy indexV2Policy = new Cosmos.IndexingPolicy() - { - IncludedPaths = new Collection() - { - new Cosmos.IncludedPath() - { - Path = "/*", - }, - new Cosmos.IncludedPath() - { - Path = $"/{nameof(MixedTypedDocument.MixedTypeField)}/?", - } - }, - - CompositeIndexes = new Collection>() - { - // Simple - new Collection() - { - new Cosmos.CompositePath() - { - Path = "/_ts", - }, - new Cosmos.CompositePath() - { - Path = "/_etag", - } - } - } - }; - - OrderByTypes primitives = OrderByTypes.Bool | OrderByTypes.Null | OrderByTypes.Number | OrderByTypes.String; - OrderByTypes nonPrimitives = OrderByTypes.Array | OrderByTypes.Object; - OrderByTypes all = primitives | nonPrimitives | OrderByTypes.Undefined; - - await this.CreateIngestQueryDelete( - ConnectionModes.Direct, - CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, - documents, - this.TestMixedTypeOrderByHelper, - new OrderByTypes[] - { - OrderByTypes.Array, - OrderByTypes.Bool, - OrderByTypes.Null, - OrderByTypes.Number, - OrderByTypes.Object, - OrderByTypes.String, - OrderByTypes.Undefined, - primitives, - nonPrimitives, - all, - }, - "/id", - indexV2Policy); - } - - private sealed class MixedTypedDocument - { - public CosmosElement MixedTypeField { get; set; } - } - - private static MixedTypedDocument GenerateMixedTypeDocument(Random random) - { - return new MixedTypedDocument() - { - MixedTypeField = GenerateRandomJsonValue(random), - }; - } - - private static CosmosElement GenerateRandomJsonValue(Random random) - { - switch (random.Next(0, 7)) - { - // Number - case 0: - return CosmosNumber64.Create(random.Next()); - // String - case 1: - return CosmosString.Create(new string('a', random.Next(0, 100))); - // Null - case 2: - return CosmosNull.Create(); - // Bool - case 3: - return CosmosBoolean.Create((random.Next() % 2) == 0); - // Object - case 4: - return CosmosObject.Create(new List>()); - // Array - case 5: - return CosmosArray.Create(new List()); - // Undefined - case 6: - return null; - default: - throw new ArgumentException(); - } - } - - private sealed class MockOrderByComparer : IComparer - { - public static readonly MockOrderByComparer Value = new MockOrderByComparer(); - - public int Compare(CosmosElement element1, CosmosElement element2) - { - return ItemComparer.Instance.Compare(element1, element2); - } - - } - - [Flags] - private enum OrderByTypes - { - Number = 1 << 0, - String = 1 << 1, - Null = 1 << 2, - Bool = 1 << 3, - Object = 1 << 4, - Array = 1 << 5, - Undefined = 1 << 6, - }; - - private async Task TestMixedTypeOrderByHelper( - Container container, - IEnumerable documents, - OrderByTypes[] args) - { - OrderByTypes[] orderByTypesList = args; - foreach (bool isDesc in new bool[] { true, false }) - { - foreach (OrderByTypes orderByTypes in orderByTypesList) - { - string orderString = isDesc ? "DESC" : "ASC"; - List mixedTypeFilters = new List(); - if (orderByTypes.HasFlag(OrderByTypes.Array)) - { - mixedTypeFilters.Add($"IS_ARRAY(c.{nameof(MixedTypedDocument.MixedTypeField)})"); - } - - if (orderByTypes.HasFlag(OrderByTypes.Bool)) - { - mixedTypeFilters.Add($"IS_BOOL(c.{nameof(MixedTypedDocument.MixedTypeField)})"); - } - - if (orderByTypes.HasFlag(OrderByTypes.Null)) - { - mixedTypeFilters.Add($"IS_NULL(c.{nameof(MixedTypedDocument.MixedTypeField)})"); - } - - if (orderByTypes.HasFlag(OrderByTypes.Number)) - { - mixedTypeFilters.Add($"IS_NUMBER(c.{nameof(MixedTypedDocument.MixedTypeField)})"); - } - - if (orderByTypes.HasFlag(OrderByTypes.Object)) - { - mixedTypeFilters.Add($"IS_OBJECT(c.{nameof(MixedTypedDocument.MixedTypeField)})"); - } - - if (orderByTypes.HasFlag(OrderByTypes.String)) - { - mixedTypeFilters.Add($"IS_STRING(c.{nameof(MixedTypedDocument.MixedTypeField)})"); - } - - if (orderByTypes.HasFlag(OrderByTypes.Undefined)) - { - mixedTypeFilters.Add($"not IS_DEFINED(c.{nameof(MixedTypedDocument.MixedTypeField)})"); - } - - string filter = mixedTypeFilters.Count() == 0 ? "true" : string.Join(" OR ", mixedTypeFilters); - - string query = $@" - SELECT c.{nameof(MixedTypedDocument.MixedTypeField)} - FROM c - WHERE {filter} - ORDER BY c.{nameof(MixedTypedDocument.MixedTypeField)} {orderString}"; - - QueryRequestOptions feedOptions = new QueryRequestOptions() - { - MaxBufferedItemCount = 1000, - MaxItemCount = 16, - MaxConcurrency = 10, - }; - - List actualFromQueryWithoutContinutionTokens; - actualFromQueryWithoutContinutionTokens = await CrossPartitionQueryTests.QueryWithoutContinuationTokensAsync( - container, - query, - queryRequestOptions: feedOptions); -#if false - For now we can not serve the query through continuation tokens correctly. - This is because we allow order by on mixed types but not comparisions across types - For example suppose the following query: - SELECT c.MixedTypeField FROM c ORDER BY c.MixedTypeField - returns: - [ - {"MixedTypeField":null}, - {"MixedTypeField":false}, - {"MixedTypeField":true}, - {"MixedTypeField":303093052}, - {"MixedTypeField":438985130}, - {"MixedTypeField":"aaaaaaaaaaa"} - ] - and we left off on 303093052 then at some point the cross partition code resumes the query by running the following: - SELECT c.MixedTypeField FROM c WHERE c.MixedTypeField > 303093052 ORDER BY c.MixedTypeField - which will only return the following: - { "MixedTypeField":438985130} - and that is because comparision across types is undefined so "aaaaaaaaaaa" > 303093052 never got emitted -#endif - - IEnumerable insertedDocs = documents - .Select(document => CosmosElement.CreateFromBuffer(Encoding.UTF8.GetBytes(document.ToString())) as CosmosObject) - .Select(document => - { - Dictionary dictionary = new Dictionary(); - if (document.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosElement value)) - { - dictionary.Add(nameof(MixedTypedDocument.MixedTypeField), value); - } - - return CosmosObject.Create(dictionary); - }); - - // Build the expected results using LINQ - IEnumerable expected = new List(); - - // Filter based on the mixedOrderByType enum - - if (orderByTypes.HasFlag(OrderByTypes.Undefined)) - { - expected = expected.Concat(insertedDocs.Where(x => !x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosElement value))); - } - - if (orderByTypes.HasFlag(OrderByTypes.Null)) - { - expected = expected.Concat(insertedDocs.Where(x => x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosNull value))); - } - - if (orderByTypes.HasFlag(OrderByTypes.Bool)) - { - expected = expected.Concat(insertedDocs.Where(x => x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosBoolean value))); - } - - if (orderByTypes.HasFlag(OrderByTypes.Number)) - { - expected = expected.Concat(insertedDocs.Where(x => x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosNumber value))); - } - - if (orderByTypes.HasFlag(OrderByTypes.String)) - { - expected = expected.Concat(insertedDocs.Where(x => x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosString value))); - } - - if (orderByTypes.HasFlag(OrderByTypes.Array)) - { - expected = expected.Concat(insertedDocs.Where(x => x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosArray value))); - } - - if (orderByTypes.HasFlag(OrderByTypes.Object)) - { - expected = expected.Concat(insertedDocs.Where(x => x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosObject value))); - } - - // Order using the mock order by comparer - if (isDesc) - { - expected = expected.OrderByDescending(x => - { - if (!x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosElement cosmosElement)) - { - cosmosElement = null; - } - - return cosmosElement; - }, MockOrderByComparer.Value); - } - else - { - expected = expected.OrderBy(x => - { - if (!x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosElement cosmosElement)) - { - cosmosElement = null; - } - - return cosmosElement; - }, MockOrderByComparer.Value); - } - - Assert.IsTrue( - expected.SequenceEqual(actualFromQueryWithoutContinutionTokens, CosmosElementEqualityComparer.Value), - $@" queryWithoutContinuations: {query}, - expected:{JsonConvert.SerializeObject(expected)}, - actual: {JsonConvert.SerializeObject(actualFromQueryWithoutContinutionTokens)}"); - - // Can't assert for reasons mentioned above - //Assert.IsTrue( - // expected.SequenceEqual(actualFromQueryWithContinutionTokens, DistinctMapTests.JsonTokenEqualityComparer.Value), - // $@" queryWithContinuations: {query}, - // expected:{JsonConvert.SerializeObject(expected)}, - // actual: {JsonConvert.SerializeObject(actualFromQueryWithContinutionTokens)}"); - } - } - } - - [TestMethod] - public async Task TestQueryCrossPartitionTopOrderBy() - { - int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; - uint numberOfDocuments = 1000; - string partitionKey = "field_0"; - - QueryOracle.QueryOracleUtil util = new QueryOracle.QueryOracle2(seed); - IEnumerable documents = util.GetDocuments(numberOfDocuments); - - await this.CreateIngestQueryDelete( - ConnectionModes.Direct | ConnectionModes.Gateway, - CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, - documents, - this.TestQueryCrossPartitionTopOrderByHelper, - partitionKey, - "/" + partitionKey); - } - - private async Task TestQueryCrossPartitionTopOrderByHelper(Container container, IEnumerable documents, string testArg) - { - string partitionKey = testArg; - IDictionary idToRangeMinKeyMap = new Dictionary(); - IRoutingMapProvider routingMapProvider = await this.Client.DocumentClient.GetPartitionKeyRangeCacheAsync(); - - ContainerProperties containerSettings = await container.ReadContainerAsync(); - foreach (Document document in documents) - { - IReadOnlyList targetRanges = await routingMapProvider.TryGetOverlappingRangesAsync( - containerSettings.ResourceId, - Range.GetPointRange( - PartitionKeyInternal.FromObjectArray( - new object[] - { - document.GetValue(partitionKey) - }, - true).GetEffectivePartitionKeyString(containerSettings.PartitionKey))); - Debug.Assert(targetRanges.Count == 1); - idToRangeMinKeyMap.Add(document.Id, targetRanges[0].MinInclusive); - } - - IList partitionKeyValues = new HashSet(documents.Select(doc => doc.GetValue(partitionKey))).ToList(); - - // Test Empty Results - List expectedResults = new List { }; - List computedResults = new List(); - - string emptyQueryText = @"SELECT TOP 5 * FROM Root r WHERE r.partitionKey = 9991123 OR r.partitionKey = 9991124 OR r.partitionKey = 99991125"; - List queryEmptyResult = await CrossPartitionQueryTests.RunQueryAsync( - container, - emptyQueryText); - - computedResults = queryEmptyResult.Select(doc => doc.Id).ToList(); - computedResults.Sort(); - expectedResults.Sort(); - - Random rand = new Random(); - Assert.AreEqual(string.Join(",", expectedResults), string.Join(",", computedResults)); - List tasks = new List(); - for (int trial = 0; trial < 1; ++trial) - { - foreach (bool fanOut in new[] { true, false }) - { - foreach (bool isParametrized in new[] { true, false }) - { - foreach (bool hasTop in new[] { false, true }) - { - foreach (bool hasOrderBy in new[] { false, true }) - { - foreach (string sortOrder in new[] { string.Empty, "ASC", "DESC" }) - { - #region Expected Documents - string topValueName = "@topValue"; - int top = rand.Next(4) * rand.Next(partitionKeyValues.Count); - string queryText; - string orderByField = "field_" + rand.Next(10); - IEnumerable filteredDocuments; - - string getTop() => - hasTop ? string.Format(CultureInfo.InvariantCulture, "TOP {0} ", isParametrized ? topValueName : top.ToString()) : string.Empty; - - string getOrderBy() => - hasOrderBy ? string.Format(CultureInfo.InvariantCulture, " ORDER BY r.{0} {1}", orderByField, sortOrder) : string.Empty; - - if (fanOut) - { - queryText = string.Format( - CultureInfo.InvariantCulture, - "SELECT {0}r.id, r.{1} FROM r{2}", - getTop(), - partitionKey, - getOrderBy()); - - filteredDocuments = documents; - } - else - { - HashSet selectedPartitionKeyValues = new HashSet(partitionKeyValues - .OrderBy(x => rand.Next()) - .ThenBy(x => x) - .Take(rand.Next(1, Math.Min(100, partitionKeyValues.Count) + 1))); - - queryText = string.Format( - CultureInfo.InvariantCulture, - "SELECT {0}r.id, r.{1} FROM r WHERE r.{2} IN ({3}){4}", - getTop(), - partitionKey, - partitionKey, - string.Join(", ", selectedPartitionKeyValues), - getOrderBy()); - - filteredDocuments = documents - .AsParallel() - .Where(doc => selectedPartitionKeyValues.Contains(doc.GetValue(partitionKey))); - } - - if (hasOrderBy) - { - switch (sortOrder) - { - case "": - case "ASC": - filteredDocuments = filteredDocuments - .AsParallel() - .OrderBy(doc => doc.GetValue(orderByField)) - .ThenBy(doc => idToRangeMinKeyMap[doc.Id]) - .ThenBy(doc => int.Parse(doc.Id, CultureInfo.InvariantCulture)); - break; - case "DESC": - filteredDocuments = filteredDocuments - .AsParallel() - .OrderByDescending(doc => doc.GetValue(orderByField)) - .ThenBy(doc => idToRangeMinKeyMap[doc.Id]) - .ThenByDescending(doc => int.Parse(doc.Id, CultureInfo.InvariantCulture)); - break; - } - } - else - { - filteredDocuments = filteredDocuments - .AsParallel() - .OrderBy(doc => idToRangeMinKeyMap[doc.Id]) - .ThenBy(doc => int.Parse(doc.Id, CultureInfo.InvariantCulture)); - } - - if (hasTop) - { - filteredDocuments = filteredDocuments.Take(top); - } - #endregion - #region Actual Documents - IEnumerable actualDocuments; - - int maxDegreeOfParallelism = hasTop ? rand.Next(4) : (rand.Next(2) == 0 ? -1 : (1 + rand.Next(0, 10))); - int? maxItemCount = rand.Next(2) == 0 ? -1 : rand.Next(1, documents.Count()); - QueryRequestOptions feedOptions = new QueryRequestOptions - { - MaxBufferedItemCount = rand.Next(2) == 0 ? -1 : rand.Next(Math.Min(100, documents.Count()), documents.Count() + 1), - MaxConcurrency = maxDegreeOfParallelism - }; - - if (rand.Next(3) == 0) - { - maxItemCount = null; - } - - QueryDefinition querySpec = new QueryDefinition(queryText); - SqlParameterCollection parameters = new SqlParameterCollection(); - if (isParametrized) - { - if (hasTop) - { - querySpec.WithParameter(topValueName, top); - } - } - - DateTime startTime = DateTime.Now; - List result = new List(); - FeedIterator query = container.GetItemQueryIterator( - querySpec, - requestOptions: feedOptions); - - while (query.HasMoreResults) - { - FeedResponse response = await query.ReadNextAsync(); - result.AddRange(response); - } - - actualDocuments = result; - - #endregion - - double time = (DateTime.Now - startTime).TotalMilliseconds; - - Trace.TraceInformation(": {0}, : {1}, : {2}, : {3}, : {4},