diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/OrderByItem.cs b/Microsoft.Azure.Cosmos/src/Query/Core/OrderByItem.cs index 1d0887dd3e..0853ac2e47 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/OrderByItem.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/OrderByItem.cs @@ -45,7 +45,7 @@ public CosmosElement Item { if (!this.cosmosObject.TryGetValue(ItemName, out CosmosElement cosmosElement)) { - throw new InvalidOperationException($"Underlying object does not have an 'item' field."); + cosmosElement = null; } return cosmosElement; diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/OrderByQuery/OrderByConsumeComparer.cs b/Microsoft.Azure.Cosmos/src/Query/Core/OrderByQuery/OrderByConsumeComparer.cs index 831ef8a35a..9fb5b7d129 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/OrderByQuery/OrderByConsumeComparer.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/OrderByQuery/OrderByConsumeComparer.cs @@ -6,9 +6,6 @@ namespace Microsoft.Azure.Cosmos.Query.ParallelQuery using System; using System.Collections.Generic; using System.Diagnostics; - using System.Globalization; - using Microsoft.Azure.Cosmos.CosmosElements; - using RMResources = Documents.RMResources; /// /// For cross partition order by queries we serve documents from the partition @@ -17,12 +14,6 @@ namespace Microsoft.Azure.Cosmos.Query.ParallelQuery /// internal sealed class OrderByConsumeComparer : IComparer { - /// - /// This flag used to determine whether we should support mixed type order by. - /// For testing purposes we might turn it on to test mixed type order by on index v2. - /// - public static bool AllowMixedTypeOrderByTestFlag = false; - /// /// The sort orders for the query (1 for each order by in the query). /// Until composite indexing is released this will just be an array of length 1. @@ -132,11 +123,6 @@ public int CompareOrderByItems(IList items1, IList ite this.sortOrders.Count == items1.Count, "SortOrders must match size of order-by items."); - if (!AllowMixedTypeOrderByTestFlag) - { - this.CheckTypeMatching(items1, items2); - } - for (int i = 0; i < this.sortOrders.Count; ++i) { int cmp = ItemComparer.Instance.Compare( @@ -151,39 +137,5 @@ public int CompareOrderByItems(IList items1, IList ite return 0; } - - /// - /// With index V1 collections we have the check the types of the items since it is impossible to support mixed typed order by for V1 collections. - /// The reason for this is, since V1 does not order types. - /// The only constraint is that all the numbers will be sorted with respect to each other and same for the strings, but strings and numbers might get interleaved. - /// Take the following example: - /// Partition 1: "A", 1, "B", 2 - /// Partition 2: 42, "Z", 0x5F3759DF - /// Step 1: Compare "A" to 42 and WLOG 42 comes first - /// Step 2: Compare "A" to "Z" and "A" comes first - /// Step 3: Compare "Z" to 1 and WLOG 1 comes first - /// Whoops: We have 42, "A", 1 and 1 should come before 42. - /// - /// The items relevant to the sort for the first partition. - /// The items relevant to the sort for the second partition. - private void CheckTypeMatching(IList items1, IList items2) - { - for (int i = 0; i < items1.Count; ++i) - { - CosmosElementType itemType1 = items1[i].Item.Type; - CosmosElementType itemType2 = items2[i].Item.Type; - - if (itemType1 != itemType2) - { - throw new NotSupportedException( - string.Format( - CultureInfo.InvariantCulture, - RMResources.UnsupportedCrossPartitionOrderByQueryOnMixedTypes, - itemType1, - itemType2, - items1[i])); - } - } - } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.cs index 2432bf0f4f..9e818220d4 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.cs @@ -2750,23 +2750,6 @@ public async Task TestMixedTypeOrderBy() } } - // Just have range indexes - Cosmos.IndexingPolicy indexV1Policy = new Cosmos.IndexingPolicy() - { - IncludedPaths = new Collection() - { - new Cosmos.IncludedPath() - { - Path = "/*", - Indexes = new Collection() - { - Cosmos.Index.Range(Cosmos.DataType.String, -1), - Cosmos.Index.Range(Cosmos.DataType.Number, -1), - } - } - } - }; - // Add a composite index to force an index v2 container to be made. Cosmos.IndexingPolicy indexV2Policy = new Cosmos.IndexingPolicy() { @@ -2775,6 +2758,10 @@ public async Task TestMixedTypeOrderBy() new Cosmos.IncludedPath() { Path = "/*", + }, + new Cosmos.IncludedPath() + { + Path = $"/{nameof(MixedTypedDocument.MixedTypeField)}/?", } }, @@ -2795,92 +2782,30 @@ public async Task TestMixedTypeOrderBy() } }; - string indexV2Api = HttpConstants.Versions.v2018_09_17; - string indexV1Api = HttpConstants.Versions.v2017_11_15; - - async Task runWithAllowMixedTypeOrderByFlag(bool allowMixedTypeOrderByTestFlag, OrderByTypes[] orderByTypes, Action expectedExcpetionHandler) - { - bool allowMixedTypeOrderByTestFlagOriginalValue = OrderByConsumeComparer.AllowMixedTypeOrderByTestFlag; - string apiVersion = allowMixedTypeOrderByTestFlag ? indexV2Api : indexV1Api; - Cosmos.IndexingPolicy indexingPolicy = allowMixedTypeOrderByTestFlag ? indexV2Policy : indexV1Policy; - try - { - OrderByConsumeComparer.AllowMixedTypeOrderByTestFlag = allowMixedTypeOrderByTestFlag; - await this.RunWithApiVersion( - apiVersion, - async () => - { - await this.CreateIngestQueryDelete>>( - ConnectionModes.Direct, - CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, - documents, - this.TestMixedTypeOrderByHelper, - new Tuple>(orderByTypes, expectedExcpetionHandler), - "/id", - indexingPolicy); - }); - } - finally - { - OrderByConsumeComparer.AllowMixedTypeOrderByTestFlag = allowMixedTypeOrderByTestFlagOriginalValue; - } - } - - bool dontAllowMixedTypes = false; - bool doAllowMixedTypes = true; - OrderByTypes primitives = OrderByTypes.Bool | OrderByTypes.Null | OrderByTypes.Number | OrderByTypes.String; OrderByTypes nonPrimitives = OrderByTypes.Array | OrderByTypes.Object; OrderByTypes all = primitives | nonPrimitives | OrderByTypes.Undefined; - // Don't allow mixed types but single type order by should still work - await runWithAllowMixedTypeOrderByFlag( - dontAllowMixedTypes, - new OrderByTypes[] - { - OrderByTypes.Array, - OrderByTypes.Bool, - OrderByTypes.Null, - OrderByTypes.Number, - OrderByTypes.Object, - OrderByTypes.String, - OrderByTypes.Undefined, - }, null); - - // If you don't allow mixed types but you run a mixed type query then you should get an exception or the results are just wrong. - await runWithAllowMixedTypeOrderByFlag( - dontAllowMixedTypes, - new OrderByTypes[] - { - all, - primitives, - }, - (exception) => - { - Assert.IsTrue( - // Either we get the weird client exception for having mixed types - exception.Message.Contains("Cannot execute cross partition order-by queries on mix types.") - // Or the results are just messed up since the pages in isolation were not mixed typed. - || exception.GetType() == typeof(AssertFailedException)); - }); - - // Mixed type orderby should work for all scenarios, - // since for now the non primitives are accepted to not be served from the index. - await runWithAllowMixedTypeOrderByFlag( - doAllowMixedTypes, - new OrderByTypes[] - { - OrderByTypes.Array, - OrderByTypes.Bool, - OrderByTypes.Null, - OrderByTypes.Number, - OrderByTypes.Object, - OrderByTypes.String, - OrderByTypes.Undefined, - primitives, - nonPrimitives, - all, - }, null); + await this.CreateIngestQueryDelete( + ConnectionModes.Direct, + CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, + documents, + this.TestMixedTypeOrderByHelper, + new OrderByTypes[] + { + OrderByTypes.Array, + OrderByTypes.Bool, + OrderByTypes.Null, + OrderByTypes.Number, + OrderByTypes.Object, + OrderByTypes.String, + OrderByTypes.Undefined, + primitives, + nonPrimitives, + all, + }, + "/id", + indexV2Policy); } private sealed class MixedTypedDocument @@ -2926,24 +2851,15 @@ private static CosmosElement GenerateRandomJsonValue(Random random) } } - private sealed class MockOrderByComparer : IComparer + private sealed class MockOrderByComparer : IComparer { public static readonly MockOrderByComparer Value = new MockOrderByComparer(); - public int Compare(object x, object y) + public int Compare(CosmosElement element1, CosmosElement element2) { - CosmosElement element1 = ObjectToCosmosElement(x); - CosmosElement element2 = ObjectToCosmosElement(y); - return ItemComparer.Instance.Compare(element1, element2); } - private static CosmosElement ObjectToCosmosElement(object obj) - { - string json = JsonConvert.SerializeObject(obj != null ? JToken.FromObject(obj) : JValue.CreateNull()); - byte[] bytes = Encoding.UTF8.GetBytes(json); - return CosmosElement.CreateFromBuffer(bytes); - } } [Flags] @@ -2961,73 +2877,70 @@ private enum OrderByTypes private async Task TestMixedTypeOrderByHelper( Container container, IEnumerable documents, - Tuple> args) + OrderByTypes[] args) { - OrderByTypes[] orderByTypesList = args.Item1; - Action expectedExceptionHandler = args.Item2; - try + OrderByTypes[] orderByTypesList = args; + foreach (bool isDesc in new bool[] { true, false }) { - foreach (bool isDesc in new bool[] { true, false }) + foreach (OrderByTypes orderByTypes in orderByTypesList) { - foreach (OrderByTypes orderByTypes in orderByTypesList) + string orderString = isDesc ? "DESC" : "ASC"; + List mixedTypeFilters = new List(); + if (orderByTypes.HasFlag(OrderByTypes.Array)) { - string orderString = isDesc ? "DESC" : "ASC"; - List mixedTypeFilters = new List(); - if (orderByTypes.HasFlag(OrderByTypes.Array)) - { - mixedTypeFilters.Add($"IS_ARRAY(c.{nameof(MixedTypedDocument.MixedTypeField)})"); - } + mixedTypeFilters.Add($"IS_ARRAY(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } - if (orderByTypes.HasFlag(OrderByTypes.Bool)) - { - mixedTypeFilters.Add($"IS_BOOL(c.{nameof(MixedTypedDocument.MixedTypeField)})"); - } + if (orderByTypes.HasFlag(OrderByTypes.Bool)) + { + mixedTypeFilters.Add($"IS_BOOL(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } - if (orderByTypes.HasFlag(OrderByTypes.Null)) - { - mixedTypeFilters.Add($"IS_NULL(c.{nameof(MixedTypedDocument.MixedTypeField)})"); - } + if (orderByTypes.HasFlag(OrderByTypes.Null)) + { + mixedTypeFilters.Add($"IS_NULL(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } - if (orderByTypes.HasFlag(OrderByTypes.Number)) - { - mixedTypeFilters.Add($"IS_NUMBER(c.{nameof(MixedTypedDocument.MixedTypeField)})"); - } + if (orderByTypes.HasFlag(OrderByTypes.Number)) + { + mixedTypeFilters.Add($"IS_NUMBER(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } - if (orderByTypes.HasFlag(OrderByTypes.Object)) - { - mixedTypeFilters.Add($"IS_OBJECT(c.{nameof(MixedTypedDocument.MixedTypeField)})"); - } + if (orderByTypes.HasFlag(OrderByTypes.Object)) + { + mixedTypeFilters.Add($"IS_OBJECT(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } - if (orderByTypes.HasFlag(OrderByTypes.String)) - { - mixedTypeFilters.Add($"IS_STRING(c.{nameof(MixedTypedDocument.MixedTypeField)})"); - } + if (orderByTypes.HasFlag(OrderByTypes.String)) + { + mixedTypeFilters.Add($"IS_STRING(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } - if (orderByTypes.HasFlag(OrderByTypes.Undefined)) - { - mixedTypeFilters.Add($"not IS_DEFINED(c.{nameof(MixedTypedDocument.MixedTypeField)})"); - } + if (orderByTypes.HasFlag(OrderByTypes.Undefined)) + { + mixedTypeFilters.Add($"not IS_DEFINED(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } - string filter = mixedTypeFilters.Count() == 0 ? "true" : string.Join(" OR ", mixedTypeFilters); + string filter = mixedTypeFilters.Count() == 0 ? "true" : string.Join(" OR ", mixedTypeFilters); - string query = $@" - SELECT VALUE c.{nameof(MixedTypedDocument.MixedTypeField)} + string query = $@" + SELECT c.{nameof(MixedTypedDocument.MixedTypeField)} FROM c WHERE {filter} ORDER BY c.{nameof(MixedTypedDocument.MixedTypeField)} {orderString}"; - QueryRequestOptions feedOptions = new QueryRequestOptions() - { - MaxBufferedItemCount = 1000, - MaxItemCount = 16, - MaxConcurrency = 10, - }; + QueryRequestOptions feedOptions = new QueryRequestOptions() + { + MaxBufferedItemCount = 1000, + MaxItemCount = 16, + MaxConcurrency = 10, + }; - List actualFromQueryWithoutContinutionTokens; - actualFromQueryWithoutContinutionTokens = await CrossPartitionQueryTests.QueryWithoutContinuationTokens( - container, - query, - queryRequestOptions: feedOptions); + List actualFromQueryWithoutContinutionTokens; + actualFromQueryWithoutContinutionTokens = await CrossPartitionQueryTests.QueryWithoutContinuationTokens( + container, + query, + queryRequestOptions: feedOptions); #if false For now we can not serve the query through continuation tokens correctly. This is because we allow order by on mixed types but not comparisions across types @@ -3049,78 +2962,97 @@ SELECT c.MixedTypeField FROM c WHERE c.MixedTypeField > 303093052 ORDER BY c.Mix and that is because comparision across types is undefined so "aaaaaaaaaaa" > 303093052 never got emitted #endif - IEnumerable insertedDocs = documents - .Select(document => (CosmosElement.CreateFromBuffer(Encoding.UTF8.GetBytes(document.ToString())) as CosmosObject)[nameof(MixedTypedDocument.MixedTypeField)]) - .Where(document => document != null); + IEnumerable insertedDocs = documents + .Select(document => CosmosElement.CreateFromBuffer(Encoding.UTF8.GetBytes(document.ToString())) as CosmosObject) + .Select(document => + { + Dictionary dictionary = new Dictionary(); + if (document.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosElement value)) + { + dictionary.Add(nameof(MixedTypedDocument.MixedTypeField), value); + } - // Build the expected results using LINQ - IEnumerable expected = new List(); + return CosmosObject.Create(dictionary); + }); - // Filter based on the mixedOrderByType enum - if (orderByTypes.HasFlag(OrderByTypes.Array)) - { - expected = expected.Concat(insertedDocs.Where(x => x?.Type == CosmosElementType.Array)); - } + // Build the expected results using LINQ + IEnumerable expected = new List(); - if (orderByTypes.HasFlag(OrderByTypes.Bool)) - { - expected = expected.Concat(insertedDocs.Where(x => x?.Type == CosmosElementType.Boolean)); - } + // Filter based on the mixedOrderByType enum - if (orderByTypes.HasFlag(OrderByTypes.Null)) - { - expected = expected.Concat(insertedDocs.Where(x => x?.Type == CosmosElementType.Null)); - } + if (orderByTypes.HasFlag(OrderByTypes.Undefined)) + { + expected = expected.Concat(insertedDocs.Where(x => !x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosElement value))); + } - if (orderByTypes.HasFlag(OrderByTypes.Number)) - { - expected = expected.Concat(insertedDocs.Where(x => x?.Type == CosmosElementType.Number)); - } + if (orderByTypes.HasFlag(OrderByTypes.Null)) + { + expected = expected.Concat(insertedDocs.Where(x => x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosNull value))); + } - if (orderByTypes.HasFlag(OrderByTypes.Object)) - { - expected = expected.Concat(insertedDocs.Where(x => x?.Type == CosmosElementType.Object)); - } + if (orderByTypes.HasFlag(OrderByTypes.Bool)) + { + expected = expected.Concat(insertedDocs.Where(x => x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosBoolean value))); + } - if (orderByTypes.HasFlag(OrderByTypes.String)) - { - expected = expected.Concat(insertedDocs.Where(x => x?.Type == CosmosElementType.String)); - } + if (orderByTypes.HasFlag(OrderByTypes.Number)) + { + expected = expected.Concat(insertedDocs.Where(x => x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosNumber value))); + } - // Order using the mock order by comparer - if (isDesc) + if (orderByTypes.HasFlag(OrderByTypes.String)) + { + expected = expected.Concat(insertedDocs.Where(x => x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosString value))); + } + + if (orderByTypes.HasFlag(OrderByTypes.Array)) + { + expected = expected.Concat(insertedDocs.Where(x => x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosArray value))); + } + + if (orderByTypes.HasFlag(OrderByTypes.Object)) + { + expected = expected.Concat(insertedDocs.Where(x => x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosObject value))); + } + + // Order using the mock order by comparer + if (isDesc) + { + expected = expected.OrderByDescending(x => { - expected = expected.OrderByDescending(x => x, MockOrderByComparer.Value); - } - else + if (!x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosElement cosmosElement)) + { + cosmosElement = null; + } + + return cosmosElement; + }, MockOrderByComparer.Value); + } + else + { + expected = expected.OrderBy(x => { - expected = expected.OrderBy(x => x, MockOrderByComparer.Value); - } + if (!x.TryGetValue(nameof(MixedTypedDocument.MixedTypeField), out CosmosElement cosmosElement)) + { + cosmosElement = null; + } - Assert.IsTrue( - expected.SequenceEqual(actualFromQueryWithoutContinutionTokens, CosmosElementEqualityComparer.Value), - $@" queryWithoutContinuations: {query}, + return cosmosElement; + }, MockOrderByComparer.Value); + } + + Assert.IsTrue( + expected.SequenceEqual(actualFromQueryWithoutContinutionTokens, CosmosElementEqualityComparer.Value), + $@" queryWithoutContinuations: {query}, expected:{JsonConvert.SerializeObject(expected)}, actual: {JsonConvert.SerializeObject(actualFromQueryWithoutContinutionTokens)}"); - // Can't assert for reasons mentioned above - //Assert.IsTrue( - // expected.SequenceEqual(actualFromQueryWithContinutionTokens, DistinctMapTests.JsonTokenEqualityComparer.Value), - // $@" queryWithContinuations: {query}, - // expected:{JsonConvert.SerializeObject(expected)}, - // actual: {JsonConvert.SerializeObject(actualFromQueryWithContinutionTokens)}"); - } - } - } - catch (Exception ex) - { - if (expectedExceptionHandler != null) - { - expectedExceptionHandler(ex); - } - else - { - throw; + // Can't assert for reasons mentioned above + //Assert.IsTrue( + // expected.SequenceEqual(actualFromQueryWithContinutionTokens, DistinctMapTests.JsonTokenEqualityComparer.Value), + // $@" queryWithContinuations: {query}, + // expected:{JsonConvert.SerializeObject(expected)}, + // actual: {JsonConvert.SerializeObject(actualFromQueryWithContinutionTokens)}"); } } } diff --git a/changelog.md b/changelog.md index 72d32c9e0e..c5869052ce 100644 --- a/changelog.md +++ b/changelog.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +- [#952](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/952) ORDER BY Undefined and Mixed Type ORDER BY support. + ### Added ### Fixed