From 7ed22ac55b569354ea94733ea5ab4601d20dd586 Mon Sep 17 00:00:00 2001 From: Kevin Pilch Date: Fri, 6 Oct 2023 16:36:44 -0700 Subject: [PATCH 1/5] [Internal] Build: Adds CodeQL support in nightly builds (#4113) * Update azure-pipelines-nightly.yml * Newlines in variables sections --- azure-pipelines-nightly.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/azure-pipelines-nightly.yml b/azure-pipelines-nightly.yml index 04a53968aa..f0f7f3faea 100644 --- a/azure-pipelines-nightly.yml +++ b/azure-pipelines-nightly.yml @@ -16,10 +16,13 @@ variables: VmImage: windows-latest # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops BuildConfiguration: Release Packaging.EnableSBOMSigning: true + Codeql.Enabled: true stages: - stage: displayName: Generate nightly GA + variables: + Codeql.BuildIdentifier: ga jobs: - template: templates/nuget-pack.yml parameters: @@ -32,6 +35,8 @@ stages: CleanupFolder: true - stage: displayName: Generate nightly preview + variables: + Codeql.BuildIdentifier: preview jobs: - template: templates/nuget-pack.yml parameters: @@ -41,4 +46,4 @@ stages: ReleasePackage: true OutputPath: '$(Build.ArtifactStagingDirectory)/bin/AnyCPU/$(BuildConfiguration)/Microsoft.Azure.Cosmos' BlobVersion: 'nightly-preview' - CleanupFolder: true \ No newline at end of file + CleanupFolder: true From 75eec5c33abff8fb07886af917d661f0963b452f Mon Sep 17 00:00:00 2001 From: Sourabh Jain Date: Mon, 9 Oct 2023 20:29:11 +0530 Subject: [PATCH 2/5] Benchmark: Fixes to show estimated cost of a container only when new container is getting created (#4109) * Showing Estimated Cost only when new container is getting created * read container to get container response * disable client telemetry by default * removed unused imports * resolve merge conflict * fixed name * fix container not found * removed the message * Update Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs Co-authored-by: Matias Quaranta * removed line space --------- Co-authored-by: Kiran Kumar Kolli Co-authored-by: Matias Quaranta --- .../Tools/Benchmark/BenchmarkConfig.cs | 12 ++++++------ .../Tools/Benchmark/Program.cs | 12 +----------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs index 0fde6a8926..2c523374bf 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs @@ -103,8 +103,8 @@ public class BenchmarkConfig [Option(Required = false, HelpText = "Disable core SDK logging")] public bool DisableCoreSdkLogging { get; set; } - [Option(Required = false, HelpText = "Disable Distributed Tracing feature from source")] - public bool DisableDistributedTracing { get; set; } = false; + [Option(Required = false, HelpText = "Enable Distributed Tracing")] + public bool EnableDistributedTracing { get; set; } = false; [Option(Required = false, HelpText = "Client Telemetry Schedule in Seconds")] public int TelemetryScheduleInSec { get; set; } @@ -138,8 +138,8 @@ public class BenchmarkConfig [Option(Required = false, HelpText = "Application Insights connection string")] public string AppInsightsConnectionString { get; set; } - [Option(Required = false, HelpText = "Disable Client Telemetry Feature in SDK. Make sure you enable it from the portal also.")] - public bool DisableClientTelemetry { get; set; } = false; + [Option(Required = false, HelpText = "Enable Client Telemetry Feature in SDK. Make sure you enable it from the portal also.")] + public bool EnableTelemetry { get; set; } = false; internal int GetTaskCount(int containerThroughput) { @@ -222,8 +222,8 @@ internal Microsoft.Azure.Cosmos.CosmosClient CreateCosmosClient(string accountKe MaxTcpConnectionsPerEndpoint = this.MaxTcpConnectionsPerEndpoint, CosmosClientTelemetryOptions = new Microsoft.Azure.Cosmos.CosmosClientTelemetryOptions() { - DisableSendingMetricsToService = this.DisableClientTelemetry, - DisableDistributedTracing = this.DisableDistributedTracing + DisableSendingMetricsToService = !this.EnableTelemetry, + DisableDistributedTracing = !this.EnableDistributedTracing } }; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs index 9bbb33b9fd..fde9d5cd2c 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs @@ -7,10 +7,8 @@ namespace CosmosBenchmark using System; using System.Collections.Generic; using System.Diagnostics; - using System.Diagnostics.Tracing; using System.IO; using System.Linq; - using System.Net; using System.Net.Http; using System.Reflection; using System.Threading; @@ -18,7 +16,6 @@ namespace CosmosBenchmark using Azure.Monitor.OpenTelemetry.Exporter; using CosmosBenchmark.Fx; using Microsoft.Azure.Cosmos; - using Microsoft.Extensions.Logging; using Newtonsoft.Json.Linq; using OpenTelemetry; using OpenTelemetry.Metrics; @@ -323,14 +320,7 @@ private static Type[] AvailableBenchmarks() private static async Task CreatePartitionedContainerAsync(BenchmarkConfig options, CosmosClient cosmosClient) { Microsoft.Azure.Cosmos.Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync(options.Database); - - // Show user cost of running this test - double estimatedCostPerMonth = 0.06 * options.Throughput; - double estimatedCostPerHour = estimatedCostPerMonth / (24 * 30); - Utility.TeeTraceInformation($"The container will cost an estimated ${Math.Round(estimatedCostPerHour, 2)} per hour (${Math.Round(estimatedCostPerMonth, 2)} per month)"); - Utility.TeeTraceInformation("Press enter to continue ..."); - Console.ReadLine(); - + string partitionKeyPath = options.PartitionKeyPath; return await database.CreateContainerIfNotExistsAsync(options.Container, partitionKeyPath, options.Throughput); } From a5dec66060a63572ac09d2c4943dc9b218650a70 Mon Sep 17 00:00:00 2001 From: Sourabh Jain Date: Tue, 10 Oct 2023 00:08:05 +0530 Subject: [PATCH 3/5] Distributed Tracing: Fixes SDK responses compatibility with opentelemetry response (#4097) * adding tets * wip * wip2 * fix code * add tests * fix test * fix test * remove consoles * fix indent and remove unused imports * internal to private rollback * added docs * removed unused imports * added exception in message * fix exception catching --- .../OpenTelemetryCoreRecorder.cs | 2 +- .../OpenTelemetry/OpenTelemetryResponse.cs | 52 +++-- .../OpenTelemetry/OpenTelemetryResponse{T}.cs | 56 +++-- .../Telemetry/OpenTelemetryRecorderTests.cs | 201 ++++++++++++++++++ 4 files changed, 284 insertions(+), 27 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Telemetry/OpenTelemetry/OpenTelemetryCoreRecorder.cs b/Microsoft.Azure.Cosmos/src/Telemetry/OpenTelemetry/OpenTelemetryCoreRecorder.cs index 3679d7c7bd..2928685d97 100644 --- a/Microsoft.Azure.Cosmos/src/Telemetry/OpenTelemetry/OpenTelemetryCoreRecorder.cs +++ b/Microsoft.Azure.Cosmos/src/Telemetry/OpenTelemetry/OpenTelemetryCoreRecorder.cs @@ -223,7 +223,7 @@ Documents.OperationType operationType this.scope.AddAttribute(OpenTelemetryAttributeKeys.RequestContentLength, this.response.RequestContentLength); this.scope.AddAttribute(OpenTelemetryAttributeKeys.ResponseContentLength, this.response.ResponseContentLength); this.scope.AddAttribute(OpenTelemetryAttributeKeys.StatusCode, (int)this.response.StatusCode); - this.scope.AddAttribute(OpenTelemetryAttributeKeys.SubStatusCode, (int)this.response.SubStatusCode); + this.scope.AddAttribute(OpenTelemetryAttributeKeys.SubStatusCode, this.response.SubStatusCode); this.scope.AddAttribute(OpenTelemetryAttributeKeys.RequestCharge, this.response.RequestCharge); this.scope.AddAttribute(OpenTelemetryAttributeKeys.ItemCount, this.response.ItemCount); this.scope.AddAttribute(OpenTelemetryAttributeKeys.ActivityId, this.response.ActivityId); diff --git a/Microsoft.Azure.Cosmos/src/Telemetry/OpenTelemetry/OpenTelemetryResponse.cs b/Microsoft.Azure.Cosmos/src/Telemetry/OpenTelemetry/OpenTelemetryResponse.cs index c9aba10ee6..09d43c6965 100644 --- a/Microsoft.Azure.Cosmos/src/Telemetry/OpenTelemetry/OpenTelemetryResponse.cs +++ b/Microsoft.Azure.Cosmos/src/Telemetry/OpenTelemetry/OpenTelemetryResponse.cs @@ -4,8 +4,10 @@ namespace Microsoft.Azure.Cosmos { + using System; using System.IO; using System.Net; + using Microsoft.Azure.Cosmos.Core.Trace; using Telemetry; internal sealed class OpenTelemetryResponse : OpenTelemetryAttributes @@ -13,28 +15,28 @@ internal sealed class OpenTelemetryResponse : OpenTelemetryAttributes internal OpenTelemetryResponse(TransactionalBatchResponse responseMessage) : this( statusCode: responseMessage.StatusCode, - requestCharge: responseMessage.Headers?.RequestCharge, + requestCharge: OpenTelemetryResponse.GetHeader(responseMessage)?.RequestCharge, responseContentLength: null, diagnostics: responseMessage.Diagnostics, - itemCount: responseMessage.Headers?.ItemCount, + itemCount: OpenTelemetryResponse.GetHeader(responseMessage)?.ItemCount, requestMessage: null, - subStatusCode: (int)responseMessage.Headers?.SubStatusCode, - activityId: responseMessage.Headers?.ActivityId, - correlationId: responseMessage.Headers?.CorrelatedActivityId) + subStatusCode: OpenTelemetryResponse.GetHeader(responseMessage)?.SubStatusCode, + activityId: OpenTelemetryResponse.GetHeader(responseMessage)?.ActivityId, + correlationId: OpenTelemetryResponse.GetHeader(responseMessage)?.CorrelatedActivityId) { } internal OpenTelemetryResponse(ResponseMessage responseMessage) : this( statusCode: responseMessage.StatusCode, - requestCharge: responseMessage.Headers?.RequestCharge, + requestCharge: OpenTelemetryResponse.GetHeader(responseMessage)?.RequestCharge, responseContentLength: OpenTelemetryResponse.GetPayloadSize(responseMessage), diagnostics: responseMessage.Diagnostics, - itemCount: responseMessage.Headers?.ItemCount, + itemCount: OpenTelemetryResponse.GetHeader(responseMessage)?.ItemCount, requestMessage: responseMessage.RequestMessage, - subStatusCode: (int)responseMessage.Headers?.SubStatusCode, - activityId: responseMessage.Headers?.ActivityId, - correlationId: responseMessage.Headers?.CorrelatedActivityId, + subStatusCode: OpenTelemetryResponse.GetHeader(responseMessage)?.SubStatusCode, + activityId: OpenTelemetryResponse.GetHeader(responseMessage)?.ActivityId, + correlationId: OpenTelemetryResponse.GetHeader(responseMessage)?.CorrelatedActivityId, operationType: responseMessage is QueryResponse ? Documents.OperationType.Query : Documents.OperationType.Invalid ) { @@ -47,7 +49,7 @@ private OpenTelemetryResponse( CosmosDiagnostics diagnostics, string itemCount, RequestMessage requestMessage, - int subStatusCode, + Documents.SubStatusCodes? subStatusCode, string activityId, string correlationId, Documents.OperationType operationType = Documents.OperationType.Invalid) @@ -58,7 +60,7 @@ private OpenTelemetryResponse( this.ResponseContentLength = responseContentLength; this.Diagnostics = diagnostics; this.ItemCount = itemCount; - this.SubStatusCode = subStatusCode; + this.SubStatusCode = (int)(subStatusCode ?? Documents.SubStatusCodes.Unknown); this.ActivityId = activityId; this.CorrelatedActivityId = correlationId; this.OperationType = operationType; @@ -74,5 +76,31 @@ private static string GetPayloadSize(ResponseMessage response) } return response?.Headers?.ContentLength; } + + private static Headers GetHeader(TransactionalBatchResponse responseMessage) + { + try + { + return responseMessage?.Headers; + } + catch (NotImplementedException ex) + { + DefaultTrace.TraceVerbose("Failed to get headers from TransactionalBatchResponse. Exception: {0}", ex); + return null; + } + } + + private static Headers GetHeader(ResponseMessage responseMessage) + { + try + { + return responseMessage?.Headers; + } + catch (NotImplementedException ex) + { + DefaultTrace.TraceVerbose("Failed to get headers from ResponseMessage. Exception: {0}", ex); + return null; + } + } } } diff --git a/Microsoft.Azure.Cosmos/src/Telemetry/OpenTelemetry/OpenTelemetryResponse{T}.cs b/Microsoft.Azure.Cosmos/src/Telemetry/OpenTelemetry/OpenTelemetryResponse{T}.cs index 4c96306a30..976b68173a 100644 --- a/Microsoft.Azure.Cosmos/src/Telemetry/OpenTelemetry/OpenTelemetryResponse{T}.cs +++ b/Microsoft.Azure.Cosmos/src/Telemetry/OpenTelemetry/OpenTelemetryResponse{T}.cs @@ -4,7 +4,9 @@ namespace Microsoft.Azure.Cosmos { + using System; using System.Net; + using Microsoft.Azure.Cosmos.Core.Trace; using Telemetry; internal sealed class OpenTelemetryResponse : OpenTelemetryAttributes @@ -12,14 +14,14 @@ internal sealed class OpenTelemetryResponse : OpenTelemetryAttributes internal OpenTelemetryResponse(FeedResponse responseMessage) : this( statusCode: responseMessage.StatusCode, - requestCharge: responseMessage.Headers?.RequestCharge, - responseContentLength: responseMessage?.Headers?.ContentLength, + requestCharge: OpenTelemetryResponse.GetHeader(responseMessage)?.RequestCharge, + responseContentLength: OpenTelemetryResponse.GetHeader(responseMessage)?.ContentLength, diagnostics: responseMessage.Diagnostics, - itemCount: responseMessage.Headers?.ItemCount, + itemCount: OpenTelemetryResponse.GetHeader(responseMessage)?.ItemCount, requestMessage: responseMessage.RequestMessage, - subStatusCode: (int)responseMessage.Headers?.SubStatusCode, - activityId: responseMessage.Headers?.ActivityId, - correlatedActivityId: responseMessage.Headers?.CorrelatedActivityId, + subStatusCode: OpenTelemetryResponse.GetHeader(responseMessage)?.SubStatusCode, + activityId: OpenTelemetryResponse.GetHeader(responseMessage)?.ActivityId, + correlatedActivityId: OpenTelemetryResponse.GetHeader(responseMessage)?.CorrelatedActivityId, operationType: responseMessage is QueryResponse ? Documents.OperationType.Query : Documents.OperationType.Invalid) { } @@ -27,14 +29,14 @@ internal OpenTelemetryResponse(FeedResponse responseMessage) internal OpenTelemetryResponse(Response responseMessage) : this( statusCode: responseMessage.StatusCode, - requestCharge: responseMessage.Headers?.RequestCharge, - responseContentLength: responseMessage?.Headers?.ContentLength, + requestCharge: OpenTelemetryResponse.GetHeader(responseMessage)?.RequestCharge, + responseContentLength: OpenTelemetryResponse.GetHeader(responseMessage)?.ContentLength, diagnostics: responseMessage.Diagnostics, - itemCount: responseMessage.Headers?.ItemCount, + itemCount: OpenTelemetryResponse.GetHeader(responseMessage)?.ItemCount, requestMessage: responseMessage.RequestMessage, - subStatusCode: (int)responseMessage.Headers?.SubStatusCode, - activityId: responseMessage.Headers?.ActivityId, - correlatedActivityId: responseMessage.Headers?.CorrelatedActivityId, + subStatusCode: OpenTelemetryResponse.GetHeader(responseMessage)?.SubStatusCode, + activityId: OpenTelemetryResponse.GetHeader(responseMessage)?.ActivityId, + correlatedActivityId: OpenTelemetryResponse.GetHeader(responseMessage)?.CorrelatedActivityId, operationType: responseMessage is QueryResponse ? Documents.OperationType.Query : Documents.OperationType.Invalid) { } @@ -46,7 +48,7 @@ private OpenTelemetryResponse( CosmosDiagnostics diagnostics, string itemCount, RequestMessage requestMessage, - int subStatusCode, + Documents.SubStatusCodes? subStatusCode, string activityId, string correlatedActivityId, Documents.OperationType operationType) @@ -57,10 +59,36 @@ private OpenTelemetryResponse( this.ResponseContentLength = responseContentLength; this.Diagnostics = diagnostics; this.ItemCount = itemCount; - this.SubStatusCode = subStatusCode; + this.SubStatusCode = (int)(subStatusCode ?? Documents.SubStatusCodes.Unknown); this.ActivityId = activityId; this.CorrelatedActivityId = correlatedActivityId; this.OperationType = operationType; } + + private static Headers GetHeader(FeedResponse responseMessage) + { + try + { + return responseMessage?.Headers; + } + catch (NotImplementedException ex) + { + DefaultTrace.TraceWarning("Failed to get headers from FeedResponse. Exception: {0}", ex); + return null; + } + } + + private static Headers GetHeader(Response responseMessage) + { + try + { + return responseMessage?.Headers; + } + catch (NotImplementedException ex) + { + DefaultTrace.TraceWarning("Failed to get headers from Response. Exception: {0}", ex); + return null; + } + } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Telemetry/OpenTelemetryRecorderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Telemetry/OpenTelemetryRecorderTests.cs index 2095999f6c..601045cba5 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Telemetry/OpenTelemetryRecorderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Telemetry/OpenTelemetryRecorderTests.cs @@ -6,12 +6,18 @@ namespace Microsoft.Azure.Cosmos.Tests.Telemetry { using System; using System.Collections.Generic; + using System.Collections.ObjectModel; using System.Linq; + using System.Net; using System.Reflection; + using System.Threading; + using System.Threading.Tasks; using Cosmos.Telemetry; using Microsoft.Azure.Cosmos.Diagnostics; + using Microsoft.Azure.Cosmos.Scripts; using Microsoft.Azure.Cosmos.Tracing; using Microsoft.VisualStudio.TestTools.UnitTesting; + using Moq; [TestClass] public class OpenTelemetryRecorderTests @@ -43,6 +49,201 @@ public void CheckExceptionsCompatibility() } } + /// + /// This test verifies whether OpenTelemetryResponse can be initialized using a specific type of response available in the SDK. + /// If any new response is added to the SDK in the future, it must be configured in either of the following dictionaries. + /// + [TestMethod] + public async Task CheckResponseCompatibility() + { + // This list contains all the response types which are not compatible with Open Telemetry Response + IReadOnlyList excludedResponses = new List() + { + "MediaResponse", // Part of dead code + "DocumentFeedResponse`1",// Part of dead code + "CosmosQuotaResponse",// Part of dead code + "StoredProcedureResponse`1" // Not supported as of now + }; + + // This dictionary contains a Key-Value pair where the Key represents the Response Type compatible with Open Telemetry Response, and the corresponding Value is a mocked instance. + // Essentially, at some point in the code, we send an instance of the response to Open Telemetry to retrieve relevant information. + IDictionary responseInstances = new Dictionary() + { + { "PartitionKeyRangeBatchResponse", await OpenTelemetryRecorderTests.GetPartitionKeyRangeBatchResponse() }, + { "TransactionalBatchResponse", await OpenTelemetryRecorderTests.GetTransactionalBatchResponse() }, + { "QueryResponse", new Mock().Object }, + { "QueryResponse`1", OpenTelemetryRecorderTests.GetQueryResponseWithGenerics()}, + { "ReadFeedResponse`1", ReadFeedResponse.CreateResponse (new ResponseMessage(HttpStatusCode.OK),MockCosmosUtil.Serializer)}, + { "ClientEncryptionKeyResponse", new Mock().Object}, + { "ContainerResponse", new Mock().Object }, + { "ItemResponse`1", new Mock>().Object }, + { "DatabaseResponse", new Mock().Object }, + { "PermissionResponse", new Mock().Object }, + { "ThroughputResponse", new Mock().Object }, + { "UserResponse", new Mock().Object }, + { "ChangeFeedEstimatorFeedResponse", ChangefeedResponseFunc}, + { "ChangeFeedEstimatorEmptyFeedResponse", ChangeFeedEstimatorEmptyFeedResponseFunc }, + { "StoredProcedureExecuteResponse`1",new Mock>().Object }, + { "StoredProcedureResponse", new Mock().Object }, + { "TriggerResponse", new Mock().Object }, + { "UserDefinedFunctionResponse", new Mock().Object } + }; + + Assembly asm = OpenTelemetryRecorderTests.GetAssemblyLocally(DllName); + + // Get all types (including internal) defined in the assembly + IEnumerable actualClasses = asm + .GetTypes() + .Where(type => + (type.Name.EndsWith("Response") || type.Name.EndsWith("Response`1")) && // Ending with Response and Response + !type.Name.Contains("OpenTelemetryResponse") && // Excluding OpenTelemetryResponse because we are testing this class + !type.IsAbstract && // Excluding abstract classes + !type.IsInterface && // Excluding interfaces + !excludedResponses.Contains(type.Name)); // Excluding all the types defined in excludedResponses list + + foreach (Type className in actualClasses) + { + Assert.IsTrue(responseInstances.TryGetValue(className.Name, out object instance), $" New Response type found i.e.{className.Name}"); + + if (instance is TransactionalBatchResponse transactionInstance) + { + _ = new OpenTelemetryResponse(transactionInstance); + } + else if (instance is ResponseMessage responseMessageInstance) + { + _ = new OpenTelemetryResponse(responseMessageInstance); + } + else if (instance is FeedResponse feedResponseInstance) + { + _ = new OpenTelemetryResponse(feedResponseInstance); + } + else if (instance is Response responseInstance) + { + _ = new OpenTelemetryResponse(responseInstance); + } + else if (instance is Response encrypResponse) + { + _ = new OpenTelemetryResponse(encrypResponse); + } + else if (instance is Response containerPropertiesResponse) + { + _ = new OpenTelemetryResponse(containerPropertiesResponse); + } + else if (instance is Response databasePropertiesResponse) + { + _ = new OpenTelemetryResponse(databasePropertiesResponse); + } + else if (instance is Response permissionPropertiesResponse) + { + _ = new OpenTelemetryResponse(permissionPropertiesResponse); + } + else if (instance is Response throughputPropertiesResponse) + { + _ = new OpenTelemetryResponse(throughputPropertiesResponse); + } + else if (instance is Response userPropertiesResponse) + { + _ = new OpenTelemetryResponse(userPropertiesResponse); + } + else if (instance is Func> fucntion) + { + _ = new OpenTelemetryResponse(fucntion(className)); + } + else if (instance is Response storedProcedureResponse) + { + _ = new OpenTelemetryResponse(storedProcedureResponse); + } + else if (instance is Response triggerResponse) + { + _ = new OpenTelemetryResponse(triggerResponse); + } + else if (instance is Response userDefinedFunctionResponse) + { + _ = new OpenTelemetryResponse(userDefinedFunctionResponse); + } + else if (instance is Response> storedProcedureExecuteResponse) + { + _ = new OpenTelemetryResponse>(storedProcedureExecuteResponse); + } + else + { + Assert.Fail("Opentelemetry does not support this response type " + className.Name); + } + } + } + + private static readonly Func> ChangefeedResponseFunc = (Type type) => + { + ConstructorInfo constructorInfo = type + .GetConstructor(BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance, null, new Type[] { typeof(ITrace), typeof(ReadOnlyCollection), typeof(double) }, null); + if (constructorInfo != null) + { + return (FeedResponse)constructorInfo.Invoke( + new object[] { + NoOpTrace.Singleton, new List().AsReadOnly(), 10 }); + } + + return null; + }; + + private static readonly Func> ChangeFeedEstimatorEmptyFeedResponseFunc = (Type type) => + { + ConstructorInfo constructorInfo = type + .GetConstructor(BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance, null, new Type[] { typeof(ITrace) }, null); + if (constructorInfo != null) + { + return (FeedResponse)constructorInfo.Invoke( + new object[] { + NoOpTrace.Singleton}); + } + + return null; + }; + + private static QueryResponse GetQueryResponseWithGenerics() + { + return QueryResponse.CreateResponse( + QueryResponse.CreateFailure( + new CosmosQueryResponseMessageHeaders(null, null, Documents.ResourceType.Document, "col"), HttpStatusCode.OK, new RequestMessage(), null, NoOpTrace.Singleton), + MockCosmosUtil.Serializer); + } + + private static async Task GetTransactionalBatchResponse(ItemBatchOperation[] arrayOperations = null) + { + if(arrayOperations == null) + { + arrayOperations = new ItemBatchOperation[1]; + arrayOperations[0] = new ItemBatchOperation(Documents.OperationType.Read, 0, new PartitionKey("0")); + } + + SinglePartitionKeyServerBatchRequest batchRequest = await SinglePartitionKeyServerBatchRequest.CreateAsync( + partitionKey: null, + operations: new ArraySegment(arrayOperations), + serializerCore: MockCosmosUtil.Serializer, + trace: NoOpTrace.Singleton, + cancellationToken: default); + + return await TransactionalBatchResponse.FromResponseMessageAsync( + new ResponseMessage(HttpStatusCode.OK) { Content = new System.IO.MemoryStream() }, + batchRequest, + MockCosmosUtil.Serializer, + true, + NoOpTrace.Singleton, + CancellationToken.None); + } + + private static async Task GetPartitionKeyRangeBatchResponse() + { + ItemBatchOperation[] arrayOperations = new ItemBatchOperation[1]; + + arrayOperations[0] = new ItemBatchOperation(Documents.OperationType.Read, 0, new PartitionKey("0")); + PartitionKeyRangeBatchResponse partitionKeyRangeBatchResponse = new PartitionKeyRangeBatchResponse( + arrayOperations.Length, + await OpenTelemetryRecorderTests.GetTransactionalBatchResponse(arrayOperations), + MockCosmosUtil.Serializer); + + return partitionKeyRangeBatchResponse; + } [TestMethod] public void MarkFailedTest() From 88fd4cdb9bd9975ae663e559e967a217745d1d02 Mon Sep 17 00:00:00 2001 From: Maya-Painter <130110800+Maya-Painter@users.noreply.github.com> Date: Mon, 9 Oct 2023 15:48:52 -0700 Subject: [PATCH 4/5] Revert "Query: Adds new system strings in JsonBinaryEncoding, replacing 1-byte user strings (#3400)" (#4108) This reverts commit 9140890d788cd43d5668d12072be6b965995a28a. --- .../JsonBinaryEncoding.FirstValueOffsets.cs | 4 +- .../src/Json/JsonBinaryEncoding.NodeTypes.cs | 4 +- .../Json/JsonBinaryEncoding.StringLengths.cs | 24 +- .../src/Json/JsonBinaryEncoding.Strings.cs | 29 ++- .../Json/JsonBinaryEncoding.SystemStrings.cs | 214 +----------------- .../Json/JsonBinaryEncoding.SystemStrings.tt | 32 --- .../src/Json/JsonBinaryEncoding.TypeMarker.cs | 33 ++- .../Json/JsonBinaryEncoding.ValueLengths.cs | 4 +- .../src/Json/JsonReader.JsonBinaryReader.cs | 4 +- .../src/Json/JsonWriter.JsonBinaryWriter.cs | 12 +- .../Json/JsonReaderTests.cs | 20 +- .../Json/JsonWriterTests.cs | 20 +- 12 files changed, 98 insertions(+), 302 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.FirstValueOffsets.cs b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.FirstValueOffsets.cs index 609ede82ef..36070e533c 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.FirstValueOffsets.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.FirstValueOffsets.cs @@ -21,11 +21,13 @@ private static class FirstValueOffsets 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - // Encoded 1-byte system string (64 values) + // Encoded 1-byte system string (32 values) 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + // Encoded 1-byte user string (32 values) 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.NodeTypes.cs b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.NodeTypes.cs index e0aa15c2c5..80885d9296 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.NodeTypes.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.NodeTypes.cs @@ -36,11 +36,13 @@ public static class NodeTypes Number, Number, Number, Number, Number, Number, Number, Number, Number, Number, Number, Number, Number, Number, Number, Number, - // Encoded 1-byte system string (64 values) + // Encoded 1-byte system string (32 values) String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, + + // Encoded 1-byte user string (32 values) String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, String, diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.StringLengths.cs b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.StringLengths.cs index 09e85f3815..ac68cfe027 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.StringLengths.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.StringLengths.cs @@ -37,7 +37,7 @@ private static class StringLengths NotStr, NotStr, NotStr, NotStr, NotStr, NotStr, NotStr, NotStr, NotStr, NotStr, NotStr, NotStr, NotStr, NotStr, NotStr, NotStr, - // Encoded 1-byte system string (64 values) + // Encoded 1-byte system string (32 values) SystemStrings.Strings[0].Utf8String.Length, SystemStrings.Strings[1].Utf8String.Length, SystemStrings.Strings[2].Utf8String.Length, SystemStrings.Strings[3].Utf8String.Length, SystemStrings.Strings[4].Utf8String.Length, SystemStrings.Strings[5].Utf8String.Length, @@ -54,22 +54,12 @@ private static class StringLengths SystemStrings.Strings[26].Utf8String.Length, SystemStrings.Strings[27].Utf8String.Length, SystemStrings.Strings[28].Utf8String.Length, SystemStrings.Strings[29].Utf8String.Length, SystemStrings.Strings[30].Utf8String.Length, SystemStrings.Strings[31].Utf8String.Length, - SystemStrings.Strings[32].Utf8String.Length, SystemStrings.Strings[33].Utf8String.Length, - SystemStrings.Strings[34].Utf8String.Length, SystemStrings.Strings[35].Utf8String.Length, - SystemStrings.Strings[36].Utf8String.Length, SystemStrings.Strings[37].Utf8String.Length, - SystemStrings.Strings[38].Utf8String.Length, SystemStrings.Strings[39].Utf8String.Length, - SystemStrings.Strings[40].Utf8String.Length, SystemStrings.Strings[41].Utf8String.Length, - SystemStrings.Strings[42].Utf8String.Length, SystemStrings.Strings[43].Utf8String.Length, - SystemStrings.Strings[44].Utf8String.Length, SystemStrings.Strings[45].Utf8String.Length, - SystemStrings.Strings[46].Utf8String.Length, SystemStrings.Strings[47].Utf8String.Length, - SystemStrings.Strings[48].Utf8String.Length, SystemStrings.Strings[49].Utf8String.Length, - SystemStrings.Strings[50].Utf8String.Length, SystemStrings.Strings[51].Utf8String.Length, - SystemStrings.Strings[52].Utf8String.Length, SystemStrings.Strings[53].Utf8String.Length, - SystemStrings.Strings[54].Utf8String.Length, SystemStrings.Strings[55].Utf8String.Length, - SystemStrings.Strings[56].Utf8String.Length, SystemStrings.Strings[57].Utf8String.Length, - SystemStrings.Strings[58].Utf8String.Length, SystemStrings.Strings[59].Utf8String.Length, - SystemStrings.Strings[60].Utf8String.Length, SystemStrings.Strings[61].Utf8String.Length, - SystemStrings.Strings[62].Utf8String.Length, SystemStrings.Strings[63].Utf8String.Length, + + // Encoded 1-byte user string (32 values) + UsrStr1, UsrStr1, UsrStr1, UsrStr1, UsrStr1, UsrStr1, UsrStr1, UsrStr1, + UsrStr1, UsrStr1, UsrStr1, UsrStr1, UsrStr1, UsrStr1, UsrStr1, UsrStr1, + UsrStr1, UsrStr1, UsrStr1, UsrStr1, UsrStr1, UsrStr1, UsrStr1, UsrStr1, + UsrStr1, UsrStr1, UsrStr1, UsrStr1, UsrStr1, UsrStr1, UsrStr1, UsrStr1, // Encoded 2-byte user string (8 values) UsrStr2, UsrStr2, UsrStr2, UsrStr2, UsrStr2, UsrStr2, UsrStr2, UsrStr2, diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.Strings.cs b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.Strings.cs index 5c3bf992a1..d3c866fbf0 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.Strings.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.Strings.cs @@ -38,11 +38,13 @@ internal static partial class JsonBinaryEncoding false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, - // Encoded 0-byte system string (64 values) + // Encoded 0-byte system string (32 values) true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, + + // Encoded true-byte user string (32 values) true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, @@ -384,14 +386,29 @@ private static bool TryGetUserStringId(ReadOnlySpan stringToken, out int u return false; } - // JsonBinaryEncoding.TypeMarker.IsTwoByteEncodedUserString(typeMarker) must be true - if (stringToken.Length < 2) + if (JsonBinaryEncoding.TypeMarker.IsOneByteEncodedUserString(typeMarker)) { - userStringId = default; - return false; + if (stringToken.Length < 1) + { + userStringId = default; + return false; + } + + userStringId = typeMarker - JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin; } + else //// JsonBinaryEncoding.TypeMarker.IsTwoByteEncodedUserString(typeMarker) + { + if (stringToken.Length < 2) + { + userStringId = default; + return false; + } - userStringId = stringToken[1] + ((stringToken[0] - JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin) * 0xFF); + const byte OneByteCount = JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMax - JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin; + userStringId = OneByteCount + + stringToken[1] + + ((stringToken[0] - JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin) * 0xFF); + } return true; } diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.SystemStrings.cs b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.SystemStrings.cs index 87f2a24062..e9edee7854 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.SystemStrings.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.SystemStrings.cs @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // ------------------------------------------------------------ -// This is auto-generated code. Modify: JsonBinaryEncoding.SystemStrings.tt: 86 +// This is auto-generated code. Modify: JsonBinaryEncoding.SystemStrings.tt: 54 namespace Microsoft.Azure.Cosmos.Json { @@ -51,38 +51,6 @@ public static class SystemStrings UtfAllString.Create("Feature"), UtfAllString.Create("FeatureCollection"), UtfAllString.Create("_id"), - UtfAllString.Create("$id"), - UtfAllString.Create("$pk"), - UtfAllString.Create("_isEdge"), - UtfAllString.Create("_partitionKey"), - UtfAllString.Create("_type"), - UtfAllString.Create("_value"), - UtfAllString.Create("data"), - UtfAllString.Create("Data"), - UtfAllString.Create("entity"), - UtfAllString.Create("isDeleted"), - UtfAllString.Create("IsDeleted"), - UtfAllString.Create("key"), - UtfAllString.Create("Key"), - UtfAllString.Create("Location"), - UtfAllString.Create("partition"), - UtfAllString.Create("partition_id"), - UtfAllString.Create("partitionKey"), - UtfAllString.Create("PartitionKey"), - UtfAllString.Create("pk"), - UtfAllString.Create("state"), - UtfAllString.Create("State"), - UtfAllString.Create("status"), - UtfAllString.Create("Status"), - UtfAllString.Create("subscriptionId"), - UtfAllString.Create("SubscriptionId"), - UtfAllString.Create("tenantId"), - UtfAllString.Create("TenantId"), - UtfAllString.Create("timestamp"), - UtfAllString.Create("Timestamp"), - UtfAllString.Create("ttl"), - UtfAllString.Create("userId"), - UtfAllString.Create("UserId"), }.ToImmutableArray(); public static int? GetSystemStringId(Utf8Span buffer) @@ -93,15 +61,11 @@ public static class SystemStrings 3 => GetSystemStringIdLength3(buffer.Span), 4 => GetSystemStringIdLength4(buffer.Span), 5 => GetSystemStringIdLength5(buffer.Span), - 6 => GetSystemStringIdLength6(buffer.Span), 7 => GetSystemStringIdLength7(buffer.Span), 8 => GetSystemStringIdLength8(buffer.Span), - 9 => GetSystemStringIdLength9(buffer.Span), 10 => GetSystemStringIdLength10(buffer.Span), 11 => GetSystemStringIdLength11(buffer.Span), 12 => GetSystemStringIdLength12(buffer.Span), - 13 => GetSystemStringIdLength13(buffer.Span), - 14 => GetSystemStringIdLength14(buffer.Span), 15 => GetSystemStringIdLength15(buffer.Span), 17 => GetSystemStringIdLength17(buffer.Span), 18 => GetSystemStringIdLength18(buffer.Span), @@ -131,11 +95,6 @@ public static class SystemStrings return 12; } - if (buffer.SequenceEqual(Strings[50].Utf8String.Span.Span)) - { - return 50; - } - return null; } private static int? GetSystemStringIdLength3(ReadOnlySpan buffer) @@ -155,31 +114,6 @@ public static class SystemStrings return 31; } - if (buffer.SequenceEqual(Strings[32].Utf8String.Span.Span)) - { - return 32; - } - - if (buffer.SequenceEqual(Strings[33].Utf8String.Span.Span)) - { - return 33; - } - - if (buffer.SequenceEqual(Strings[43].Utf8String.Span.Span)) - { - return 43; - } - - if (buffer.SequenceEqual(Strings[44].Utf8String.Span.Span)) - { - return 44; - } - - if (buffer.SequenceEqual(Strings[61].Utf8String.Span.Span)) - { - return 61; - } - return null; } private static int? GetSystemStringIdLength4(ReadOnlySpan buffer) @@ -214,16 +148,6 @@ public static class SystemStrings return 27; } - if (buffer.SequenceEqual(Strings[38].Utf8String.Span.Span)) - { - return 38; - } - - if (buffer.SequenceEqual(Strings[39].Utf8String.Span.Span)) - { - return 39; - } - return null; } private static int? GetSystemStringIdLength5(ReadOnlySpan buffer) @@ -258,55 +182,6 @@ public static class SystemStrings return 28; } - if (buffer.SequenceEqual(Strings[36].Utf8String.Span.Span)) - { - return 36; - } - - if (buffer.SequenceEqual(Strings[51].Utf8String.Span.Span)) - { - return 51; - } - - if (buffer.SequenceEqual(Strings[52].Utf8String.Span.Span)) - { - return 52; - } - - return null; - } - private static int? GetSystemStringIdLength6(ReadOnlySpan buffer) - { - if (buffer.SequenceEqual(Strings[37].Utf8String.Span.Span)) - { - return 37; - } - - if (buffer.SequenceEqual(Strings[40].Utf8String.Span.Span)) - { - return 40; - } - - if (buffer.SequenceEqual(Strings[53].Utf8String.Span.Span)) - { - return 53; - } - - if (buffer.SequenceEqual(Strings[54].Utf8String.Span.Span)) - { - return 54; - } - - if (buffer.SequenceEqual(Strings[62].Utf8String.Span.Span)) - { - return 62; - } - - if (buffer.SequenceEqual(Strings[63].Utf8String.Span.Span)) - { - return 63; - } - return null; } private static int? GetSystemStringIdLength7(ReadOnlySpan buffer) @@ -321,11 +196,6 @@ public static class SystemStrings return 29; } - if (buffer.SequenceEqual(Strings[34].Utf8String.Span.Span)) - { - return 34; - } - return null; } private static int? GetSystemStringIdLength8(ReadOnlySpan buffer) @@ -335,50 +205,6 @@ public static class SystemStrings return 10; } - if (buffer.SequenceEqual(Strings[45].Utf8String.Span.Span)) - { - return 45; - } - - if (buffer.SequenceEqual(Strings[57].Utf8String.Span.Span)) - { - return 57; - } - - if (buffer.SequenceEqual(Strings[58].Utf8String.Span.Span)) - { - return 58; - } - - return null; - } - private static int? GetSystemStringIdLength9(ReadOnlySpan buffer) - { - if (buffer.SequenceEqual(Strings[41].Utf8String.Span.Span)) - { - return 41; - } - - if (buffer.SequenceEqual(Strings[42].Utf8String.Span.Span)) - { - return 42; - } - - if (buffer.SequenceEqual(Strings[46].Utf8String.Span.Span)) - { - return 46; - } - - if (buffer.SequenceEqual(Strings[59].Utf8String.Span.Span)) - { - return 59; - } - - if (buffer.SequenceEqual(Strings[60].Utf8String.Span.Span)) - { - return 60; - } - return null; } private static int? GetSystemStringIdLength10(ReadOnlySpan buffer) @@ -426,44 +252,6 @@ public static class SystemStrings return 20; } - if (buffer.SequenceEqual(Strings[47].Utf8String.Span.Span)) - { - return 47; - } - - if (buffer.SequenceEqual(Strings[48].Utf8String.Span.Span)) - { - return 48; - } - - if (buffer.SequenceEqual(Strings[49].Utf8String.Span.Span)) - { - return 49; - } - - return null; - } - private static int? GetSystemStringIdLength13(ReadOnlySpan buffer) - { - if (buffer.SequenceEqual(Strings[35].Utf8String.Span.Span)) - { - return 35; - } - - return null; - } - private static int? GetSystemStringIdLength14(ReadOnlySpan buffer) - { - if (buffer.SequenceEqual(Strings[55].Utf8String.Span.Span)) - { - return 55; - } - - if (buffer.SequenceEqual(Strings[56].Utf8String.Span.Span)) - { - return 56; - } - return null; } private static int? GetSystemStringIdLength15(ReadOnlySpan buffer) diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.SystemStrings.tt b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.SystemStrings.tt index c369e3e489..fd06958ed0 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.SystemStrings.tt +++ b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.SystemStrings.tt @@ -43,38 +43,6 @@ string[] systemStrings = new string[] "Feature", "FeatureCollection", "_id", - "$id", - "$pk", - "_isEdge", - "_partitionKey", - "_type", - "_value", - "data", - "Data", - "entity", - "isDeleted", - "IsDeleted", - "key", - "Key", - "Location", - "partition", - "partition_id", - "partitionKey", - "PartitionKey", - "pk", - "state", - "State", - "status", - "Status", - "subscriptionId", - "SubscriptionId", - "tenantId", - "TenantId", - "timestamp", - "Timestamp", - "ttl", - "userId", - "UserId", }; int[] distinctStringLengths = systemStrings.Select(x => x.Length).Distinct().OrderBy(x => x).ToArray(); diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.TypeMarker.cs b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.TypeMarker.cs index 5c17074451..8fab1e22fb 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.TypeMarker.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.TypeMarker.cs @@ -27,7 +27,7 @@ public readonly struct TypeMarker public const byte LiteralIntMax = LiteralIntMin + 32; #endregion - #region [0x20, 0x60): Encoded 1-byte system string (64 values) + #region [0x20, 0x40): Encoded 1-byte system string (32 values) /// /// The first type marker for a system string whose value can be encoded in a 1 byte type marker. /// @@ -36,14 +36,26 @@ public readonly struct TypeMarker /// /// The last type marker for a system string whose value can be encoded in a 1 byte type marker. /// - public const byte SystemString1ByteLengthMax = SystemString1ByteLengthMin + 64; + public const byte SystemString1ByteLengthMax = SystemString1ByteLengthMin + 32; + #endregion + + #region [0x40, 0x60): Encoded 1-byte user string (32 values) + /// + /// The first type marker for a user string whose value can be encoded in a 1 byte type marker. + /// + public const byte UserString1ByteLengthMin = SystemString1ByteLengthMax; + + /// + /// The last type marker for a user string whose value can be encoded in a 1 byte type marker. + /// + public const byte UserString1ByteLengthMax = UserString1ByteLengthMin + 32; #endregion #region [0x60, 0x68): 2-byte user string (8 values) /// /// The first type marker for a system string whose value can be encoded in a 2 byte type marker. /// - public const byte UserString2ByteLengthMin = SystemString1ByteLengthMax; + public const byte UserString2ByteLengthMin = UserString1ByteLengthMax; /// /// The last type marker for a system string whose value can be encoded in a 2 byte type marker. @@ -455,6 +467,17 @@ public static bool IsSystemString(byte typeMarker) return InRange(typeMarker, SystemString1ByteLengthMin, SystemString1ByteLengthMax); } + /// + /// Gets whether a typeMarker is for a one byte encoded user string. + /// + /// The input type marker. + /// Whether the typeMarker is for a one byte encoded user string. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsOneByteEncodedUserString(byte typeMarker) + { + return InRange(typeMarker, UserString1ByteLengthMin, UserString1ByteLengthMax); + } + /// /// Gets whether a typeMarker is for a two byte encoded user string. /// @@ -474,7 +497,7 @@ public static bool IsTwoByteEncodedUserString(byte typeMarker) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsUserString(byte typeMarker) { - return IsTwoByteEncodedUserString(typeMarker); + return IsOneByteEncodedUserString(typeMarker) || IsTwoByteEncodedUserString(typeMarker); } /// @@ -485,7 +508,7 @@ public static bool IsUserString(byte typeMarker) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsOneByteEncodedString(byte typeMarker) { - return InRange(typeMarker, SystemString1ByteLengthMin, SystemString1ByteLengthMax); + return InRange(typeMarker, SystemString1ByteLengthMin, UserString1ByteLengthMax); } /// diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.ValueLengths.cs b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.ValueLengths.cs index c8b8daac7d..29427c9dbd 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.ValueLengths.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.ValueLengths.cs @@ -43,11 +43,13 @@ private static class ValueLengths 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - // Encoded 1-byte system string (64 values) + // Encoded 1-byte system string (32 values) 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + + // Encoded 1-byte user string (32 values) 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonReader.JsonBinaryReader.cs b/Microsoft.Azure.Cosmos/src/Json/JsonReader.JsonBinaryReader.cs index 1c64b1999a..02b11a9cc1 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonReader.JsonBinaryReader.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonReader.JsonBinaryReader.cs @@ -26,11 +26,13 @@ abstract partial class JsonReader : IJsonReader JsonTokenType.Number, JsonTokenType.Number, JsonTokenType.Number, JsonTokenType.Number, JsonTokenType.Number, JsonTokenType.Number, JsonTokenType.Number, JsonTokenType.Number, JsonTokenType.Number, JsonTokenType.Number, JsonTokenType.Number, JsonTokenType.Number, JsonTokenType.Number, JsonTokenType.Number, JsonTokenType.Number, JsonTokenType.Number, - // Encoded 1-byte system string (64 values) + // Encoded 1-byte system string (32 values) JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, + + // Encoded 1-byte user string (32 values) JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, JsonTokenType.String, diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonWriter.JsonBinaryWriter.cs b/Microsoft.Azure.Cosmos/src/Json/JsonWriter.JsonBinaryWriter.cs index 0df6582cf6..6b20d99f59 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonWriter.JsonBinaryWriter.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonWriter.JsonBinaryWriter.cs @@ -81,16 +81,18 @@ private enum RawValueType : byte RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, - // Encoded 1-byte system string (64 values) - RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, - RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, - RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, - RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, + // Encoded 1-byte system string (32 values) RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, RawValueType.Token, + // Encoded 1-byte user string (32 values) + RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, + RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, + RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, + RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, + // Encoded 2-byte user string (8 values) RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, RawValueType.StrUsr, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonReaderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonReaderTests.cs index 52571f33e9..2f31613d08 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonReaderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonReaderTests.cs @@ -2281,9 +2281,9 @@ public void SimpleObjectTest() List elements = new List { - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 0 }, + new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin }, new byte[] { JsonBinaryEncoding.TypeMarker.LiteralIntMin + 10 }, - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 1 }, + new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin + 1) }, new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.EncodedStringLengthMin + "example glossary".Length), 101, 120, 97, 109, 112, 108, 101, 32, 103, 108, 111, 115, 115, 97, 114, 121 } }; byte[] elementsBytes = elements.SelectMany(x => x).ToArray(); @@ -2405,25 +2405,25 @@ public void AllPrimitivesObjectTest() new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.SystemString1ByteLengthMin + 12) }, new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.EncodedStringLengthMin + "7029d079-4016-4436-b7da-36c0bae54ff6".Length), 55, 48, 50, 57, 100, 48, 55, 57, 45, 52, 48, 49, 54, 45, 52, 52, 51, 54, 45, 98, 55, 100, 97, 45, 51, 54, 99, 48, 98, 97, 101, 53, 52, 102, 102, 54 }, - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 0 }, + new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin }, new byte[] { JsonBinaryEncoding.TypeMarker.NumberDouble, 0x98, 0x8B, 0x30, 0xE3, 0xCB, 0x45, 0xC8, 0x3F }, - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 1 }, + new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin + 1) }, new byte[] { JsonBinaryEncoding.TypeMarker.NumberInt32, 0x19, 0xDF, 0xB6, 0xB0 }, - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 2 }, + new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin + 2) }, new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.EncodedStringLengthMin + "XCPCFXPHHF".Length), 88, 67, 80, 67, 70, 88, 80, 72, 72, 70 }, - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 3 }, + new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin + 3) }, new byte[] { JsonBinaryEncoding.TypeMarker.True }, - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 4 }, + new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin + 4) }, new byte[] { JsonBinaryEncoding.TypeMarker.Null }, - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 5 }, + new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin + 5) }, new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.EncodedStringLengthMin + "2526-07-11T18:18:16.4520716".Length), 50, 53, 50, 54, 45, 48, 55, 45, 49, 49, 84, 49, 56, 58, 49, 56, 58, 49, 54, 46, 52, 53, 50, 48, 55, 49, 54 }, - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 6 } + new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin + 6) } }; List innerObjectElements = new List @@ -2447,7 +2447,7 @@ public void AllPrimitivesObjectTest() elements.Add(new byte[] { JsonBinaryEncoding.TypeMarker.Object1ByteLength, (byte)innerObjectElementsBytes.Length }); elements.Add(innerObjectElementsBytes); - elements.Add(new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 7 }); + elements.Add(new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin + 7) }); elements.Add(new byte[] { JsonBinaryEncoding.TypeMarker.String1ByteLength, (byte)"tiger diamond newbrunswick snowleopard chocolate dog snowleopard turtle cat sapphire peach sapphire vancouver white chocolate horse diamond lion superlongcolourname ruby".Length, 116, 105, 103, 101, 114, 32, 100, 105, 97, 109, 111, 110, 100, 32, 110, 101, 119, 98, 114, 117, 110, 115, 119, 105, 99, 107, 32, 115, 110, 111, 119, 108, 101, 111, 112, 97, 114, 100, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 32, 100, 111, 103, 32, 115, 110, 111, 119, 108, 101, 111, 112, 97, 114, 100, 32, 116, 117, 114, 116, 108, 101, 32, 99, 97, 116, 32, 115, 97, 112, 112, 104, 105, 114, 101, 32, 112, 101, 97, 99, 104, 32, 115, 97, 112, 112, 104, 105, 114, 101, 32, 118, 97, 110, 99, 111, 117, 118, 101, 114, 32, 119, 104, 105, 116, 101, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 32, 104, 111, 114, 115, 101, 32, 100, 105, 97, 109, 111, 110, 100, 32, 108, 105, 111, 110, 32, 115, 117, 112, 101, 114, 108, 111, 110, 103, 99, 111, 108, 111, 117, 114, 110, 97, 109, 101, 32, 114, 117, 98, 121 }); byte[] elementsBytes = elements.SelectMany(x => x).ToArray(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonWriterTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonWriterTests.cs index b44cb49b1f..0997b4dfd2 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonWriterTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonWriterTests.cs @@ -2353,9 +2353,9 @@ public void SimpleObjectTest() List elements = new List { - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 0 }, + new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin }, new byte[] { JsonBinaryEncoding.TypeMarker.LiteralIntMin + 10 }, - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 1 }, + new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin + 1) }, new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.EncodedStringLengthMin + "example glossary".Length), 101, 120, 97, 109, 112, 108, 101, 32, 103, 108, 111, 115, 115, 97, 114, 121 } }; byte[] elementsBytes = elements.SelectMany(x => x).ToArray(); @@ -2484,25 +2484,25 @@ public void AllPrimitivesObjectTest() new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.SystemString1ByteLengthMin + 12) }, new byte[] { JsonBinaryEncoding.TypeMarker.LowercaseGuidString, 0x07, 0x92, 0x0D, 0x97, 0x04, 0x61, 0x44, 0x63, 0x7B, 0xAD, 0x63, 0x0C, 0xAB, 0x5E, 0xF4, 0x6F }, - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 0 }, + new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin }, new byte[] { JsonBinaryEncoding.TypeMarker.NumberDouble, 0x98, 0x8B, 0x30, 0xE3, 0xCB, 0x45, 0xC8, 0x3F }, - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 1 }, + new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin + 1) }, new byte[] { JsonBinaryEncoding.TypeMarker.NumberInt32, 0x19, 0xDF, 0xB6, 0xB0 }, - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 2 }, + new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin + 2) }, new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.EncodedStringLengthMin + "XCPCFXPHHF".Length), 88, 67, 80, 67, 70, 88, 80, 72, 72, 70 }, - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 3 }, + new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin + 3) }, new byte[] { JsonBinaryEncoding.TypeMarker.True }, - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 4 }, + new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin + 4) }, new byte[] { JsonBinaryEncoding.TypeMarker.Null }, - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 5 }, + new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin + 5) }, new byte[] { JsonBinaryEncoding.TypeMarker.CompressedDateTimeString, 0x1B, 0x63, 0x73, 0x1C, 0xC8, 0x22, 0x2E, 0xB9, 0x92, 0x2B, 0xD7, 0x65, 0x13, 0x28, 0x07 }, - new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 6 } + new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin + 6) } }; List innerObjectElements = new List @@ -2526,7 +2526,7 @@ public void AllPrimitivesObjectTest() elements.Add(new byte[] { JsonBinaryEncoding.TypeMarker.Object1ByteLength, (byte)innerObjectElementsBytes.Length }); elements.Add(innerObjectElementsBytes); - elements.Add(new byte[] { (byte)JsonBinaryEncoding.TypeMarker.UserString2ByteLengthMin, 7 }); + elements.Add(new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin + 7) }); elements.Add(new byte[] { JsonBinaryEncoding.TypeMarker.Packed7BitStringLength1, (byte)"tiger diamond newbrunswick snowleopard chocolate dog snowleopard turtle cat sapphire peach sapphire vancouver white chocolate horse diamond lion superlongcolourname ruby".Length, 0xF4, 0xF4, 0xB9, 0x2C, 0x07, 0x91, 0xD3, 0xE1, 0xF6, 0xDB, 0x4D, 0x06, 0xB9, 0xCB, 0x77, 0xB1, 0xBC, 0xEE, 0x9E, 0xDF, 0xD3, 0xE3, 0x35, 0x68, 0xEE, 0x7E, 0xDF, 0xD9, 0xE5, 0x37, 0x3C, 0x2C, 0x27, 0x83, 0xC6, 0xE8, 0xF7, 0xF8, 0xCD, 0x0E, 0xD3, 0xCB, 0x20, 0xF2, 0xFB, 0x0C, 0x9A, 0xBB, 0xDF, 0x77, 0x76, 0xF9, 0x0D, 0x0F, 0xCB, 0xC9, 0x20, 0x7A, 0x5D, 0x4E, 0x67, 0x97, 0x41, 0xE3, 0x30, 0x1D, 0x34, 0x0F, 0xC3, 0xE1, 0xE8, 0xB4, 0xBC, 0x0C, 0x82, 0x97, 0xC3, 0x63, 0x34, 0x68, 0x1E, 0x86, 0xC3, 0xD1, 0x69, 0x79, 0x19, 0x64, 0x0F, 0xBB, 0xC7, 0xEF, 0xBA, 0xBD, 0x2C, 0x07, 0xDD, 0xD1, 0x69, 0x7A, 0x19, 0x34, 0x46, 0xBF, 0xC7, 0x6F, 0x76, 0x98, 0x5E, 0x06, 0xA1, 0xDF, 0xF2, 0x79, 0x19, 0x44, 0x4E, 0x87, 0xDB, 0x6F, 0x37, 0x19, 0xC4, 0x4E, 0xBF, 0xDD, 0xA0, 0x79, 0x1D, 0x5E, 0x96, 0xB3, 0xDF, 0xEE, 0xF3, 0xF8, 0xCD, 0x7E, 0xD7, 0xE5, 0xEE, 0x70, 0xBB, 0x0C, 0x92, 0xD7, 0xC5, 0x79 }); byte[] elementsBytes = elements.SelectMany(x => x).ToArray(); From e2fb347a4a72801fc6c0fcbcd0fc6fa06f48c3db Mon Sep 17 00:00:00 2001 From: iain Date: Tue, 10 Oct 2023 16:59:39 +0100 Subject: [PATCH 5/5] CosmosClientOptions: Adds support for multiple formats of Azure region names (#4016) * Allow ComosClientOptions to take ApplicationRegion and ApplicationPreferredRegions in multiple region name formats. This is a proposed fix for - https://github.com/Azure/azure-cosmos-dotnet-v3/issues/2330 * Address PR comment to avoid duplicating list of names. * Remove the map table cache The map table is only used on initialization, so there's no need to keep a cache of it for the lifetime of the application * Only convert the region names when the client is initializing The cache is created before converting all the names, so it only needs created once, but doesn't remain for the entire lifetime of the application * Update tests * Make RegionNameMapper an instantiable class Instead of having a prepare/clear cache system on a static class, make RegionNameMapper a class that gets instantiated for use and let the ctor handle it. * Remove debugging * Update tests to actually test things --------- Co-authored-by: Pradeep Chellappan Co-authored-by: Pradeep Chellappan <94089783+pradeep-chellappan@users.noreply.github.com> Co-authored-by: Matias Quaranta Co-authored-by: Kiran Kumar Kolli --- .../src/CosmosClientOptions.cs | 19 +- .../src/RegionNameMapper.cs | 53 ++++++ .../CosmosClientOptionsUnitTests.cs | 167 +++++++++++++++++- 3 files changed, 230 insertions(+), 9 deletions(-) create mode 100644 Microsoft.Azure.Cosmos/src/RegionNameMapper.cs diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index b3b21e4f3b..aac863f0e7 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -785,14 +785,17 @@ internal virtual ConnectionPolicy GetConnectionPolicy(int clientId) connectionPolicy.CosmosClientTelemetryOptions = this.CosmosClientTelemetryOptions; } - if (this.ApplicationRegion != null) - { - connectionPolicy.SetCurrentLocation(this.ApplicationRegion); - } - - if (this.ApplicationPreferredRegions != null) - { - connectionPolicy.SetPreferredLocations(this.ApplicationPreferredRegions); + RegionNameMapper mapper = new RegionNameMapper(); + if (!string.IsNullOrEmpty(this.ApplicationRegion)) + { + connectionPolicy.SetCurrentLocation(mapper.GetCosmosDBRegionName(this.ApplicationRegion)); + } + + if (this.ApplicationPreferredRegions != null) + { + List mappedRegions = this.ApplicationPreferredRegions.Select(s => mapper.GetCosmosDBRegionName(s)).ToList(); + + connectionPolicy.SetPreferredLocations(mappedRegions); } if (this.MaxRetryAttemptsOnRateLimitedRequests != null) diff --git a/Microsoft.Azure.Cosmos/src/RegionNameMapper.cs b/Microsoft.Azure.Cosmos/src/RegionNameMapper.cs new file mode 100644 index 0000000000..dbcf0d7c08 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/RegionNameMapper.cs @@ -0,0 +1,53 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos +{ + using System; + using System.Collections.Generic; + using System.Reflection; + + /// + /// Maps a normalized region name to the format that CosmosDB is expecting (for e.g. from 'westus2' to 'West US 2') + /// + internal sealed class RegionNameMapper + { + private readonly Dictionary normalizedToCosmosDBRegionNameMapping; + + public RegionNameMapper() + { + FieldInfo[] fields = typeof(Regions).GetFields(BindingFlags.Public | BindingFlags.Static); + + this.normalizedToCosmosDBRegionNameMapping = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (FieldInfo field in fields) + { + this.normalizedToCosmosDBRegionNameMapping[field.Name] = field.GetValue(null).ToString(); + } + } + + /// + /// Given a normalized region name, this function retrieves the region name in the format that CosmosDB expects. + /// If the region is not known, the same value as input is returned. + /// + /// An Azure region name in a normalized format. The input is not case sensitive. + /// A string that contains the region name in the format that CosmosDB expects. + public string GetCosmosDBRegionName(string normalizedRegionName) + { + if (string.IsNullOrEmpty(normalizedRegionName)) + { + return string.Empty; + } + + normalizedRegionName = normalizedRegionName.Replace(" ", string.Empty); + if (this.normalizedToCosmosDBRegionNameMapping.TryGetValue(normalizedRegionName, + out string cosmosDBRegionName)) + { + return cosmosDBRegionName; + } + + return normalizedRegionName; + } + } +} diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs index f3c45adf48..04d51ea06c 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs @@ -6,7 +6,8 @@ namespace Microsoft.Azure.Cosmos.Tests { using System; using System.Collections; - using System.Collections.Generic; + using System.Collections.Generic; + using System.Collections.ObjectModel; using System.Linq; using System.Net; using System.Net.Http; @@ -556,6 +557,170 @@ public void WithQuorumReadWithEventualConsistencyAccount() Assert.IsTrue(cosmosClientOptions.EnableUpgradeConsistencyToLocalQuorum); } + [TestMethod] + public void VerifyRegionNameFormatConversionForApplicationRegion() + { + CosmosClientOptions cosmosClientOptions = new CosmosClientOptions(); + cosmosClientOptions.ApplicationRegion = "westus2"; + + ConnectionPolicy policy = cosmosClientOptions.GetConnectionPolicy(0); + + // Need to see Regions.WestUS2 in the list, but not "westus2" + bool seenWestUS2 = false; + bool seenNormalized = false; + + foreach (string region in policy.PreferredLocations) + { + if (region == "westus2") + { + seenNormalized = true; + } + + if (region == Regions.WestUS2) + { + seenWestUS2 = true; + } + } + Assert.IsTrue(seenWestUS2); + Assert.IsFalse(seenNormalized); + } + + [TestMethod] + public void VerifyRegionNameFormatConversionBypassForApplicationRegion() + { + CosmosClientOptions cosmosClientOptions = new CosmosClientOptions(); + + // No conversion for expected format. + cosmosClientOptions.ApplicationRegion = Regions.NorthCentralUS; + + ConnectionPolicy policy = cosmosClientOptions.GetConnectionPolicy(0); + + Assert.AreEqual(Regions.NorthCentralUS, policy.PreferredLocations[0]); + + // Ignore unknown values. + cosmosClientOptions.ApplicationRegion = null; + + policy = cosmosClientOptions.GetConnectionPolicy(0); + + Assert.AreEqual(0, policy.PreferredLocations.Count); + + cosmosClientOptions.ApplicationRegion = string.Empty; + policy = cosmosClientOptions.GetConnectionPolicy(0); + + Assert.AreEqual(0, policy.PreferredLocations.Count); + + cosmosClientOptions.ApplicationRegion = "Invalid region"; + Assert.ThrowsException(() => cosmosClientOptions.GetConnectionPolicy(0)); + } + + [TestMethod] + public void VerifyRegionNameFormatConversionForApplicationPreferredRegions() + { + CosmosClientOptions cosmosClientOptions = new CosmosClientOptions(); + cosmosClientOptions.ApplicationPreferredRegions = new List {"westus2", "usdodcentral", Regions.ChinaNorth3}; + + ConnectionPolicy policy = cosmosClientOptions.GetConnectionPolicy(0); + + bool seenUSDodCentral = false; + bool seenWestUS2 = false; + bool seenChinaNorth3 = false; + bool seenNormalizedUSDodCentral = false; + bool seenNormalizedWestUS2 = false; + + foreach (string region in policy.PreferredLocations) + { + if (region == Regions.USDoDCentral) + { + seenUSDodCentral = true; + } + + if (region == Regions.WestUS2) + { + seenWestUS2 = true; + } + + if (region == Regions.ChinaNorth3) + { + seenChinaNorth3 = true; + } + + if (region == "westus2") + { + seenNormalizedWestUS2 = true; + } + + if (region == "usdodcentral") + { + seenNormalizedUSDodCentral = true; + } + } + + Assert.IsTrue(seenChinaNorth3); + Assert.IsTrue(seenWestUS2); + Assert.IsTrue(seenUSDodCentral); + Assert.IsFalse(seenNormalizedUSDodCentral); + Assert.IsFalse(seenNormalizedWestUS2); + } + + [TestMethod] + public void VerifyRegionNameFormatConversionBypassForInvalidApplicationPreferredRegions() + { + CosmosClientOptions cosmosClientOptions = new CosmosClientOptions(); + + // List contains valid and invalid values + cosmosClientOptions.ApplicationPreferredRegions = new List + { + null, + string.Empty, + Regions.JioIndiaCentral, + "westus2", + "Invalid region" + }; + + ConnectionPolicy policy = cosmosClientOptions.GetConnectionPolicy(0); + + bool seenJioIndiaCentral = false; + bool seenWestUS2 = false; + bool seenNormalized = false; + + foreach (string region in policy.PreferredLocations) + { + if (region == Regions.JioIndiaCentral) + { + seenJioIndiaCentral = true; + } + + if (region == Regions.WestUS2) + { + seenWestUS2 = true; + } + + if (region == "westus2") + { + seenNormalized = true; + } + } + + Assert.IsTrue(seenJioIndiaCentral); + Assert.IsTrue(seenWestUS2); + Assert.IsFalse(seenNormalized); + } + + [TestMethod] + public void RegionNameMappingTest() + { + RegionNameMapper mapper = new RegionNameMapper(); + + // Test normalized name + Assert.AreEqual(Regions.WestUS2, mapper.GetCosmosDBRegionName("westus2")); + + // Test with spaces + Assert.AreEqual(Regions.WestUS2, mapper.GetCosmosDBRegionName("west us 2")); + + // Test for case insenstive + Assert.AreEqual(Regions.WestUS2, mapper.GetCosmosDBRegionName("wEsTuS2")); + } + [TestMethod] public void InvalidApplicationNameCatchTest() {