From c083f18fb90966e8e49be01d9ac60901e0b62c3b Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Mon, 26 Jun 2023 17:59:48 -0700 Subject: [PATCH 1/4] Code changes to add replica validation feature in cosmos client options. --- .../src/ConnectionPolicy.cs | 12 + .../src/CosmosClientOptions.cs | 18 + Microsoft.Azure.Cosmos/src/DocumentClient.cs | 3 +- .../src/Fluent/CosmosClientBuilder.cs | 18 + .../src/Routing/GlobalAddressResolver.cs | 6 +- .../CosmosReadManyItemsTests.cs | 60 +++- .../Contracts/DotNetPreviewSDKAPI.json | 24 ++ .../CosmosBadReplicaTests.cs | 318 +++++++++--------- .../CosmosClientOptionsUnitTests.cs | 9 +- 9 files changed, 281 insertions(+), 187 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs index 7abfd76deb..bd91a96b15 100644 --- a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs @@ -459,6 +459,18 @@ public Func HttpClientFactory set; } + /// + /// Gets or sets the boolean flag to enable replica validation. + /// + /// + /// The default value for this parameter is false. + /// + public bool EnableAdvancedReplicaSelectionForTcp + { + get; + set; + } + /// /// (Direct/TCP) This is an advanced setting that controls the number of TCP connections that will be opened eagerly to each Cosmos DB back-end. /// diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index 1fce195915..3b6479016a 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -346,6 +346,23 @@ public ConnectionMode ConnectionMode /// public bool? EnableContentResponseOnWrite { get; set; } + /// + /// Gets or sets the advanced replica selection flag. The advanced replica selection logic keeps track of the replica connection + /// status, and based on status, it prioritizes the replicas which show healthy stable connections, so that the requests can be sent + /// confidently to the particular replica. This helps the cosmos client to become more resilient and effective to any connectivity issues. + /// The default value for this parameter is 'false'. + /// + /// + /// This is optimal for latency-sensitive workloads. Does not apply if is used. + /// + /// +#if PREVIEW + public +#else + internal +#endif + bool EnableAdvancedReplicaSelectionForTcp { get; set; } + /// /// (Direct/TCP) Controls the amount of idle time after which unused connections are closed. /// @@ -758,6 +775,7 @@ internal virtual ConnectionPolicy GetConnectionPolicy(int clientId) EnablePartitionLevelFailover = this.EnablePartitionLevelFailover, PortReuseMode = this.portReuseMode, EnableTcpConnectionEndpointRediscovery = this.EnableTcpConnectionEndpointRediscovery, + EnableAdvancedReplicaSelectionForTcp = this.EnableAdvancedReplicaSelectionForTcp, HttpClientFactory = this.httpClientFactory, ServerCertificateCustomValidationCallback = this.ServerCertificateCustomValidationCallback }; diff --git a/Microsoft.Azure.Cosmos/src/DocumentClient.cs b/Microsoft.Azure.Cosmos/src/DocumentClient.cs index 752ec8f5a5..11564e37ff 100644 --- a/Microsoft.Azure.Cosmos/src/DocumentClient.cs +++ b/Microsoft.Azure.Cosmos/src/DocumentClient.cs @@ -233,7 +233,6 @@ public DocumentClient(Uri serviceEndpoint, this.Initialize(serviceEndpoint, connectionPolicy, desiredConsistencyLevel); this.initTaskCache = new AsyncCacheNonBlocking(cancellationToken: this.cancellationTokenSource.Token); - this.isReplicaAddressValidationEnabled = ConfigurationManager.IsReplicaAddressValidationEnabled(); } /// @@ -6703,7 +6702,7 @@ private void CreateStoreModel(bool subscribeRntbdStatus) !this.enableRntbdChannel, this.UseMultipleWriteLocations && (this.accountServiceConfiguration.DefaultConsistencyLevel != Documents.ConsistencyLevel.Strong), true, - enableReplicaValidation: this.isReplicaAddressValidationEnabled); + enableReplicaValidation: this.ConnectionPolicy.EnableAdvancedReplicaSelectionForTcp); if (subscribeRntbdStatus) { diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index 8b72bfffa4..35970bcca0 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -621,6 +621,24 @@ public CosmosClientBuilder WithContentResponseOnWrite(bool contentResponseOnWrit return this; } + /// + /// Enables the advanced replica selection flag. The advanced replica selection logic keeps track of the replica connection status, + /// and based on status, it prioritizes the replicas which are connected to the backend, so that the requests can be sent + /// confidently to the particular replica. This helps the cosmos client to become more resilient and effictive to any connection + /// timeouts. The default value for this parameter is false. + /// + /// The object +#if PREVIEW + public +#else + internal +#endif + CosmosClientBuilder WithAdvancedReplicaSelectionEnabledForTcp() + { + this.clientOptions.EnableAdvancedReplicaSelectionForTcp = true; + return this; + } + /// /// The event handler to be invoked before the request is sent. /// diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalAddressResolver.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalAddressResolver.cs index 8ac0719dcd..ac7df37f33 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GlobalAddressResolver.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GlobalAddressResolver.cs @@ -11,7 +11,6 @@ namespace Microsoft.Azure.Cosmos.Routing using System.Net; using System.Threading; using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.ChangeFeed.Exceptions; using Microsoft.Azure.Cosmos.Common; using Microsoft.Azure.Cosmos.Core.Trace; using Microsoft.Azure.Cosmos.Resource.CosmosExceptions; @@ -38,7 +37,7 @@ internal sealed class GlobalAddressResolver : IAddressResolverExtension, IDispos private readonly CosmosHttpClient httpClient; private readonly ConcurrentDictionary addressCacheByEndpoint; private readonly bool enableTcpConnectionEndpointRediscovery; - private readonly bool isReplicaAddressValidationEnabled; + private readonly bool replicaAddressValidationEnabled; private IOpenConnectionsHandler openConnectionsHandler; public GlobalAddressResolver( @@ -66,6 +65,7 @@ public GlobalAddressResolver( ? GlobalAddressResolver.MaxBackupReadRegions : 0; this.enableTcpConnectionEndpointRediscovery = connectionPolicy.EnableTcpConnectionEndpointRediscovery; + this.replicaAddressValidationEnabled = connectionPolicy.EnableAdvancedReplicaSelectionForTcp; this.isReplicaAddressValidationEnabled = ConfigurationManager.IsReplicaAddressValidationEnabled(); @@ -284,7 +284,7 @@ private EndpointCache GetOrAddEndpoint(Uri endpoint) this.httpClient, this.openConnectionsHandler, enableTcpConnectionEndpointRediscovery: this.enableTcpConnectionEndpointRediscovery, - replicaAddressValidationEnabled: this.isReplicaAddressValidationEnabled); + replicaAddressValidationEnabled: this.replicaAddressValidationEnabled); string location = this.endpointManager.GetLocation(endpoint); AddressResolver addressResolver = new AddressResolver(null, new NullRequestSigner(), location); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosReadManyItemsTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosReadManyItemsTests.cs index d215b6b18b..2747531a77 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosReadManyItemsTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosReadManyItemsTests.cs @@ -51,27 +51,57 @@ public async Task Cleanup() } [TestMethod] - public async Task ReadManyTypedTest() + [DataRow(true, DisplayName = "Validates Read Many scenario with advanced replica selection enabled.")] + [DataRow(false, DisplayName = "Validates Read Many scenario with advanced replica selection disabled.")] + public async Task ReadManyTypedTestWithAdvancedReplicaSelection( + bool advancedReplicaSelectionEnabled) { - List<(string, PartitionKey)> itemList = new List<(string, PartitionKey)>(); - for (int i=0; i<10; i++) + CosmosClient cosmosClient = advancedReplicaSelectionEnabled + ? TestCommon.CreateCosmosClient( + customizeClientBuilder: (CosmosClientBuilder builder) => builder.WithAdvancedReplicaSelectionEnabledForTcp()) + : TestCommon.CreateCosmosClient(); + + Database database = null; + try { - itemList.Add((i.ToString(), new PartitionKey("pk" + i.ToString()))); - } + database = await cosmosClient.CreateDatabaseAsync("ReadManyTypedTestScenarioDb"); + Container container = await database.CreateContainerAsync("ReadManyTypedTestContainer", "/pk"); - FeedResponse feedResponse= await this.Container.ReadManyItemsAsync(itemList); - Assert.IsNotNull(feedResponse); - Assert.AreEqual(feedResponse.Count, 10); - Assert.IsTrue(feedResponse.Headers.RequestCharge > 0); - Assert.IsNotNull(feedResponse.Diagnostics); + // Create items with different pk values + for (int i = 0; i < 500; i++) + { + ToDoActivity item = ToDoActivity.CreateRandomToDoActivity(); + item.pk = "pk" + i.ToString(); + item.id = i.ToString(); + ItemResponse itemResponse = await container.CreateItemAsync(item); + Assert.AreEqual(HttpStatusCode.Created, itemResponse.StatusCode); + } + + List<(string, PartitionKey)> itemList = new List<(string, PartitionKey)>(); + for (int i = 0; i < 20; i++) + { + itemList.Add((i.ToString(), new PartitionKey("pk" + i.ToString()))); + } + + FeedResponse feedResponse = await container.ReadManyItemsAsync(itemList); + Assert.IsNotNull(feedResponse); + Assert.AreEqual(20, feedResponse.Count); + Assert.IsTrue(feedResponse.Headers.RequestCharge > 0); + Assert.IsNotNull(feedResponse.Diagnostics); - int count = 0; - foreach (ToDoActivity item in feedResponse) + int count = 0; + foreach (ToDoActivity item in feedResponse) + { + count++; + Assert.IsNotNull(item); + } + Assert.AreEqual(20, count); + } + finally { - count++; - Assert.IsNotNull(item); + await database.DeleteAsync(); + cosmosClient.Dispose(); } - Assert.AreEqual(count, 10); } [TestMethod] diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json index 8546a93b2f..f717ea525d 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json @@ -355,6 +355,18 @@ "Microsoft.Azure.Cosmos.CosmosClientOptions;System.Object;IsAbstract:False;IsSealed:False;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { "Subclasses": {}, "Members": { + "Boolean EnableAdvancedReplicaSelectionForTcp": { + "Type": "Property", + "Attributes": [], + "MethodInfo": "Boolean EnableAdvancedReplicaSelectionForTcp;CanRead:True;CanWrite:True;Boolean get_EnableAdvancedReplicaSelectionForTcp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_EnableAdvancedReplicaSelectionForTcp(Boolean);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Boolean get_EnableAdvancedReplicaSelectionForTcp()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "Boolean get_EnableAdvancedReplicaSelectionForTcp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Boolean get_IsDistributedTracingEnabled()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ @@ -367,6 +379,13 @@ "Attributes": [], "MethodInfo": "Boolean IsDistributedTracingEnabled;CanRead:True;CanWrite:True;Boolean get_IsDistributedTracingEnabled();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_IsDistributedTracingEnabled(Boolean);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "Void set_EnableAdvancedReplicaSelectionForTcp(Boolean)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "Void set_EnableAdvancedReplicaSelectionForTcp(Boolean);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void set_IsDistributedTracingEnabled(Boolean)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ @@ -454,6 +473,11 @@ "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder;System.Object;IsAbstract:False;IsSealed:False;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { "Subclasses": {}, "Members": { + "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithAdvancedReplicaSelectionEnabledForTcp()": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithAdvancedReplicaSelectionEnabledForTcp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithDistributedTracing(Boolean)": { "Type": "Method", "Attributes": [], diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosBadReplicaTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosBadReplicaTests.cs index 7a5b387e8e..804c61507f 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosBadReplicaTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosBadReplicaTests.cs @@ -26,186 +26,174 @@ public class CosmosBadReplicaTests public async Task TestGoneFromServiceScenarioAsync( bool enableReplicaValidation) { - try - { - Environment.SetEnvironmentVariable( - variable: ConfigurationManager.ReplicaConnectivityValidationEnabled, - value: enableReplicaValidation.ToString()); - - Mock mockHttpHandler = new Mock(MockBehavior.Strict); - Uri endpoint = MockSetupsHelper.SetupSingleRegionAccount( - "mockAccountInfo", - consistencyLevel: ConsistencyLevel.Session, - mockHttpHandler, - out string primaryRegionEndpoint); - - string databaseName = "mockDbName"; - string containerName = "mockContainerName"; - string containerRid = "ccZ1ANCszwk="; - Documents.ResourceId cRid = Documents.ResourceId.Parse(containerRid); - MockSetupsHelper.SetupContainerProperties( - mockHttpHandler: mockHttpHandler, - regionEndpoint: primaryRegionEndpoint, - databaseName: databaseName, - containerName: containerName, - containerRid: containerRid); - - MockSetupsHelper.SetupSinglePartitionKeyRange( - mockHttpHandler, - primaryRegionEndpoint, - cRid, - out IReadOnlyList partitionKeyRanges); - - List replicaIds1 = new List() + Mock mockHttpHandler = new Mock(MockBehavior.Strict); + Uri endpoint = MockSetupsHelper.SetupSingleRegionAccount( + "mockAccountInfo", + consistencyLevel: ConsistencyLevel.Session, + mockHttpHandler, + out string primaryRegionEndpoint); + + string databaseName = "mockDbName"; + string containerName = "mockContainerName"; + string containerRid = "ccZ1ANCszwk="; + Documents.ResourceId cRid = Documents.ResourceId.Parse(containerRid); + MockSetupsHelper.SetupContainerProperties( + mockHttpHandler: mockHttpHandler, + regionEndpoint: primaryRegionEndpoint, + databaseName: databaseName, + containerName: containerName, + containerRid: containerRid); + + MockSetupsHelper.SetupSinglePartitionKeyRange( + mockHttpHandler, + primaryRegionEndpoint, + cRid, + out IReadOnlyList partitionKeyRanges); + + List replicaIds1 = new List() + { + "11111111111111111", + "22222222222222222", + "33333333333333333", + "44444444444444444", + }; + + HttpResponseMessage replicaSet1 = MockSetupsHelper.CreateAddresses( + replicaIds1, + partitionKeyRanges.First(), + "eastus", + cRid); + + // One replica changed on the refresh + List replicaIds2 = new List() + { + "11111111111111111", + "22222222222222222", + "33333333333333333", + "55555555555555555", + }; + + HttpResponseMessage replicaSet2 = MockSetupsHelper.CreateAddresses( + replicaIds2, + partitionKeyRanges.First(), + "eastus", + cRid); + + bool delayCacheRefresh = true; + bool delayRefreshUnblocked = false; + mockHttpHandler.SetupSequence(x => x.SendAsync( + It.Is(r => r.RequestUri.ToString().Contains("addresses")), It.IsAny())) + .Returns(Task.FromResult(replicaSet1)) + .Returns(async ()=> + { + //block cache refresh to verify bad replica is not visited during refresh + while (delayCacheRefresh) + { + await Task.Delay(TimeSpan.FromMilliseconds(20)); + } + + delayRefreshUnblocked = true; + return replicaSet2; + }); + + int callBack = 0; + List urisVisited = new List(); + Mock mockTransportClient = new Mock(MockBehavior.Strict); + mockTransportClient.Setup(x => x.InvokeResourceOperationAsync(It.IsAny(), It.IsAny())) + .Callback((t, _) => urisVisited.Add(t)) + .Returns(() => { - "11111111111111111", - "22222222222222222", - "33333333333333333", - "44444444444444444", - }; + callBack++; + if (callBack == 1) + { + throw Documents.Rntbd.TransportExceptions.GetGoneException( + new Uri("https://localhost:8081"), + Guid.NewGuid(), + new Documents.TransportException(Documents.TransportErrorCode.ConnectionBroken, + null, + Guid.NewGuid(), + new Uri("https://localhost:8081"), + "Mock", + userPayload: true, + payloadSent: false)); + } - HttpResponseMessage replicaSet1 = MockSetupsHelper.CreateAddresses( - replicaIds1, - partitionKeyRanges.First(), - "eastus", - cRid); + return Task.FromResult(new Documents.StoreResponse() + { + Status = 200, + Headers = new Documents.Collections.StoreResponseNameValueCollection() + { + ActivityId = Guid.NewGuid().ToString(), + LSN = "12345", + PartitionKeyRangeId = "0", + GlobalCommittedLSN = "12345", + SessionToken = "1#12345#1=12345" + }, + ResponseBody = new MemoryStream() + }); + }); - // One replica changed on the refresh - List replicaIds2 = new List() + CosmosClientOptions cosmosClientOptions = new CosmosClientOptions() { - "11111111111111111", - "22222222222222222", - "33333333333333333", - "55555555555555555", + ConsistencyLevel = Cosmos.ConsistencyLevel.Session, + HttpClientFactory = () => new HttpClient(new HttpHandlerHelper(mockHttpHandler.Object)), + TransportClientHandlerFactory = (original) => mockTransportClient.Object, + EnableAdvancedReplicaSelectionForTcp = enableReplicaValidation, }; - HttpResponseMessage replicaSet2 = MockSetupsHelper.CreateAddresses( - replicaIds2, - partitionKeyRanges.First(), - "eastus", - cRid); - - bool delayCacheRefresh = true; - bool delayRefreshUnblocked = false; - mockHttpHandler.SetupSequence(x => x.SendAsync( - It.Is(r => r.RequestUri.ToString().Contains("addresses")), It.IsAny())) - .Returns(Task.FromResult(replicaSet1)) - .Returns(async ()=> - { - //block cache refresh to verify bad replica is not visited during refresh - while (delayCacheRefresh) - { - await Task.Delay(TimeSpan.FromMilliseconds(20)); - } - - delayRefreshUnblocked = true; - return replicaSet2; - }); - - int callBack = 0; - List urisVisited = new List(); - Mock mockTransportClient = new Mock(MockBehavior.Strict); - mockTransportClient.Setup(x => x.InvokeResourceOperationAsync(It.IsAny(), It.IsAny())) - .Callback((t, _) => urisVisited.Add(t)) - .Returns(() => + using (CosmosClient customClient = new CosmosClient( + endpoint.ToString(), + Convert.ToBase64String(Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())), + cosmosClientOptions)) + { + try { - callBack++; - if (callBack == 1) + Container container = customClient.GetContainer(databaseName, containerName); + + for (int i = 0; i < 20; i++) { - throw Documents.Rntbd.TransportExceptions.GetGoneException( - new Uri("https://localhost:8081"), - Guid.NewGuid(), - new Documents.TransportException(Documents.TransportErrorCode.ConnectionBroken, - null, - Guid.NewGuid(), - new Uri("https://localhost:8081"), - "Mock", - userPayload: true, - payloadSent: false)); + ResponseMessage response = await container.ReadItemStreamAsync(Guid.NewGuid().ToString(), new Cosmos.PartitionKey(Guid.NewGuid().ToString())); + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); } - return Task.FromResult(new Documents.StoreResponse() - { - Status = 200, - Headers = new Documents.Collections.StoreResponseNameValueCollection() - { - ActivityId = Guid.NewGuid().ToString(), - LSN = "12345", - PartitionKeyRangeId = "0", - GlobalCommittedLSN = "12345", - SessionToken = "1#12345#1=12345" - }, - ResponseBody = new MemoryStream() - }); - }); + mockTransportClient.VerifyAll(); + mockHttpHandler.VerifyAll(); - CosmosClientOptions cosmosClientOptions = new CosmosClientOptions() - { - ConsistencyLevel = Cosmos.ConsistencyLevel.Session, - HttpClientFactory = () => new HttpClient(new HttpHandlerHelper(mockHttpHandler.Object)), - TransportClientHandlerFactory = (original) => mockTransportClient.Object, - }; - - using (CosmosClient customClient = new CosmosClient( - endpoint.ToString(), - Convert.ToBase64String(Encoding.UTF8.GetBytes(Guid.NewGuid().ToString())), - cosmosClientOptions)) - { - try + Documents.TransportAddressUri failedReplica = urisVisited.First(); + + // With replica validation enabled in preview mode, the failed replica will be validated as a part of the flow, + // and because any subsequent validation/ connection will be successful, the failed replica will now be marked + // as connected, thus it will be visited more than once. However, note that when replice validation is disabled, + // no validation is done thus the URI will be marked as unhealthy as expected. Therefore the uri will be visited + // just once. + Assert.IsTrue( + enableReplicaValidation + ? urisVisited.Any(x => x.Equals(failedReplica)) + : urisVisited.Count(x => x.Equals(failedReplica)) == 1); + + urisVisited.Clear(); + delayCacheRefresh = false; + do { - Container container = customClient.GetContainer(databaseName, containerName); - - for (int i = 0; i < 20; i++) - { - ResponseMessage response = await container.ReadItemStreamAsync(Guid.NewGuid().ToString(), new Cosmos.PartitionKey(Guid.NewGuid().ToString())); - Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); - } - - mockTransportClient.VerifyAll(); - mockHttpHandler.VerifyAll(); - - Documents.TransportAddressUri failedReplica = urisVisited.First(); - - // With replica validation enabled in preview mode, the failed replica will be validated as a part of the flow, - // and because any subsequent validation/ connection will be successful, the failed replica will now be marked - // as connected, thus it will be visited more than once. However, note that when replice validation is disabled, - // no validation is done thus the URI will be marked as unhealthy as expected. Therefore the uri will be visited - // just once. - Assert.IsTrue( - enableReplicaValidation - ? urisVisited.Any(x => x.Equals(failedReplica)) - : urisVisited.Count(x => x.Equals(failedReplica)) == 1); - - urisVisited.Clear(); - delayCacheRefresh = false; - do - { - await Task.Delay(TimeSpan.FromMilliseconds(100)); - }while (!delayRefreshUnblocked); - - for (int i = 0; i < 20; i++) - { - ResponseMessage response = await container.ReadItemStreamAsync(Guid.NewGuid().ToString(), new Cosmos.PartitionKey(Guid.NewGuid().ToString())); - Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); - } - - Assert.AreEqual(4, urisVisited.ToHashSet().Count()); - - // Clears all the setups. No network calls should be done on the next operation. - mockHttpHandler.Reset(); - mockTransportClient.Reset(); - } - finally + await Task.Delay(TimeSpan.FromMilliseconds(100)); + }while (!delayRefreshUnblocked); + + for (int i = 0; i < 20; i++) { - mockTransportClient.Setup(x => x.Dispose()); + ResponseMessage response = await container.ReadItemStreamAsync(Guid.NewGuid().ToString(), new Cosmos.PartitionKey(Guid.NewGuid().ToString())); + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); } + + Assert.AreEqual(4, urisVisited.ToHashSet().Count()); + + // Clears all the setups. No network calls should be done on the next operation. + mockHttpHandler.Reset(); + mockTransportClient.Reset(); + } + finally + { + mockTransportClient.Setup(x => x.Dispose()); } - } - finally - { - Environment.SetEnvironmentVariable( - variable: ConfigurationManager.ReplicaConnectivityValidationEnabled, - value: null); } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs index efb06d7cc5..1d20ff4a66 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs @@ -81,6 +81,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.IsNull(clientOptions.HttpClientFactory); Assert.AreNotEqual(consistencyLevel, clientOptions.ConsistencyLevel); Assert.IsFalse(clientOptions.EnablePartitionLevelFailover); + Assert.IsFalse(clientOptions.EnableAdvancedReplicaSelectionForTcp); //Verify GetConnectionPolicy returns the correct values for default ConnectionPolicy policy = clientOptions.GetConnectionPolicy(clientId: 0); @@ -97,6 +98,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.IsNull(policy.HttpClientFactory); Assert.AreNotEqual(Cosmos.ConsistencyLevel.Session, clientOptions.ConsistencyLevel); Assert.IsFalse(policy.EnablePartitionLevelFailover); + Assert.IsFalse(policy.EnableAdvancedReplicaSelectionForTcp); cosmosClientBuilder.WithApplicationRegion(region) .WithConnectionModeGateway(maxConnections, webProxy) @@ -108,7 +110,8 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() .WithBulkExecution(true) .WithSerializerOptions(cosmosSerializerOptions) .WithConsistencyLevel(consistencyLevel) - .WithPartitionLevelFailoverEnabled(); + .WithPartitionLevelFailoverEnabled() + .WithAdvancedReplicaSelectionEnabledForTcp(); cosmosClient = cosmosClientBuilder.Build(new MockDocumentClient()); clientOptions = cosmosClient.ClientOptions; @@ -131,6 +134,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.IsTrue(clientOptions.AllowBulkExecution); Assert.AreEqual(consistencyLevel, clientOptions.ConsistencyLevel); Assert.IsTrue(clientOptions.EnablePartitionLevelFailover); + Assert.IsTrue(clientOptions.EnableAdvancedReplicaSelectionForTcp); //Verify GetConnectionPolicy returns the correct values policy = clientOptions.GetConnectionPolicy(clientId: 0); @@ -145,7 +149,8 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.AreEqual((int)maxRetryWaitTime.TotalSeconds, policy.RetryOptions.MaxRetryWaitTimeInSeconds); Assert.AreEqual((Documents.ConsistencyLevel)consistencyLevel, clientOptions.GetDocumentsConsistencyLevel()); Assert.IsTrue(policy.EnablePartitionLevelFailover); - + Assert.IsTrue(policy.EnableAdvancedReplicaSelectionForTcp); + IReadOnlyList preferredLocations = new List() { Regions.AustraliaCentral, Regions.AustraliaCentral2 }; //Verify Direct Mode settings cosmosClientBuilder = new CosmosClientBuilder( From aa28b03aef75a7b2d7155752a8b5274d18bc0d42 Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Fri, 7 Jul 2023 11:24:41 -0700 Subject: [PATCH 2/4] Code changes to mark replica validation as internal feature. --- Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs | 7 +------ Microsoft.Azure.Cosmos/src/DocumentClient.cs | 1 - Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs | 7 +------ .../src/Routing/GlobalAddressResolver.cs | 2 -- 4 files changed, 2 insertions(+), 15 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index 3b6479016a..55dee24242 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -356,12 +356,7 @@ public ConnectionMode ConnectionMode /// This is optimal for latency-sensitive workloads. Does not apply if is used. /// /// -#if PREVIEW - public -#else - internal -#endif - bool EnableAdvancedReplicaSelectionForTcp { get; set; } + internal bool EnableAdvancedReplicaSelectionForTcp { get; set; } /// /// (Direct/TCP) Controls the amount of idle time after which unused connections are closed. diff --git a/Microsoft.Azure.Cosmos/src/DocumentClient.cs b/Microsoft.Azure.Cosmos/src/DocumentClient.cs index 11564e37ff..7157fce026 100644 --- a/Microsoft.Azure.Cosmos/src/DocumentClient.cs +++ b/Microsoft.Azure.Cosmos/src/DocumentClient.cs @@ -113,7 +113,6 @@ internal partial class DocumentClient : IDisposable, IAuthorizationTokenProvider private const string DefaultInitTaskKey = "InitTaskKey"; private readonly bool IsLocalQuorumConsistency = false; - private readonly bool isReplicaAddressValidationEnabled; //Auth internal readonly AuthorizationTokenProvider cosmosAuthorization; diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index 35970bcca0..e9fccd3045 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -628,12 +628,7 @@ public CosmosClientBuilder WithContentResponseOnWrite(bool contentResponseOnWrit /// timeouts. The default value for this parameter is false. /// /// The object -#if PREVIEW - public -#else - internal -#endif - CosmosClientBuilder WithAdvancedReplicaSelectionEnabledForTcp() + internal CosmosClientBuilder WithAdvancedReplicaSelectionEnabledForTcp() { this.clientOptions.EnableAdvancedReplicaSelectionForTcp = true; return this; diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalAddressResolver.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalAddressResolver.cs index ac7df37f33..3c97e9e343 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GlobalAddressResolver.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GlobalAddressResolver.cs @@ -67,8 +67,6 @@ public GlobalAddressResolver( this.enableTcpConnectionEndpointRediscovery = connectionPolicy.EnableTcpConnectionEndpointRediscovery; this.replicaAddressValidationEnabled = connectionPolicy.EnableAdvancedReplicaSelectionForTcp; - this.isReplicaAddressValidationEnabled = ConfigurationManager.IsReplicaAddressValidationEnabled(); - this.maxEndpoints = maxBackupReadEndpoints + 2; // for write and alternate write endpoint (during failover) this.addressCacheByEndpoint = new ConcurrentDictionary(); From 7a0bbcf8ee36b29e1832eea97390f953e6604a04 Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Fri, 7 Jul 2023 11:34:28 -0700 Subject: [PATCH 3/4] Code changes to update the preview contract! --- .../Contracts/DotNetPreviewSDKAPI.json | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json index f717ea525d..8546a93b2f 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json @@ -355,18 +355,6 @@ "Microsoft.Azure.Cosmos.CosmosClientOptions;System.Object;IsAbstract:False;IsSealed:False;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { "Subclasses": {}, "Members": { - "Boolean EnableAdvancedReplicaSelectionForTcp": { - "Type": "Property", - "Attributes": [], - "MethodInfo": "Boolean EnableAdvancedReplicaSelectionForTcp;CanRead:True;CanWrite:True;Boolean get_EnableAdvancedReplicaSelectionForTcp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_EnableAdvancedReplicaSelectionForTcp(Boolean);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Boolean get_EnableAdvancedReplicaSelectionForTcp()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Boolean get_EnableAdvancedReplicaSelectionForTcp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, "Boolean get_IsDistributedTracingEnabled()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ @@ -379,13 +367,6 @@ "Attributes": [], "MethodInfo": "Boolean IsDistributedTracingEnabled;CanRead:True;CanWrite:True;Boolean get_IsDistributedTracingEnabled();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_IsDistributedTracingEnabled(Boolean);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Void set_EnableAdvancedReplicaSelectionForTcp(Boolean)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Void set_EnableAdvancedReplicaSelectionForTcp(Boolean);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, "Void set_IsDistributedTracingEnabled(Boolean)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ @@ -473,11 +454,6 @@ "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder;System.Object;IsAbstract:False;IsSealed:False;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { "Subclasses": {}, "Members": { - "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithAdvancedReplicaSelectionEnabledForTcp()": { - "Type": "Method", - "Attributes": [], - "MethodInfo": "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithAdvancedReplicaSelectionEnabledForTcp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithDistributedTracing(Boolean)": { "Type": "Method", "Attributes": [], From 37beed5cb77316fec172edb8bdd41d3c574cfe6f Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Fri, 7 Jul 2023 17:38:55 -0700 Subject: [PATCH 4/4] Code changes to remove replica validation flag from client builder. --- Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs | 1 - .../src/Fluent/CosmosClientBuilder.cs | 13 ------------- .../CosmosReadManyItemsTests.cs | 9 +++++---- .../CosmosClientOptionsUnitTests.cs | 7 +++---- 4 files changed, 8 insertions(+), 22 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index 55dee24242..702ff3f4e4 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -355,7 +355,6 @@ public ConnectionMode ConnectionMode /// /// This is optimal for latency-sensitive workloads. Does not apply if is used. /// - /// internal bool EnableAdvancedReplicaSelectionForTcp { get; set; } /// diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index e9fccd3045..8b72bfffa4 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -621,19 +621,6 @@ public CosmosClientBuilder WithContentResponseOnWrite(bool contentResponseOnWrit return this; } - /// - /// Enables the advanced replica selection flag. The advanced replica selection logic keeps track of the replica connection status, - /// and based on status, it prioritizes the replicas which are connected to the backend, so that the requests can be sent - /// confidently to the particular replica. This helps the cosmos client to become more resilient and effictive to any connection - /// timeouts. The default value for this parameter is false. - /// - /// The object - internal CosmosClientBuilder WithAdvancedReplicaSelectionEnabledForTcp() - { - this.clientOptions.EnableAdvancedReplicaSelectionForTcp = true; - return this; - } - /// /// The event handler to be invoked before the request is sent. /// diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosReadManyItemsTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosReadManyItemsTests.cs index 2747531a77..05184d7826 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosReadManyItemsTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosReadManyItemsTests.cs @@ -56,12 +56,13 @@ public async Task Cleanup() public async Task ReadManyTypedTestWithAdvancedReplicaSelection( bool advancedReplicaSelectionEnabled) { - CosmosClient cosmosClient = advancedReplicaSelectionEnabled - ? TestCommon.CreateCosmosClient( - customizeClientBuilder: (CosmosClientBuilder builder) => builder.WithAdvancedReplicaSelectionEnabledForTcp()) - : TestCommon.CreateCosmosClient(); + CosmosClientOptions clientOptions = new () + { + EnableAdvancedReplicaSelectionForTcp = advancedReplicaSelectionEnabled, + }; Database database = null; + CosmosClient cosmosClient = TestCommon.CreateCosmosClient(clientOptions); try { database = await cosmosClient.CreateDatabaseAsync("ReadManyTypedTestScenarioDb"); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs index 1d20ff4a66..02d38e3a1f 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs @@ -110,8 +110,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() .WithBulkExecution(true) .WithSerializerOptions(cosmosSerializerOptions) .WithConsistencyLevel(consistencyLevel) - .WithPartitionLevelFailoverEnabled() - .WithAdvancedReplicaSelectionEnabledForTcp(); + .WithPartitionLevelFailoverEnabled(); cosmosClient = cosmosClientBuilder.Build(new MockDocumentClient()); clientOptions = cosmosClient.ClientOptions; @@ -134,7 +133,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.IsTrue(clientOptions.AllowBulkExecution); Assert.AreEqual(consistencyLevel, clientOptions.ConsistencyLevel); Assert.IsTrue(clientOptions.EnablePartitionLevelFailover); - Assert.IsTrue(clientOptions.EnableAdvancedReplicaSelectionForTcp); + Assert.IsFalse(clientOptions.EnableAdvancedReplicaSelectionForTcp); //Verify GetConnectionPolicy returns the correct values policy = clientOptions.GetConnectionPolicy(clientId: 0); @@ -149,7 +148,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.AreEqual((int)maxRetryWaitTime.TotalSeconds, policy.RetryOptions.MaxRetryWaitTimeInSeconds); Assert.AreEqual((Documents.ConsistencyLevel)consistencyLevel, clientOptions.GetDocumentsConsistencyLevel()); Assert.IsTrue(policy.EnablePartitionLevelFailover); - Assert.IsTrue(policy.EnableAdvancedReplicaSelectionForTcp); + Assert.IsFalse(policy.EnableAdvancedReplicaSelectionForTcp); IReadOnlyList preferredLocations = new List() { Regions.AustraliaCentral, Regions.AustraliaCentral2 }; //Verify Direct Mode settings