From 83cee904a00f9033cb7900799a1db1f2faf2498f Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Tue, 16 Jan 2024 16:37:36 -0800 Subject: [PATCH 01/16] Code changes to add regional endpoints for account metadata calls. --- .../src/ConnectionPolicy.cs | 39 +++++++++++++++ .../src/CosmosClientOptions.cs | 40 +++++++++++++++ .../src/GatewayAccountReader.cs | 49 ++++++++++++++++--- 3 files changed, 121 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs index bd1b23b16c..9d8c33709c 100644 --- a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs @@ -29,6 +29,7 @@ internal sealed class ConnectionPolicy private Protocol connectionProtocol; private ObservableCollection preferredLocations; + private ObservableCollection regionalEndpoints; /// /// Initializes a new instance of the class to connect to the Azure Cosmos DB service. @@ -43,6 +44,7 @@ public ConnectionPolicy() this.MediaReadMode = MediaReadMode.Buffered; this.UserAgentContainer = new UserAgentContainer(clientId: 0); this.preferredLocations = new ObservableCollection(); + this.regionalEndpoints = new ObservableCollection(); this.EnableEndpointDiscovery = true; this.MaxConnectionLimit = defaultMaxConcurrentConnectionLimit; this.RetryOptions = new RetryOptions(); @@ -90,6 +92,20 @@ public void SetPreferredLocations(IReadOnlyList regions) } } + public void SetRegionalEndpoints(IReadOnlyList regionalEndpoints) + { + if (regionalEndpoints == null) + { + throw new ArgumentNullException(nameof(regionalEndpoints)); + } + + this.regionalEndpoints.Clear(); + foreach (string endpoint in regionalEndpoints) + { + this.regionalEndpoints.Add(endpoint); + } + } + /// /// Gets or sets the maximum number of concurrent fanout requests sent to the Azure Cosmos DB service. /// @@ -270,6 +286,29 @@ public Collection PreferredLocations } } + /// + /// Gets and sets the preferred locations (regions) for geo-replicated database accounts in the Azure Cosmos DB service. + /// For example, "East US" as the preferred location. + /// + /// + /// + /// When is true and the value of this property is non-empty, + /// the SDK uses the locations in the collection in the order they are specified to perform operations, + /// otherwise if the value of this property is not specified, + /// the SDK uses the write region as the preferred location for all operations. + /// + /// + /// If is set to false, the value of this property is ignored. + /// + /// + public Collection RegionalEndpoints + { + get + { + return this.regionalEndpoints; + } + } + /// /// Gets or sets the flag to enable endpoint discovery for geo-replicated database accounts in the Azure Cosmos DB service. /// diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index a90a2e04a6..19eab066c9 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -190,6 +190,41 @@ public string ApplicationName /// /// High availability on regional outages public IReadOnlyList ApplicationPreferredRegions { get; set; } + + /// + /// Gets and sets the preferred regions for geo-replicated database accounts in the Azure Cosmos DB service. + /// + /// + /// + /// During the CosmosClient initialization the account information, including the available regions, is obtained from the . + /// The CosmosClient will use the value of to populate the preferred list with the account available regions that intersect with its value. + /// If the value of contains regions that are not an available region in the account, the values will be ignored. If the these invalid regions are added later to the account, the CosmosClient will use them if they are higher in the preference order. + /// + /// + /// If during CosmosClient initialization, the is not reachable, the CosmosClient will attempt to recover and obtain the account information issuing requests to the regions in in the order that they are listed. + /// + /// + /// See also Diagnose + /// and troubleshoot the availability of Cosmos SDKs for more details. + /// + /// + /// This configuration is an alternative to , either one can be set but not both. + /// + /// + /// + /// + /// (){ "custom.p-1.documents.azure.com", "custom.p-2.documents.azure.com" } + /// }; + /// + /// CosmosClient client = new CosmosClient("endpoint", "key", clientOptions); + /// ]]> + /// + /// + /// High availability on regional outages + public IReadOnlyList RegionalEndpoints { get; set; } /// /// Get or set the maximum number of concurrent connections allowed for the target @@ -792,6 +827,11 @@ internal virtual ConnectionPolicy GetConnectionPolicy(int clientId) List mappedRegions = this.ApplicationPreferredRegions.Select(s => mapper.GetCosmosDBRegionName(s)).ToList(); connectionPolicy.SetPreferredLocations(mappedRegions); + } + + if (this.RegionalEndpoints != null) + { + connectionPolicy.SetRegionalEndpoints(this.RegionalEndpoints); } if (this.MaxRetryAttemptsOnRateLimitedRequests != null) diff --git a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs index 5413103fee..a1fdad7b0a 100644 --- a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs +++ b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs @@ -5,10 +5,11 @@ namespace Microsoft.Azure.Cosmos { using System; - using System.Globalization; + using System.Collections.Generic; using System.Net.Http; using System.Threading; using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Core.Trace; using Microsoft.Azure.Cosmos.Resource.CosmosExceptions; using Microsoft.Azure.Cosmos.Routing; using Microsoft.Azure.Cosmos.Tracing; @@ -22,6 +23,7 @@ internal sealed class GatewayAccountReader private readonly AuthorizationTokenProvider cosmosAuthorization; private readonly CosmosHttpClient httpClient; private readonly Uri serviceEndpoint; + private readonly IEnumerator serviceEndpointEnumerator; private readonly CancellationToken cancellationToken; // Backlog: Auth abstractions are spilling through. 4 arguments for this CTOR are result of it. @@ -36,6 +38,22 @@ public GatewayAccountReader(Uri serviceEndpoint, this.cosmosAuthorization = cosmosAuthorization ?? throw new ArgumentNullException(nameof(AuthorizationTokenProvider)); this.connectionPolicy = connectionPolicy; this.cancellationToken = cancellationToken; + + List serviceEndpoints = new () + { + serviceEndpoint + }; + + if (this.connectionPolicy.RegionalEndpoints != null) + { + foreach (string regionalEndpoint in this.connectionPolicy.RegionalEndpoints) + { + // Do the regional endpoint lookup + serviceEndpoints.Add(new Uri(regionalEndpoint)); + } + } + + this.serviceEndpointEnumerator = serviceEndpoints.GetEnumerator(); } private async Task GetDatabaseAccountAsync(Uri serviceEndpoint) @@ -86,13 +104,30 @@ await this.cosmosAuthorization.AddAuthorizationHeaderAsync( public async Task InitializeReaderAsync() { - AccountProperties databaseAccount = await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( - defaultEndpoint: this.serviceEndpoint, - locations: this.connectionPolicy.PreferredLocations, - getDatabaseAccountFn: this.GetDatabaseAccountAsync, - cancellationToken: this.cancellationToken); + int attemptCounter = 1; + List exceptionList = new (); + + while (this.serviceEndpointEnumerator.MoveNext()) + { + try + { + AccountProperties databaseAccount = await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( + defaultEndpoint: this.serviceEndpointEnumerator.Current, + locations: this.connectionPolicy.PreferredLocations, + getDatabaseAccountFn: this.GetDatabaseAccountAsync, + cancellationToken: this.cancellationToken); + + return databaseAccount; + } + catch (Exception ex) + { + DefaultTrace.TraceWarning("Attempt: {0}, Exception occurred while fetching account details: {1}", attemptCounter, ex.Message); + exceptionList.Add(ex); + attemptCounter++; + } + } - return databaseAccount; + throw new AggregateException("Unable to get account information.", exceptionList); } } } From 1e12015717753776bc4d4554dd347916f7461120 Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Thu, 18 Jan 2024 16:23:07 -0800 Subject: [PATCH 02/16] Code changes to refactor some codes. --- .../src/GatewayAccountReader.cs | 15 +++++---- .../GatewayAccountReaderTests.cs | 32 +++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs index a1fdad7b0a..5699d1d60a 100644 --- a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs +++ b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs @@ -44,12 +44,14 @@ public GatewayAccountReader(Uri serviceEndpoint, serviceEndpoint }; - if (this.connectionPolicy.RegionalEndpoints != null) + if (this.connectionPolicy.RegionalEndpoints != null + && this.connectionPolicy.RegionalEndpoints.Count > 0) { foreach (string regionalEndpoint in this.connectionPolicy.RegionalEndpoints) { - // Do the regional endpoint lookup - serviceEndpoints.Add(new Uri(regionalEndpoint)); + // Add all of the regional endpoints to the service endpoints list. + serviceEndpoints.Add( + new Uri(regionalEndpoint)); } } @@ -121,13 +123,14 @@ public async Task InitializeReaderAsync() } catch (Exception ex) { - DefaultTrace.TraceWarning("Attempt: {0}, Exception occurred while fetching account details: {1}", attemptCounter, ex.Message); + DefaultTrace.TraceWarning("Attempt: {0}, Exception occurred while fetching account details: {1}", attemptCounter++, ex.Message); exceptionList.Add(ex); - attemptCounter++; } } - throw new AggregateException("Unable to get account information.", exceptionList); + throw exceptionList.Count > 1 + ? new AggregateException("Unable to get account information.", exceptionList) + : exceptionList[0]; } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs index cf54b12a52..12951b4ce4 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs @@ -16,6 +16,7 @@ namespace Microsoft.Azure.Cosmos using Microsoft.Azure.Cosmos.Tests; using Microsoft.Azure.Cosmos.Tracing; using Microsoft.Azure.Cosmos.Tracing.TraceData; + using System.Collections.Generic; /// /// Tests for . @@ -100,6 +101,37 @@ public void DocumentClient_BuildHttpClientFactory_WithFactory() .Verify(f => f(), Times.Once); } + [TestMethod] + public async Task InitializeReaderAsync_WhenInvokedWithRegionalEndpoints_ShouldRetryWhenPrimaryEndpointFails() + { + HttpMessageHandler messageHandler = new CustomMessageHandler(); + HttpClient staticHttpClient = new HttpClient(messageHandler); + + Mock mockedHttpClient = new(); + + ConnectionPolicy connectionPolicy = new() + { + EnablePartitionLevelFailover = true, + ConnectionMode = ConnectionMode.Direct, + }; + + connectionPolicy.SetRegionalEndpoints( + new List() + { + "https://dkppaf1.documents-test.windows-int.net:443/", + "https://dkppaf2.documents-test.windows-int.net:443/", + "https://dkppaf6.documents-test.windows-int.net:443/", + }); + + GatewayAccountReader accountReader = new GatewayAccountReader( + serviceEndpoint: new Uri("https://localhost"), + cosmosAuthorization: Mock.Of(), + connectionPolicy: connectionPolicy, + httpClient: MockCosmosUtil.CreateCosmosHttpClient(() => staticHttpClient)); + + AggregateException exception = await Assert.ThrowsExceptionAsync(() => accountReader.InitializeReaderAsync()); + } + public class CustomMessageHandler : HttpMessageHandler { protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) From 7f68b28c5b1fd7d1113d349b855c694d1adb339c Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Sun, 21 Jan 2024 18:58:09 -0800 Subject: [PATCH 03/16] Code changes to add unit tests. --- .../GatewayAccountReaderTests.cs | 99 +++++++++++++++++-- 1 file changed, 89 insertions(+), 10 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs index 12951b4ce4..62a7f21d36 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs @@ -17,6 +17,8 @@ namespace Microsoft.Azure.Cosmos using Microsoft.Azure.Cosmos.Tracing; using Microsoft.Azure.Cosmos.Tracing.TraceData; using System.Collections.Generic; + using System.Collections.ObjectModel; + using System.Text; /// /// Tests for . @@ -102,34 +104,111 @@ public void DocumentClient_BuildHttpClientFactory_WithFactory() } [TestMethod] - public async Task InitializeReaderAsync_WhenInvokedWithRegionalEndpoints_ShouldRetryWhenPrimaryEndpointFails() + [DataRow(true, DisplayName = "Validate that when regional endpoints are provided in the connection policy, the request will be retried in the regional endpoints.")] + [DataRow(false, DisplayName = "Validate that when regional endpoints are not provided in the connection policy, the request will be failed in the primary endpoint.")] + public async Task InitializeReaderAsync_WhenRegionalEndpointsProvided_ShouldRetryWithRegionalEndpointsWhenPrimaryFails( + bool regionalEndpointsProvided) { - HttpMessageHandler messageHandler = new CustomMessageHandler(); - HttpClient staticHttpClient = new HttpClient(messageHandler); + string accountPropertiesResponse = "{\r\n \"_self\": \"\",\r\n \"id\": \"localhost\",\r\n \"_rid\": \"127.0.0.1\",\r\n \"media\": \"//media/\",\r\n \"addresses\": \"//addresses/\",\r\n \"_dbs\": \"//dbs/\",\r\n \"writableLocations\": [\r\n {\r\n \"name\": \"South Central US\",\r\n \"databaseAccountEndpoint\": \"https://127.0.0.1:8081/\"\r\n }" + + "\r\n ],\r\n \"readableLocations\": [\r\n {\r\n \"name\": \"South Central US\",\r\n \"databaseAccountEndpoint\": \"https://127.0.0.1:8081/\"\r\n }\r\n ],\r\n \"enableMultipleWriteLocations\": false,\r\n \"userReplicationPolicy\": {\r\n \"asyncReplication\": false,\r\n \"minReplicaSetSize\": 1,\r\n \"maxReplicasetSize\": 4\r\n },\r\n \"userConsistencyPolicy\": {\r\n " + + "\"defaultConsistencyLevel\": \"Session\"\r\n },\r\n \"systemReplicationPolicy\": {\r\n \"minReplicaSetSize\": 1,\r\n \"maxReplicasetSize\": 4\r\n },\r\n \"readPolicy\": {\r\n \"primaryReadCoefficient\": 1,\r\n \"secondaryReadCoefficient\": 1\r\n },\r\n \"queryEngineConfiguration\": \"{\\\"maxSqlQueryInputLength\\\":262144,\\\"maxJoinsPerSqlQuery\\\":5," + + "\\\"maxLogicalAndPerSqlQuery\\\":500,\\\"maxLogicalOrPerSqlQuery\\\":500,\\\"maxUdfRefPerSqlQuery\\\":10,\\\"maxInExpressionItemsCount\\\":16000,\\\"queryMaxInMemorySortDocumentCount\\\":500,\\\"maxQueryRequestTimeoutFraction\\\":0.9,\\\"sqlAllowNonFiniteNumbers\\\":false,\\\"sqlAllowAggregateFunctions\\\":true,\\\"sqlAllowSubQuery\\\":true,\\\"sqlAllowScalarSubQuery\\\":true,\\\"allowNewKeywords\\\":true,\\\"" + + "sqlAllowLike\\\":true,\\\"sqlAllowGroupByClause\\\":true,\\\"maxSpatialQueryCells\\\":12,\\\"spatialMaxGeometryPointCount\\\":256,\\\"sqlDisableOptimizationFlags\\\":0,\\\"sqlAllowTop\\\":true,\\\"enableSpatialIndexing\\\":true}\"\r\n}"; + + StringContent content = new(accountPropertiesResponse); + HttpResponseMessage responseMessage = new() + { + StatusCode = HttpStatusCode.OK, + Content = content, + }; + + Mock mockHttpClient = new(); + mockHttpClient + .SetupSequence(x => x.GetAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ThrowsAsync(new Exception("Service is Unavailable at the Moment.")) + .ThrowsAsync(new Exception("Service is Unavailable at the Moment.")) + .ReturnsAsync(responseMessage); + + ConnectionPolicy connectionPolicy = new() + { + ConnectionMode = ConnectionMode.Direct, + }; + + if (regionalEndpointsProvided) + { + connectionPolicy.SetRegionalEndpoints( + new List() + { + "https://testfed2.documents-test.windows-int.net:443/", + "https://testfed3.documents-test.windows-int.net:443/", + "https://testfed4.documents-test.windows-int.net:443/", + }); + } + + GatewayAccountReader accountReader = new GatewayAccountReader( + serviceEndpoint: new Uri("https://testfed1.documents-test.windows-int.net:443/"), + cosmosAuthorization: Mock.Of(), + connectionPolicy: connectionPolicy, + httpClient: mockHttpClient.Object); + + if (regionalEndpointsProvided) + { + AccountProperties accountProperties = await accountReader.InitializeReaderAsync(); + + Assert.IsNotNull(accountProperties); + Assert.AreEqual("localhost", accountProperties.Id); + Assert.AreEqual("127.0.0.1", accountProperties.ResourceId); + } + else + { + Exception exception = await Assert.ThrowsExceptionAsync(() => accountReader.InitializeReaderAsync()); + Assert.IsNotNull(exception); + Assert.AreEqual("Service is Unavailable at the Moment.", exception.Message); + } + } - Mock mockedHttpClient = new(); + [TestMethod] + public async Task InitializeReaderAsync_WhenRegionalEndpointsProvided_ShouldThrowAggregateExceptionWithAllEndpointsFail() + { + Mock mockHttpClient = new(); + mockHttpClient + .Setup(x => x.GetAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ThrowsAsync(new Exception("Service is Unavailable at the Moment.")); ConnectionPolicy connectionPolicy = new() { - EnablePartitionLevelFailover = true, ConnectionMode = ConnectionMode.Direct, }; connectionPolicy.SetRegionalEndpoints( new List() { - "https://dkppaf1.documents-test.windows-int.net:443/", - "https://dkppaf2.documents-test.windows-int.net:443/", - "https://dkppaf6.documents-test.windows-int.net:443/", + "https://testfed2.documents-test.windows-int.net:443/", + "https://testfed3.documents-test.windows-int.net:443/", + "https://testfed4.documents-test.windows-int.net:443/", }); GatewayAccountReader accountReader = new GatewayAccountReader( - serviceEndpoint: new Uri("https://localhost"), + serviceEndpoint: new Uri("https://testfed1.documents-test.windows-int.net:443/"), cosmosAuthorization: Mock.Of(), connectionPolicy: connectionPolicy, - httpClient: MockCosmosUtil.CreateCosmosHttpClient(() => staticHttpClient)); + httpClient: mockHttpClient.Object); AggregateException exception = await Assert.ThrowsExceptionAsync(() => accountReader.InitializeReaderAsync()); + Assert.IsNotNull(exception); + Assert.AreEqual("Service is Unavailable at the Moment.", exception.InnerException.Message); } public class CustomMessageHandler : HttpMessageHandler From 7aa43f881ea703121fa5a5ecfc4d2a49dd6b1d28 Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Mon, 22 Jan 2024 17:52:48 -0800 Subject: [PATCH 04/16] Code changes to make minor code clean-up. --- .../Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs index 62a7f21d36..5bad96963f 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs @@ -17,8 +17,6 @@ namespace Microsoft.Azure.Cosmos using Microsoft.Azure.Cosmos.Tracing; using Microsoft.Azure.Cosmos.Tracing.TraceData; using System.Collections.Generic; - using System.Collections.ObjectModel; - using System.Text; /// /// Tests for . @@ -104,6 +102,7 @@ public void DocumentClient_BuildHttpClientFactory_WithFactory() } [TestMethod] + [Owner("dkunda")] [DataRow(true, DisplayName = "Validate that when regional endpoints are provided in the connection policy, the request will be retried in the regional endpoints.")] [DataRow(false, DisplayName = "Validate that when regional endpoints are not provided in the connection policy, the request will be failed in the primary endpoint.")] public async Task InitializeReaderAsync_WhenRegionalEndpointsProvided_ShouldRetryWithRegionalEndpointsWhenPrimaryFails( @@ -174,6 +173,7 @@ public async Task InitializeReaderAsync_WhenRegionalEndpointsProvided_ShouldRetr } [TestMethod] + [Owner("dkunda")] public async Task InitializeReaderAsync_WhenRegionalEndpointsProvided_ShouldThrowAggregateExceptionWithAllEndpointsFail() { Mock mockHttpClient = new(); From 60a080352b2d4e901b547e59c90ac101fd1c6357 Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Tue, 23 Jan 2024 13:18:33 -0800 Subject: [PATCH 05/16] Code changes to fix tests. Refactored API. --- .../src/ConnectionPolicy.cs | 9 +++++- .../src/CosmosClientOptions.cs | 14 ++++----- .../src/Fluent/CosmosClientBuilder.cs | 30 +++++++++++++++++++ .../Contracts/DotNetSDKAPI.json | 24 +++++++++++++++ .../CosmosClientOptionsUnitTests.cs | 20 ++++++++++--- .../CosmosClientTests.cs | 2 +- .../GatewayAccountReaderTests.cs | 4 +-- 7 files changed, 86 insertions(+), 17 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs index 9d8c33709c..b6c11aca2a 100644 --- a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs @@ -92,7 +92,14 @@ public void SetPreferredLocations(IReadOnlyList regions) } } - public void SetRegionalEndpoints(IReadOnlyList regionalEndpoints) + /// + /// Sets the regional endpoints required to fetch account information from + /// private domain names. + /// + /// An instance of containing the regional endpoints + /// provided by the customer. + public void SetRegionalEndpoints( + ISet regionalEndpoints) { if (regionalEndpoints == null) { diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index 19eab066c9..c2705c2404 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -192,31 +192,27 @@ public string ApplicationName public IReadOnlyList ApplicationPreferredRegions { get; set; } /// - /// Gets and sets the preferred regions for geo-replicated database accounts in the Azure Cosmos DB service. + /// Gets and sets the regional private endpoints for geo-replicated database accounts in the Azure Cosmos DB service. /// /// /// /// During the CosmosClient initialization the account information, including the available regions, is obtained from the . - /// The CosmosClient will use the value of to populate the preferred list with the account available regions that intersect with its value. - /// If the value of contains regions that are not an available region in the account, the values will be ignored. If the these invalid regions are added later to the account, the CosmosClient will use them if they are higher in the preference order. + /// Should the global endpoint become inaccessible, the CosmosClient will attempt to obtain the account information issuing requests to the regional endpoints provided in . /// /// - /// If during CosmosClient initialization, the is not reachable, the CosmosClient will attempt to recover and obtain the account information issuing requests to the regions in in the order that they are listed. + /// Nevertheless, this parameter remains optional and is recommended for implementation when a customer has configured a private endpoint for their Cosmos DB account. /// /// /// See also Diagnose /// and troubleshoot the availability of Cosmos SDKs for more details. /// - /// - /// This configuration is an alternative to , either one can be set but not both. - /// /// /// /// /// (){ "custom.p-1.documents.azure.com", "custom.p-2.documents.azure.com" } + /// RegionalEndpoints = new HashSet(){ "custom.p-1.documents.azure.com", "custom.p-2.documents.azure.com" } /// }; /// /// CosmosClient client = new CosmosClient("endpoint", "key", clientOptions); @@ -224,7 +220,7 @@ public string ApplicationName /// /// /// High availability on regional outages - public IReadOnlyList RegionalEndpoints { get; set; } + public ISet RegionalEndpoints { get; set; } /// /// Get or set the maximum number of concurrent connections allowed for the target diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index 59325ecb83..8752d9b541 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -301,6 +301,36 @@ public CosmosClientBuilder WithApplicationPreferredRegions(IReadOnlyList return this; } + /// + /// Sets the regional private endpoints for geo-replicated database accounts in the Azure Cosmos DB service. + /// During the CosmosClient initialization the account information, including the available regions, is obtained from the . + /// Should the global endpoint become inaccessible, the CosmosClient will attempt to obtain the account information issuing requests to the regional endpoints + /// provided in the regionalEndpoints set. + /// + /// A set of string containing the regional private endpoints for the cosmos db account. + /// + /// This function is optional and is recommended for implementation when a customer has configured one or more private endpoints for their Cosmos DB account. + /// + /// + /// The example below creates a new instance of with the regional endpoints. + /// + /// () { "https://region-1.documents-test.windows-int.net:443/", "https://region-2.documents-test.windows-int.net:443/" }); + /// CosmosClient client = cosmosClientBuilder.Build(); + /// ]]> + /// + /// + /// The current . + /// + public CosmosClientBuilder WithRegionalEndpoints(ISet regionalEndpoints) + { + this.clientOptions.RegionalEndpoints = regionalEndpoints; + return this; + } + /// /// Limits the operations to the provided endpoint on the CosmosClientBuilder constructor. /// diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json index 46bdf1c170..73edcfd67f 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json @@ -2782,6 +2782,18 @@ ], "MethodInfo": "System.Collections.Generic.IReadOnlyList`1[System.String] get_ApplicationPreferredRegions();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "System.Collections.Generic.ISet`1[System.String] get_RegionalEndpoints()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "System.Collections.Generic.ISet`1[System.String] get_RegionalEndpoints();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Collections.Generic.ISet`1[System.String] RegionalEndpoints": { + "Type": "Property", + "Attributes": [], + "MethodInfo": "System.Collections.Generic.ISet`1[System.String] RegionalEndpoints;CanRead:True;CanWrite:True;System.Collections.Generic.ISet`1[System.String] get_RegionalEndpoints();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_RegionalEndpoints(System.Collections.Generic.ISet`1[System.String]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "System.Collections.ObjectModel.Collection`1[Microsoft.Azure.Cosmos.RequestHandler] CustomHandlers[Newtonsoft.Json.JsonConverterAttribute(typeof(Microsoft.Azure.Cosmos.CosmosClientOptions+ClientOptionJsonConverter))]": { "Type": "Property", "Attributes": [ @@ -3096,6 +3108,13 @@ "Attributes": [], "MethodInfo": "Void set_PortReuseMode(System.Nullable`1[Microsoft.Azure.Cosmos.PortReuseMode]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "Void set_RegionalEndpoints(System.Collections.Generic.ISet`1[System.String])[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "Void set_RegionalEndpoints(System.Collections.Generic.ISet`1[System.String]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void set_RequestTimeout(System.TimeSpan)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ @@ -4599,6 +4618,11 @@ "Attributes": [], "MethodInfo": "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithLimitToEndpoint(Boolean);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithRegionalEndpoints(System.Collections.Generic.ISet`1[System.String])": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithRegionalEndpoints(System.Collections.Generic.ISet`1[System.String]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithRequestTimeout(System.TimeSpan)": { "Type": "Method", "Attributes": [], diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs index b625858c09..6405edce9a 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs @@ -156,7 +156,9 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.IsFalse(policy.EnablePartitionLevelFailover); Assert.IsTrue(clientOptions.EnableAdvancedReplicaSelectionForTcp.Value); - IReadOnlyList preferredLocations = new List() { Regions.AustraliaCentral, Regions.AustraliaCentral2 }; + IReadOnlyList preferredLocations = new List() { Regions.AustraliaCentral, Regions.AustraliaCentral2 }; + ISet regionalEndpoints = new HashSet() { "https://testfed2.documents-test.windows-int.net:443/", "https://testfed4.documents-test.windows-int.net:443/" }; + //Verify Direct Mode settings cosmosClientBuilder = new CosmosClientBuilder( accountEndpoint: endpoint, @@ -168,7 +170,8 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() maxTcpConnectionsPerEndpoint, portReuseMode, enableTcpConnectionEndpointRediscovery) - .WithApplicationPreferredRegions(preferredLocations) + .WithApplicationPreferredRegions(preferredLocations) + .WithRegionalEndpoints(regionalEndpoints) .WithClientTelemetryOptions(new CosmosClientTelemetryOptions() { DisableDistributedTracing = false, @@ -188,7 +191,8 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.AreEqual(maxTcpConnectionsPerEndpoint, clientOptions.MaxTcpConnectionsPerEndpoint); Assert.AreEqual(portReuseMode, clientOptions.PortReuseMode); Assert.IsTrue(clientOptions.EnableTcpConnectionEndpointRediscovery); - CollectionAssert.AreEqual(preferredLocations.ToArray(), clientOptions.ApplicationPreferredRegions.ToArray()); + CollectionAssert.AreEqual(preferredLocations.ToArray(), clientOptions.ApplicationPreferredRegions.ToArray()); + CollectionAssert.AreEqual(regionalEndpoints.ToArray(), clientOptions.RegionalEndpoints.ToArray()); Assert.AreEqual(TimeSpan.FromMilliseconds(100), clientOptions.CosmosClientTelemetryOptions.CosmosThresholdOptions.PointOperationLatencyThreshold); Assert.AreEqual(TimeSpan.FromMilliseconds(100), clientOptions.CosmosClientTelemetryOptions.CosmosThresholdOptions.NonPointOperationLatencyThreshold); Assert.IsFalse(clientOptions.CosmosClientTelemetryOptions.DisableDistributedTracing); @@ -320,6 +324,13 @@ public void CosmosClientOptions_WhenPartitionLevelFailoverEnabledAndPreferredReg Regions.NorthCentralUS, Regions.WestUS, Regions.EastAsia, + }) + .WithRegionalEndpoints( + new HashSet() + { + "https://testfed2.documents-test.windows-int.net:443/", + "https://testfed3.documents-test.windows-int.net:443/", + "https://testfed4.documents-test.windows-int.net:443/", }); CosmosClientOptions clientOptions = cosmosClientBuilder.Build().ClientOptions; @@ -337,7 +348,8 @@ public void CosmosClientOptions_WhenPartitionLevelFailoverEnabledAndPreferredReg Assert.IsFalse(clientOptions.AllowBulkExecution); Assert.AreEqual(consistencyLevel, clientOptions.ConsistencyLevel); Assert.IsTrue(clientOptions.EnablePartitionLevelFailover); - Assert.IsNotNull(clientOptions.ApplicationPreferredRegions); + Assert.IsNotNull(clientOptions.ApplicationPreferredRegions); + Assert.IsNotNull(clientOptions.RegionalEndpoints); } finally { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientTests.cs index 6740001f53..353e94b642 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientTests.cs @@ -138,7 +138,7 @@ public async Task InvalidKey_ExceptionFullStacktrace(string endpoint, string key } catch (Exception ex) { - Assert.IsTrue(ex.StackTrace.Contains("GatewayAccountReader.GetDatabaseAccountAsync"), ex.StackTrace); + Assert.IsTrue(ex.StackTrace.Contains("GatewayAccountReader.InitializeReaderAsync"), ex.StackTrace); } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs index 5bad96963f..7275c2f613 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs @@ -142,7 +142,7 @@ public async Task InitializeReaderAsync_WhenRegionalEndpointsProvided_ShouldRetr if (regionalEndpointsProvided) { connectionPolicy.SetRegionalEndpoints( - new List() + new HashSet() { "https://testfed2.documents-test.windows-int.net:443/", "https://testfed3.documents-test.windows-int.net:443/", @@ -193,7 +193,7 @@ public async Task InitializeReaderAsync_WhenRegionalEndpointsProvided_ShouldThro }; connectionPolicy.SetRegionalEndpoints( - new List() + new HashSet() { "https://testfed2.documents-test.windows-int.net:443/", "https://testfed3.documents-test.windows-int.net:443/", From 6efe50711cff5f8cef424dbf6f72228d732a3a3a Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Wed, 24 Jan 2024 23:44:14 -0800 Subject: [PATCH 06/16] Code changes to refactor the enumeration logic inside global endpoint manager. --- .../src/GatewayAccountReader.cs | 51 ++--------- .../src/Routing/GlobalEndpointManager.cs | 85 +++++++++++++------ .../GlobalEndpointManagerTest.cs | 10 +++ 3 files changed, 76 insertions(+), 70 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs index 5699d1d60a..c225438c6d 100644 --- a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs +++ b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs @@ -9,7 +9,6 @@ namespace Microsoft.Azure.Cosmos using System.Net.Http; using System.Threading; using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Core.Trace; using Microsoft.Azure.Cosmos.Resource.CosmosExceptions; using Microsoft.Azure.Cosmos.Routing; using Microsoft.Azure.Cosmos.Tracing; @@ -23,7 +22,6 @@ internal sealed class GatewayAccountReader private readonly AuthorizationTokenProvider cosmosAuthorization; private readonly CosmosHttpClient httpClient; private readonly Uri serviceEndpoint; - private readonly IEnumerator serviceEndpointEnumerator; private readonly CancellationToken cancellationToken; // Backlog: Auth abstractions are spilling through. 4 arguments for this CTOR are result of it. @@ -38,24 +36,6 @@ public GatewayAccountReader(Uri serviceEndpoint, this.cosmosAuthorization = cosmosAuthorization ?? throw new ArgumentNullException(nameof(AuthorizationTokenProvider)); this.connectionPolicy = connectionPolicy; this.cancellationToken = cancellationToken; - - List serviceEndpoints = new () - { - serviceEndpoint - }; - - if (this.connectionPolicy.RegionalEndpoints != null - && this.connectionPolicy.RegionalEndpoints.Count > 0) - { - foreach (string regionalEndpoint in this.connectionPolicy.RegionalEndpoints) - { - // Add all of the regional endpoints to the service endpoints list. - serviceEndpoints.Add( - new Uri(regionalEndpoint)); - } - } - - this.serviceEndpointEnumerator = serviceEndpoints.GetEnumerator(); } private async Task GetDatabaseAccountAsync(Uri serviceEndpoint) @@ -106,31 +86,14 @@ await this.cosmosAuthorization.AddAuthorizationHeaderAsync( public async Task InitializeReaderAsync() { - int attemptCounter = 1; - List exceptionList = new (); - - while (this.serviceEndpointEnumerator.MoveNext()) - { - try - { - AccountProperties databaseAccount = await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( - defaultEndpoint: this.serviceEndpointEnumerator.Current, - locations: this.connectionPolicy.PreferredLocations, - getDatabaseAccountFn: this.GetDatabaseAccountAsync, - cancellationToken: this.cancellationToken); - - return databaseAccount; - } - catch (Exception ex) - { - DefaultTrace.TraceWarning("Attempt: {0}, Exception occurred while fetching account details: {1}", attemptCounter++, ex.Message); - exceptionList.Add(ex); - } - } + AccountProperties databaseAccount = await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( + defaultEndpoint: this.serviceEndpoint, + locations: this.connectionPolicy.PreferredLocations, + regionalEndpoints: this.connectionPolicy.RegionalEndpoints, + getDatabaseAccountFn: this.GetDatabaseAccountAsync, + cancellationToken: this.cancellationToken); - throw exceptionList.Count > 1 - ? new AggregateException("Unable to get account information.", exceptionList) - : exceptionList[0]; + return databaseAccount; } } } diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs index 15ac52bea0..dc7fba9bd6 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs @@ -116,13 +116,32 @@ public Uri GetHubUri() /// public static async Task GetDatabaseAccountFromAnyLocationsAsync( Uri defaultEndpoint, - IList? locations, + IList? locations, + IList regionalEndpoints, Func> getDatabaseAccountFn, CancellationToken cancellationToken) { + IList serviceEndpoints = new List() + { + // Add the default endpoint to the service endpoints list. + defaultEndpoint + }; + + if (regionalEndpoints != null + && regionalEndpoints.Count > 0) + { + foreach (string regionalEndpoint in regionalEndpoints) + { + // Add all of the regional endpoints to the service endpoints list. + serviceEndpoints.Add( + new Uri(regionalEndpoint)); + } + } + using (GetAccountPropertiesHelper threadSafeGetAccountHelper = new GetAccountPropertiesHelper( defaultEndpoint, - locations?.GetEnumerator(), + locations?.GetEnumerator(), + serviceEndpoints.GetEnumerator(), getDatabaseAccountFn, cancellationToken)) { @@ -137,7 +156,8 @@ private class GetAccountPropertiesHelper : IDisposable { private readonly CancellationTokenSource CancellationTokenSource; private readonly Uri DefaultEndpoint; - private readonly IEnumerator? Locations; + private readonly IEnumerator? Locations; + private readonly IEnumerator ServiceEndpointEnumerator; private readonly Func> GetDatabaseAccountFn; private readonly List TransientExceptions = new List(); private AccountProperties? AccountProperties = null; @@ -146,13 +166,15 @@ private class GetAccountPropertiesHelper : IDisposable public GetAccountPropertiesHelper( Uri defaultEndpoint, - IEnumerator? locations, + IEnumerator? locations, + IEnumerator serviceEndpointEnumerator, Func> getDatabaseAccountFn, CancellationToken cancellationToken) { this.DefaultEndpoint = defaultEndpoint; this.Locations = locations; - this.GetDatabaseAccountFn = getDatabaseAccountFn; + this.GetDatabaseAccountFn = getDatabaseAccountFn; + this.ServiceEndpointEnumerator = serviceEndpointEnumerator; this.CancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); } @@ -163,30 +185,40 @@ public async Task GetAccountPropertiesAsync() { return await this.GetOnlyGlobalEndpointAsync(); } - - Task globalEndpointTask = this.GetAndUpdateAccountPropertiesAsync(this.DefaultEndpoint); - - // Start a timer to start secondary requests in parallel. - Task timerTask = Task.Delay(TimeSpan.FromSeconds(5)); - await Task.WhenAny(globalEndpointTask, timerTask); - if (this.AccountProperties != null) - { - return this.AccountProperties; - } - - if (this.NonRetriableException != null) - { - ExceptionDispatchInfo.Capture(this.NonRetriableException).Throw(); + + // We first iterate through all the service endpoints, including the + // global and private endpoints to fetch the account information. If all the + // attempt fails to fetch the metadata, we will append the preferred region name + // as a suffix to the default global endpoint, and try to retrieve the account + // information. + HashSet tasksToWaitOn = new (); + while (this.ServiceEndpointEnumerator.MoveNext()) + { + Task serviceEndpointTask = this.GetAndUpdateAccountPropertiesAsync( + endpoint: this.ServiceEndpointEnumerator.Current); + + // Start a timer to start secondary requests in parallel. + Task timerTask = Task.Delay(TimeSpan.FromSeconds(5)); + await Task.WhenAny(serviceEndpointTask, timerTask); + if (this.AccountProperties != null) + { + return this.AccountProperties; + } + else + { + tasksToWaitOn.Add(serviceEndpointTask); + } + + if (this.NonRetriableException != null) + { + ExceptionDispatchInfo.Capture(this.NonRetriableException).Throw(); + } } // Start 2 additional tasks to try to get the account information // from the preferred region list since global account has not succeed yet. - HashSet tasksToWaitOn = new HashSet - { - globalEndpointTask, - this.TryGetAccountPropertiesFromAllLocationsAsync(), - this.TryGetAccountPropertiesFromAllLocationsAsync() - }; + tasksToWaitOn.Add(this.TryGetAccountPropertiesFromAllLocationsAsync()); + tasksToWaitOn.Add(this.TryGetAccountPropertiesFromAllLocationsAsync()); while (tasksToWaitOn.Any()) { @@ -602,7 +634,8 @@ internal async Task GetDatabaseAccountAsync(bool forceRefresh obsoleteValue: null, singleValueInitFunc: () => GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( this.defaultEndpoint, - this.connectionPolicy.PreferredLocations, + this.connectionPolicy.PreferredLocations, + this.connectionPolicy.RegionalEndpoints, this.GetDatabaseAccountAsync, this.cancellationTokenSource.Token), cancellationToken: this.cancellationTokenSource.Token, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs index 28e24d7e55..93db075e97 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs @@ -118,6 +118,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, + regionalEndpoints: null, getDatabaseAccountFn: (uri) => throw new Exception("The operation should be canceled and never make the network call."), cancellationTokenSource.Token); @@ -147,6 +148,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, + regionalEndpoints: null, getDatabaseAccountFn: (uri) => { count++; @@ -177,6 +179,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, + regionalEndpoints: null, getDatabaseAccountFn: async (uri) => { count++; @@ -208,6 +211,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, + regionalEndpoints: null, getDatabaseAccountFn: (uri) => { count++; @@ -239,6 +243,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, + regionalEndpoints: null, getDatabaseAccountFn: (uri) => { count++; @@ -307,6 +312,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "southeastasia", "northcentralus" }, + regionalEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); @@ -329,6 +335,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "southeastasia", "northcentralus" }, + regionalEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); stopwatch.Stop(); @@ -352,6 +359,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "southeastasia", "northcentralus" }, + regionalEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); @@ -373,6 +381,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "southeastasia", "northcentralus" }, + regionalEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); @@ -398,6 +407,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "westus6", "westus7", }, + regionalEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); From fb5af02f8b4035d0c7a0335674a5f7e9b9bb0c8c Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Wed, 24 Jan 2024 23:57:50 -0800 Subject: [PATCH 07/16] Code changes to address review comments. --- Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs | 2 +- Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs | 5 +++-- Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs index b6c11aca2a..bf148a67c5 100644 --- a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs @@ -99,7 +99,7 @@ public void SetPreferredLocations(IReadOnlyList regions) /// An instance of containing the regional endpoints /// provided by the customer. public void SetRegionalEndpoints( - ISet regionalEndpoints) + IEnumerable regionalEndpoints) { if (regionalEndpoints == null) { diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index 102db5161e..e3d4d46111 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -200,7 +200,8 @@ public string ApplicationName /// Should the global endpoint become inaccessible, the CosmosClient will attempt to obtain the account information issuing requests to the regional endpoints provided in . /// /// - /// Nevertheless, this parameter remains optional and is recommended for implementation when a customer has configured a private endpoint for their Cosmos DB account. + /// Nevertheless, this parameter remains optional and is recommended for implementation when a customer has configured a private endpoint with a custom DNS hostname + /// (instead of accountname-region.documents.azure.com) etc. for their Cosmos DB account. /// /// /// See also Diagnose @@ -220,7 +221,7 @@ public string ApplicationName /// /// /// High availability on regional outages - public ISet RegionalEndpoints { get; set; } + public IEnumerable RegionalEndpoints { get; set; } /// /// Get or set the maximum number of concurrent connections allowed for the target diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index d6101e77c1..18e7ff7878 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -309,7 +309,8 @@ public CosmosClientBuilder WithApplicationPreferredRegions(IReadOnlyList /// /// A set of string containing the regional private endpoints for the cosmos db account. /// - /// This function is optional and is recommended for implementation when a customer has configured one or more private endpoints for their Cosmos DB account. + /// This function is optional and is recommended for implementation when a customer has configured one or more private endpoints with a custom DNS + /// hostname (instead of accountname-region.documents.azure.com) etc. for their Cosmos DB account. /// /// /// The example below creates a new instance of with the regional endpoints. From 55f1ace33e88206f4d847fafa5c9360c575f8563 Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Thu, 25 Jan 2024 00:04:56 -0800 Subject: [PATCH 08/16] Code changes to fix minor API parameter. --- Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index 18e7ff7878..fd06112e10 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -326,7 +326,7 @@ public CosmosClientBuilder WithApplicationPreferredRegions(IReadOnlyList /// /// The current . /// - public CosmosClientBuilder WithRegionalEndpoints(ISet regionalEndpoints) + public CosmosClientBuilder WithRegionalEndpoints(IEnumerable regionalEndpoints) { this.clientOptions.RegionalEndpoints = regionalEndpoints; return this; From 94b00586113e11969cb17151d67cad43ab03e097 Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Thu, 25 Jan 2024 16:01:13 -0800 Subject: [PATCH 09/16] Code changes to update the API naming. --- .../src/ConnectionPolicy.cs | 12 ++--- .../src/CosmosClientOptions.cs | 12 ++--- .../src/Fluent/CosmosClientBuilder.cs | 16 +++---- .../Contracts/DotNetSDKAPI.json | 46 +++++++++---------- .../CosmosClientOptionsUnitTests.cs | 8 ++-- .../GatewayAccountReaderTests.cs | 16 +++---- 6 files changed, 55 insertions(+), 55 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs index bf148a67c5..1cc638c9eb 100644 --- a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs @@ -96,18 +96,18 @@ public void SetPreferredLocations(IReadOnlyList regions) /// Sets the regional endpoints required to fetch account information from /// private domain names. /// - /// An instance of containing the regional endpoints + /// An instance of containing the custom DNS endpoints /// provided by the customer. - public void SetRegionalEndpoints( - IEnumerable regionalEndpoints) + public void SetCustomEndpoints( + IEnumerable customEndpoints) { - if (regionalEndpoints == null) + if (customEndpoints == null) { - throw new ArgumentNullException(nameof(regionalEndpoints)); + throw new ArgumentNullException(nameof(customEndpoints)); } this.regionalEndpoints.Clear(); - foreach (string endpoint in regionalEndpoints) + foreach (string endpoint in customEndpoints) { this.regionalEndpoints.Add(endpoint); } diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index e3d4d46111..6267343766 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -192,12 +192,12 @@ public string ApplicationName public IReadOnlyList ApplicationPreferredRegions { get; set; } /// - /// Gets and sets the regional private endpoints for geo-replicated database accounts in the Azure Cosmos DB service. + /// Gets and sets the custom private endpoints for geo-replicated database accounts in the Azure Cosmos DB service. /// /// /// /// During the CosmosClient initialization the account information, including the available regions, is obtained from the . - /// Should the global endpoint become inaccessible, the CosmosClient will attempt to obtain the account information issuing requests to the regional endpoints provided in . + /// Should the global endpoint become inaccessible, the CosmosClient will attempt to obtain the account information issuing requests to the custom endpoints provided in . /// /// /// Nevertheless, this parameter remains optional and is recommended for implementation when a customer has configured a private endpoint with a custom DNS hostname @@ -213,7 +213,7 @@ public string ApplicationName /// (){ "custom.p-1.documents.azure.com", "custom.p-2.documents.azure.com" } + /// AccountInitializationCustomEndpoints = new HashSet(){ "custom.p-1.documents.azure.com", "custom.p-2.documents.azure.com" } /// }; /// /// CosmosClient client = new CosmosClient("endpoint", "key", clientOptions); @@ -221,7 +221,7 @@ public string ApplicationName /// /// /// High availability on regional outages - public IEnumerable RegionalEndpoints { get; set; } + public IEnumerable AccountInitializationCustomEndpoints { get; set; } /// /// Get or set the maximum number of concurrent connections allowed for the target @@ -828,9 +828,9 @@ internal virtual ConnectionPolicy GetConnectionPolicy(int clientId) connectionPolicy.SetPreferredLocations(mappedRegions); } - if (this.RegionalEndpoints != null) + if (this.AccountInitializationCustomEndpoints != null) { - connectionPolicy.SetRegionalEndpoints(this.RegionalEndpoints); + connectionPolicy.SetCustomEndpoints(this.AccountInitializationCustomEndpoints); } if (this.MaxRetryAttemptsOnRateLimitedRequests != null) diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index fd06112e10..15a330405f 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -302,12 +302,12 @@ public CosmosClientBuilder WithApplicationPreferredRegions(IReadOnlyList } /// - /// Sets the regional private endpoints for geo-replicated database accounts in the Azure Cosmos DB service. + /// Sets the custom private endpoints for geo-replicated database accounts in the Azure Cosmos DB service. /// During the CosmosClient initialization the account information, including the available regions, is obtained from the . - /// Should the global endpoint become inaccessible, the CosmosClient will attempt to obtain the account information issuing requests to the regional endpoints - /// provided in the regionalEndpoints set. + /// Should the global endpoint become inaccessible, the CosmosClient will attempt to obtain the account information issuing requests to the custom endpoints + /// provided in the customAccountEndpoints list. /// - /// A set of string containing the regional private endpoints for the cosmos db account. + /// An instance of of string containing the custom private endpoints for the cosmos db account. /// /// This function is optional and is recommended for implementation when a customer has configured one or more private endpoints with a custom DNS /// hostname (instead of accountname-region.documents.azure.com) etc. for their Cosmos DB account. @@ -319,16 +319,16 @@ public CosmosClientBuilder WithApplicationPreferredRegions(IReadOnlyList /// CosmosClientBuilder cosmosClientBuilder = new CosmosClientBuilder( /// accountEndpoint: "https://testcosmos.documents.azure.com:443/", /// authKeyOrResourceToken: "SuperSecretKey") - /// .WithRegionalEndpoints(new HashSet() { "https://region-1.documents-test.windows-int.net:443/", "https://region-2.documents-test.windows-int.net:443/" }); + /// .WithCustomAccountEndpoints(new HashSet() { "https://region-1.documents-test.windows-int.net:443/", "https://region-2.documents-test.windows-int.net:443/" }); /// CosmosClient client = cosmosClientBuilder.Build(); /// ]]> /// /// /// The current . - /// - public CosmosClientBuilder WithRegionalEndpoints(IEnumerable regionalEndpoints) + /// + public CosmosClientBuilder WithCustomAccountEndpoints(IEnumerable customAccountEndpoints) { - this.clientOptions.RegionalEndpoints = regionalEndpoints; + this.clientOptions.AccountInitializationCustomEndpoints = customAccountEndpoints; return this; } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json index 1ded6f70d6..7e35a28970 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json @@ -2770,29 +2770,29 @@ ], "MethodInfo": "Microsoft.Azure.Cosmos.CosmosSerializer Serializer;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.CosmosSerializer get_Serializer();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_Serializer(Microsoft.Azure.Cosmos.CosmosSerializer);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Collections.Generic.IReadOnlyList`1[System.String] ApplicationPreferredRegions": { + "System.Collections.Generic.IEnumerable`1[System.String] AccountInitializationCustomEndpoints": { "Type": "Property", "Attributes": [], - "MethodInfo": "System.Collections.Generic.IReadOnlyList`1[System.String] ApplicationPreferredRegions;CanRead:True;CanWrite:True;System.Collections.Generic.IReadOnlyList`1[System.String] get_ApplicationPreferredRegions();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_ApplicationPreferredRegions(System.Collections.Generic.IReadOnlyList`1[System.String]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "System.Collections.Generic.IEnumerable`1[System.String] AccountInitializationCustomEndpoints;CanRead:True;CanWrite:True;System.Collections.Generic.IEnumerable`1[System.String] get_AccountInitializationCustomEndpoints();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_AccountInitializationCustomEndpoints(System.Collections.Generic.IEnumerable`1[System.String]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Collections.Generic.IReadOnlyList`1[System.String] get_ApplicationPreferredRegions()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "System.Collections.Generic.IEnumerable`1[System.String] get_AccountInitializationCustomEndpoints()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ "CompilerGeneratedAttribute" ], - "MethodInfo": "System.Collections.Generic.IReadOnlyList`1[System.String] get_ApplicationPreferredRegions();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "System.Collections.Generic.IEnumerable`1[System.String] get_AccountInitializationCustomEndpoints();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Collections.Generic.ISet`1[System.String] get_RegionalEndpoints()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "System.Collections.Generic.IReadOnlyList`1[System.String] ApplicationPreferredRegions": { + "Type": "Property", + "Attributes": [], + "MethodInfo": "System.Collections.Generic.IReadOnlyList`1[System.String] ApplicationPreferredRegions;CanRead:True;CanWrite:True;System.Collections.Generic.IReadOnlyList`1[System.String] get_ApplicationPreferredRegions();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_ApplicationPreferredRegions(System.Collections.Generic.IReadOnlyList`1[System.String]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Collections.Generic.IReadOnlyList`1[System.String] get_ApplicationPreferredRegions()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ "CompilerGeneratedAttribute" ], - "MethodInfo": "System.Collections.Generic.ISet`1[System.String] get_RegionalEndpoints();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "System.Collections.Generic.ISet`1[System.String] RegionalEndpoints": { - "Type": "Property", - "Attributes": [], - "MethodInfo": "System.Collections.Generic.ISet`1[System.String] RegionalEndpoints;CanRead:True;CanWrite:True;System.Collections.Generic.ISet`1[System.String] get_RegionalEndpoints();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_RegionalEndpoints(System.Collections.Generic.ISet`1[System.String]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "System.Collections.Generic.IReadOnlyList`1[System.String] get_ApplicationPreferredRegions();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "System.Collections.ObjectModel.Collection`1[Microsoft.Azure.Cosmos.RequestHandler] CustomHandlers[Newtonsoft.Json.JsonConverterAttribute(typeof(Microsoft.Azure.Cosmos.CosmosClientOptions+ClientOptionJsonConverter))]": { "Type": "Property", @@ -2993,6 +2993,13 @@ "Attributes": [], "MethodInfo": "[Void .ctor(), Void .ctor()]" }, + "Void set_AccountInitializationCustomEndpoints(System.Collections.Generic.IEnumerable`1[System.String])[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "Void set_AccountInitializationCustomEndpoints(System.Collections.Generic.IEnumerable`1[System.String]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void set_AllowBulkExecution(Boolean)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ @@ -3108,13 +3115,6 @@ "Attributes": [], "MethodInfo": "Void set_PortReuseMode(System.Nullable`1[Microsoft.Azure.Cosmos.PortReuseMode]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Void set_RegionalEndpoints(System.Collections.Generic.ISet`1[System.String])[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Void set_RegionalEndpoints(System.Collections.Generic.ISet`1[System.String]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, "Void set_RequestTimeout(System.TimeSpan)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ @@ -4603,6 +4603,11 @@ "Attributes": [], "MethodInfo": "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithContentResponseOnWrite(Boolean);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithCustomAccountEndpoints(System.Collections.Generic.IEnumerable`1[System.String])": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithCustomAccountEndpoints(System.Collections.Generic.IEnumerable`1[System.String]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithCustomSerializer(Microsoft.Azure.Cosmos.CosmosSerializer)": { "Type": "Method", "Attributes": [], @@ -4618,11 +4623,6 @@ "Attributes": [], "MethodInfo": "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithLimitToEndpoint(Boolean);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithRegionalEndpoints(System.Collections.Generic.ISet`1[System.String])": { - "Type": "Method", - "Attributes": [], - "MethodInfo": "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithRegionalEndpoints(System.Collections.Generic.ISet`1[System.String]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithRequestTimeout(System.TimeSpan)": { "Type": "Method", "Attributes": [], diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs index 6405edce9a..71527ed6e9 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs @@ -171,7 +171,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() portReuseMode, enableTcpConnectionEndpointRediscovery) .WithApplicationPreferredRegions(preferredLocations) - .WithRegionalEndpoints(regionalEndpoints) + .WithCustomAccountEndpoints(regionalEndpoints) .WithClientTelemetryOptions(new CosmosClientTelemetryOptions() { DisableDistributedTracing = false, @@ -192,7 +192,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.AreEqual(portReuseMode, clientOptions.PortReuseMode); Assert.IsTrue(clientOptions.EnableTcpConnectionEndpointRediscovery); CollectionAssert.AreEqual(preferredLocations.ToArray(), clientOptions.ApplicationPreferredRegions.ToArray()); - CollectionAssert.AreEqual(regionalEndpoints.ToArray(), clientOptions.RegionalEndpoints.ToArray()); + CollectionAssert.AreEqual(regionalEndpoints.ToArray(), clientOptions.AccountInitializationCustomEndpoints.ToArray()); Assert.AreEqual(TimeSpan.FromMilliseconds(100), clientOptions.CosmosClientTelemetryOptions.CosmosThresholdOptions.PointOperationLatencyThreshold); Assert.AreEqual(TimeSpan.FromMilliseconds(100), clientOptions.CosmosClientTelemetryOptions.CosmosThresholdOptions.NonPointOperationLatencyThreshold); Assert.IsFalse(clientOptions.CosmosClientTelemetryOptions.DisableDistributedTracing); @@ -325,7 +325,7 @@ public void CosmosClientOptions_WhenPartitionLevelFailoverEnabledAndPreferredReg Regions.WestUS, Regions.EastAsia, }) - .WithRegionalEndpoints( + .WithCustomAccountEndpoints( new HashSet() { "https://testfed2.documents-test.windows-int.net:443/", @@ -349,7 +349,7 @@ public void CosmosClientOptions_WhenPartitionLevelFailoverEnabledAndPreferredReg Assert.AreEqual(consistencyLevel, clientOptions.ConsistencyLevel); Assert.IsTrue(clientOptions.EnablePartitionLevelFailover); Assert.IsNotNull(clientOptions.ApplicationPreferredRegions); - Assert.IsNotNull(clientOptions.RegionalEndpoints); + Assert.IsNotNull(clientOptions.AccountInitializationCustomEndpoints); } finally { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs index 7275c2f613..646d3f7d35 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs @@ -103,10 +103,10 @@ public void DocumentClient_BuildHttpClientFactory_WithFactory() [TestMethod] [Owner("dkunda")] - [DataRow(true, DisplayName = "Validate that when regional endpoints are provided in the connection policy, the request will be retried in the regional endpoints.")] - [DataRow(false, DisplayName = "Validate that when regional endpoints are not provided in the connection policy, the request will be failed in the primary endpoint.")] - public async Task InitializeReaderAsync_WhenRegionalEndpointsProvided_ShouldRetryWithRegionalEndpointsWhenPrimaryFails( - bool regionalEndpointsProvided) + [DataRow(true, DisplayName = "Validate that when custom endpoints are provided in the connection policy, the request will be retried in the regional endpoints.")] + [DataRow(false, DisplayName = "Validate that when custom endpoints are not provided in the connection policy, the request will be failed in the primary endpoint.")] + public async Task InitializeReaderAsync_WhenCustomEndpointsProvided_ShouldRetryWithPrivateCustomEndpointsWhenPrimaryFails( + bool customEndpointsProvided) { string accountPropertiesResponse = "{\r\n \"_self\": \"\",\r\n \"id\": \"localhost\",\r\n \"_rid\": \"127.0.0.1\",\r\n \"media\": \"//media/\",\r\n \"addresses\": \"//addresses/\",\r\n \"_dbs\": \"//dbs/\",\r\n \"writableLocations\": [\r\n {\r\n \"name\": \"South Central US\",\r\n \"databaseAccountEndpoint\": \"https://127.0.0.1:8081/\"\r\n }" + "\r\n ],\r\n \"readableLocations\": [\r\n {\r\n \"name\": \"South Central US\",\r\n \"databaseAccountEndpoint\": \"https://127.0.0.1:8081/\"\r\n }\r\n ],\r\n \"enableMultipleWriteLocations\": false,\r\n \"userReplicationPolicy\": {\r\n \"asyncReplication\": false,\r\n \"minReplicaSetSize\": 1,\r\n \"maxReplicasetSize\": 4\r\n },\r\n \"userConsistencyPolicy\": {\r\n " + @@ -139,9 +139,9 @@ public async Task InitializeReaderAsync_WhenRegionalEndpointsProvided_ShouldRetr ConnectionMode = ConnectionMode.Direct, }; - if (regionalEndpointsProvided) + if (customEndpointsProvided) { - connectionPolicy.SetRegionalEndpoints( + connectionPolicy.SetCustomEndpoints( new HashSet() { "https://testfed2.documents-test.windows-int.net:443/", @@ -156,7 +156,7 @@ public async Task InitializeReaderAsync_WhenRegionalEndpointsProvided_ShouldRetr connectionPolicy: connectionPolicy, httpClient: mockHttpClient.Object); - if (regionalEndpointsProvided) + if (customEndpointsProvided) { AccountProperties accountProperties = await accountReader.InitializeReaderAsync(); @@ -192,7 +192,7 @@ public async Task InitializeReaderAsync_WhenRegionalEndpointsProvided_ShouldThro ConnectionMode = ConnectionMode.Direct, }; - connectionPolicy.SetRegionalEndpoints( + connectionPolicy.SetCustomEndpoints( new HashSet() { "https://testfed2.documents-test.windows-int.net:443/", From 9cc57add224d43449405130f0e04acf0bf4e0c9c Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Thu, 25 Jan 2024 16:50:13 -0800 Subject: [PATCH 10/16] Code changes to update some attribute names. --- .../src/ConnectionPolicy.cs | 27 ++++++++----------- .../src/GatewayAccountReader.cs | 3 +-- .../src/Routing/GlobalEndpointManager.cs | 14 +++++----- .../GlobalEndpointManagerTest.cs | 20 +++++++------- 4 files changed, 29 insertions(+), 35 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs index 1cc638c9eb..97cabbb950 100644 --- a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs @@ -29,7 +29,7 @@ internal sealed class ConnectionPolicy private Protocol connectionProtocol; private ObservableCollection preferredLocations; - private ObservableCollection regionalEndpoints; + private ObservableCollection customPrivateEndpoints; /// /// Initializes a new instance of the class to connect to the Azure Cosmos DB service. @@ -44,7 +44,7 @@ public ConnectionPolicy() this.MediaReadMode = MediaReadMode.Buffered; this.UserAgentContainer = new UserAgentContainer(clientId: 0); this.preferredLocations = new ObservableCollection(); - this.regionalEndpoints = new ObservableCollection(); + this.customPrivateEndpoints = new ObservableCollection(); this.EnableEndpointDiscovery = true; this.MaxConnectionLimit = defaultMaxConcurrentConnectionLimit; this.RetryOptions = new RetryOptions(); @@ -93,7 +93,7 @@ public void SetPreferredLocations(IReadOnlyList regions) } /// - /// Sets the regional endpoints required to fetch account information from + /// Sets the custom private endpoints required to fetch account information from /// private domain names. /// /// An instance of containing the custom DNS endpoints @@ -106,10 +106,10 @@ public void SetCustomEndpoints( throw new ArgumentNullException(nameof(customEndpoints)); } - this.regionalEndpoints.Clear(); + this.customPrivateEndpoints.Clear(); foreach (string endpoint in customEndpoints) { - this.regionalEndpoints.Add(endpoint); + this.customPrivateEndpoints.Add(endpoint); } } @@ -294,25 +294,20 @@ public Collection PreferredLocations } /// - /// Gets and sets the preferred locations (regions) for geo-replicated database accounts in the Azure Cosmos DB service. - /// For example, "East US" as the preferred location. + /// Gets the custom private endpoints for geo-replicated database accounts in the Azure Cosmos DB service. /// /// /// - /// When is true and the value of this property is non-empty, - /// the SDK uses the locations in the collection in the order they are specified to perform operations, - /// otherwise if the value of this property is not specified, - /// the SDK uses the write region as the preferred location for all operations. - /// - /// - /// If is set to false, the value of this property is ignored. + /// During the CosmosClient initialization the account information, including the available regions, is obtained from the . + /// Should the global endpoint become inaccessible, the CosmosClient will attempt to obtain the account information issuing requests to the custom endpoints + /// provided in the customAccountEndpoints list. /// /// - public Collection RegionalEndpoints + public Collection CustomPrivateEndpoints { get { - return this.regionalEndpoints; + return this.customPrivateEndpoints; } } diff --git a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs index c225438c6d..1fe880f800 100644 --- a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs +++ b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs @@ -5,7 +5,6 @@ namespace Microsoft.Azure.Cosmos { using System; - using System.Collections.Generic; using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -89,7 +88,7 @@ public async Task InitializeReaderAsync() AccountProperties databaseAccount = await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( defaultEndpoint: this.serviceEndpoint, locations: this.connectionPolicy.PreferredLocations, - regionalEndpoints: this.connectionPolicy.RegionalEndpoints, + customPrivateEndpoints: this.connectionPolicy.CustomPrivateEndpoints, getDatabaseAccountFn: this.GetDatabaseAccountAsync, cancellationToken: this.cancellationToken); diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs index dc7fba9bd6..81fed6c66a 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs @@ -117,7 +117,7 @@ public Uri GetHubUri() public static async Task GetDatabaseAccountFromAnyLocationsAsync( Uri defaultEndpoint, IList? locations, - IList regionalEndpoints, + IList customPrivateEndpoints, Func> getDatabaseAccountFn, CancellationToken cancellationToken) { @@ -127,14 +127,14 @@ public static async Task GetDatabaseAccountFromAnyLocationsAs defaultEndpoint }; - if (regionalEndpoints != null - && regionalEndpoints.Count > 0) + if (customPrivateEndpoints != null + && customPrivateEndpoints.Count > 0) { - foreach (string regionalEndpoint in regionalEndpoints) + foreach (string customEndpoint in customPrivateEndpoints) { - // Add all of the regional endpoints to the service endpoints list. + // Add all of the custom private endpoints to the service endpoints list. serviceEndpoints.Add( - new Uri(regionalEndpoint)); + new Uri(customEndpoint)); } } @@ -635,7 +635,7 @@ internal async Task GetDatabaseAccountAsync(bool forceRefresh singleValueInitFunc: () => GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( this.defaultEndpoint, this.connectionPolicy.PreferredLocations, - this.connectionPolicy.RegionalEndpoints, + this.connectionPolicy.CustomPrivateEndpoints, this.GetDatabaseAccountAsync, this.cancellationTokenSource.Token), cancellationToken: this.cancellationTokenSource.Token, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs index 93db075e97..c08ce53d07 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs @@ -118,7 +118,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, - regionalEndpoints: null, + customPrivateEndpoints: null, getDatabaseAccountFn: (uri) => throw new Exception("The operation should be canceled and never make the network call."), cancellationTokenSource.Token); @@ -148,7 +148,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, - regionalEndpoints: null, + customPrivateEndpoints: null, getDatabaseAccountFn: (uri) => { count++; @@ -179,7 +179,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, - regionalEndpoints: null, + customPrivateEndpoints: null, getDatabaseAccountFn: async (uri) => { count++; @@ -211,7 +211,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, - regionalEndpoints: null, + customPrivateEndpoints: null, getDatabaseAccountFn: (uri) => { count++; @@ -243,7 +243,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, - regionalEndpoints: null, + customPrivateEndpoints: null, getDatabaseAccountFn: (uri) => { count++; @@ -312,7 +312,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "southeastasia", "northcentralus" }, - regionalEndpoints: null, + customPrivateEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); @@ -335,7 +335,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "southeastasia", "northcentralus" }, - regionalEndpoints: null, + customPrivateEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); stopwatch.Stop(); @@ -359,7 +359,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "southeastasia", "northcentralus" }, - regionalEndpoints: null, + customPrivateEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); @@ -381,7 +381,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "southeastasia", "northcentralus" }, - regionalEndpoints: null, + customPrivateEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); @@ -407,7 +407,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "westus6", "westus7", }, - regionalEndpoints: null, + customPrivateEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); From 348e4a7edb32a4984ae0ed3c13cd9e383e1e8733 Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Mon, 29 Jan 2024 15:49:45 -0800 Subject: [PATCH 11/16] Code changes to refactor service endpoint creation logic. --- .../src/Routing/GlobalEndpointManager.cs | 181 +++++++++--------- 1 file changed, 87 insertions(+), 94 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs index 81fed6c66a..e9c4469a1c 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs @@ -121,27 +121,10 @@ public static async Task GetDatabaseAccountFromAnyLocationsAs Func> getDatabaseAccountFn, CancellationToken cancellationToken) { - IList serviceEndpoints = new List() - { - // Add the default endpoint to the service endpoints list. - defaultEndpoint - }; - - if (customPrivateEndpoints != null - && customPrivateEndpoints.Count > 0) - { - foreach (string customEndpoint in customPrivateEndpoints) - { - // Add all of the custom private endpoints to the service endpoints list. - serviceEndpoints.Add( - new Uri(customEndpoint)); - } - } - using (GetAccountPropertiesHelper threadSafeGetAccountHelper = new GetAccountPropertiesHelper( defaultEndpoint, - locations?.GetEnumerator(), - serviceEndpoints.GetEnumerator(), + locations, + customPrivateEndpoints, getDatabaseAccountFn, cancellationToken)) { @@ -166,16 +149,19 @@ private class GetAccountPropertiesHelper : IDisposable public GetAccountPropertiesHelper( Uri defaultEndpoint, - IEnumerator? locations, - IEnumerator serviceEndpointEnumerator, + IList? locations, + IList customPrivateEndpoints, Func> getDatabaseAccountFn, CancellationToken cancellationToken) { this.DefaultEndpoint = defaultEndpoint; - this.Locations = locations; + this.Locations = locations?.GetEnumerator(); this.GetDatabaseAccountFn = getDatabaseAccountFn; - this.ServiceEndpointEnumerator = serviceEndpointEnumerator; - this.CancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + this.CancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + this.ServiceEndpointEnumerator = GetAccountPropertiesHelper.GetServiceEndpointEnumerator( + defaultEndpoint, + locations, + customPrivateEndpoints); } public async Task GetAccountPropertiesAsync() @@ -186,39 +172,29 @@ public async Task GetAccountPropertiesAsync() return await this.GetOnlyGlobalEndpointAsync(); } - // We first iterate through all the service endpoints, including the - // global and private endpoints to fetch the account information. If all the - // attempt fails to fetch the metadata, we will append the preferred region name - // as a suffix to the default global endpoint, and try to retrieve the account - // information. - HashSet tasksToWaitOn = new (); - while (this.ServiceEndpointEnumerator.MoveNext()) + Task globalEndpointTask = this.GetAndUpdateAccountPropertiesAsync(this.DefaultEndpoint); + + // Start a timer to start secondary requests in parallel. + Task timerTask = Task.Delay(TimeSpan.FromSeconds(5)); + await Task.WhenAny(globalEndpointTask, timerTask); + if (this.AccountProperties != null) { - Task serviceEndpointTask = this.GetAndUpdateAccountPropertiesAsync( - endpoint: this.ServiceEndpointEnumerator.Current); + return this.AccountProperties; + } - // Start a timer to start secondary requests in parallel. - Task timerTask = Task.Delay(TimeSpan.FromSeconds(5)); - await Task.WhenAny(serviceEndpointTask, timerTask); - if (this.AccountProperties != null) - { - return this.AccountProperties; - } - else - { - tasksToWaitOn.Add(serviceEndpointTask); - } + if (this.NonRetriableException != null) + { + ExceptionDispatchInfo.Capture(this.NonRetriableException).Throw(); + } - if (this.NonRetriableException != null) - { - ExceptionDispatchInfo.Capture(this.NonRetriableException).Throw(); - } - } - - // Start 2 additional tasks to try to get the account information - // from the preferred region list since global account has not succeed yet. - tasksToWaitOn.Add(this.TryGetAccountPropertiesFromAllLocationsAsync()); - tasksToWaitOn.Add(this.TryGetAccountPropertiesFromAllLocationsAsync()); + // Start 2 additional tasks to try to get the account information + // from the preferred region list since global account has not succeed yet. + HashSet tasksToWaitOn = new HashSet + { + globalEndpointTask, + this.TryGetAccountPropertiesFromAllLocationsAsync(), + this.TryGetAccountPropertiesFromAllLocationsAsync() + }; while (tasksToWaitOn.Any()) { @@ -286,49 +262,23 @@ private async Task GetOnlyGlobalEndpointAsync() /// list of locations. /// private async Task TryGetAccountPropertiesFromAllLocationsAsync() - { - while (this.TryMoveNextLocationThreadSafe( - out string? location)) - { - if (location == null) - { - DefaultTrace.TraceCritical("GlobalEndpointManager: location is null for TryMoveNextLocationThreadSafe"); - return; - } - - await this.TryGetAccountPropertiesFromRegionalEndpointsAsync(location); - } - } - - private bool TryMoveNextLocationThreadSafe( - out string? location) - { - if (this.CancellationTokenSource.IsCancellationRequested - || this.Locations == null) - { - location = null; - return false; - } - - lock (this.Locations) - { - if (!this.Locations.MoveNext()) - { - location = null; - return false; - } - - location = this.Locations.Current; - return true; + { + if (this.CancellationTokenSource.IsCancellationRequested) + { + return; + } + + // We first iterate through all the private endpoints to fetch the account information. + // If all the attempt fails to fetch the metadata from the private endpoints, we will + // attempt to retrieve the account information from the regional endpoints constructed + // using the preferred regions list. + while (this.ServiceEndpointEnumerator.MoveNext()) + { + await this.GetAndUpdateAccountPropertiesAsync( + endpoint: this.ServiceEndpointEnumerator.Current); } } - private Task TryGetAccountPropertiesFromRegionalEndpointsAsync(string location) - { - return this.GetAndUpdateAccountPropertiesAsync( - LocationHelper.GetLocationEndpoint(this.DefaultEndpoint, location)); - } - private async Task GetAndUpdateAccountPropertiesAsync(Uri endpoint) { try @@ -381,6 +331,49 @@ private static bool IsNonRetriableException(Exception exception) return false; } + /// + /// Returns an enumerator containing the private and regional service endpoints to iterate over. + /// + /// An instance of containing the default global endpoint. + /// An instance of containing the preferred location names. + /// An instance of containing the private endpoints. + /// An instance of containing the service endpoints. + private static IEnumerator GetServiceEndpointEnumerator( + Uri defaultEndpoint, + IList? locations, + IList customPrivateEndpoints) + { + // We first iterate through all the private endpoints and add them to the service endpoints list. + IList serviceEndpoints = new List(); + + if (customPrivateEndpoints != null + && customPrivateEndpoints.Count > 0) + { + foreach (string customEndpoint in customPrivateEndpoints) + { + // Add all of the custom private endpoints to the service endpoints list. + serviceEndpoints.Add( + new Uri(customEndpoint)); + } + } + + // The next step is to construct and add all of the regional endpoints to the service endpoints list. + // The regional endpoints will be created by appending the preferred region name as a suffix to the + // default global endpoint. + if (locations != null + && locations.Count > 0) + { + foreach (string location in locations) + { + // Add all of the regional endpoints to the service endpoints list. + serviceEndpoints.Add( + LocationHelper.GetLocationEndpoint(defaultEndpoint, location)); + } + } + + return serviceEndpoints.GetEnumerator(); + } + public void Dispose() { if (Interlocked.Increment(ref this.disposeCounter) == 1) From ea51744c7c21d56e1df2989e24a4f325d33ddd60 Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Mon, 29 Jan 2024 16:12:50 -0800 Subject: [PATCH 12/16] Code changes to address review comments. --- .../src/ConnectionPolicy.cs | 14 ++++++------- .../src/CosmosClientOptions.cs | 6 +++--- .../src/Fluent/CosmosClientBuilder.cs | 4 ++-- .../src/GatewayAccountReader.cs | 2 +- .../src/Routing/GlobalEndpointManager.cs | 20 +++++++++---------- .../GatewayAccountReaderTests.cs | 4 ++-- .../GlobalEndpointManagerTest.cs | 20 +++++++++---------- 7 files changed, 35 insertions(+), 35 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs index 97cabbb950..ba781c01d0 100644 --- a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs @@ -29,7 +29,7 @@ internal sealed class ConnectionPolicy private Protocol connectionProtocol; private ObservableCollection preferredLocations; - private ObservableCollection customPrivateEndpoints; + private ObservableCollection accountInitializationCustomEndpoints; /// /// Initializes a new instance of the class to connect to the Azure Cosmos DB service. @@ -44,7 +44,7 @@ public ConnectionPolicy() this.MediaReadMode = MediaReadMode.Buffered; this.UserAgentContainer = new UserAgentContainer(clientId: 0); this.preferredLocations = new ObservableCollection(); - this.customPrivateEndpoints = new ObservableCollection(); + this.accountInitializationCustomEndpoints = new ObservableCollection(); this.EnableEndpointDiscovery = true; this.MaxConnectionLimit = defaultMaxConcurrentConnectionLimit; this.RetryOptions = new RetryOptions(); @@ -98,7 +98,7 @@ public void SetPreferredLocations(IReadOnlyList regions) /// /// An instance of containing the custom DNS endpoints /// provided by the customer. - public void SetCustomEndpoints( + public void SetAccountInitializationCustomEndpoints( IEnumerable customEndpoints) { if (customEndpoints == null) @@ -106,10 +106,10 @@ public void SetCustomEndpoints( throw new ArgumentNullException(nameof(customEndpoints)); } - this.customPrivateEndpoints.Clear(); + this.accountInitializationCustomEndpoints.Clear(); foreach (string endpoint in customEndpoints) { - this.customPrivateEndpoints.Add(endpoint); + this.accountInitializationCustomEndpoints.Add(endpoint); } } @@ -303,11 +303,11 @@ public Collection PreferredLocations /// provided in the customAccountEndpoints list. /// /// - public Collection CustomPrivateEndpoints + public Collection AccountInitializationCustomEndpoints { get { - return this.customPrivateEndpoints; + return this.accountInitializationCustomEndpoints; } } diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index b727bbeb7d..043b9559b8 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -193,7 +193,7 @@ public string ApplicationName public IReadOnlyList ApplicationPreferredRegions { get; set; } /// - /// Gets and sets the custom private endpoints for geo-replicated database accounts in the Azure Cosmos DB service. + /// Gets and sets the custom endpoints to use for account initialization for geo-replicated database accounts in the Azure Cosmos DB service. /// /// /// @@ -201,7 +201,7 @@ public string ApplicationName /// Should the global endpoint become inaccessible, the CosmosClient will attempt to obtain the account information issuing requests to the custom endpoints provided in . /// /// - /// Nevertheless, this parameter remains optional and is recommended for implementation when a customer has configured a private endpoint with a custom DNS hostname + /// Nevertheless, this parameter remains optional and is recommended for implementation when a customer has configured an endpoint with a custom DNS hostname /// (instead of accountname-region.documents.azure.com) etc. for their Cosmos DB account. /// /// @@ -833,7 +833,7 @@ internal virtual ConnectionPolicy GetConnectionPolicy(int clientId) if (this.AccountInitializationCustomEndpoints != null) { - connectionPolicy.SetCustomEndpoints(this.AccountInitializationCustomEndpoints); + connectionPolicy.SetAccountInitializationCustomEndpoints(this.AccountInitializationCustomEndpoints); } if (this.MaxRetryAttemptsOnRateLimitedRequests != null) diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index 4c6151bc18..f761b82180 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -309,14 +309,14 @@ public CosmosClientBuilder WithApplicationPreferredRegions(IReadOnlyList } /// - /// Sets the custom private endpoints for geo-replicated database accounts in the Azure Cosmos DB service. + /// Sets the custom endpoints to use for account initialization for geo-replicated database accounts in the Azure Cosmos DB service. /// During the CosmosClient initialization the account information, including the available regions, is obtained from the . /// Should the global endpoint become inaccessible, the CosmosClient will attempt to obtain the account information issuing requests to the custom endpoints /// provided in the customAccountEndpoints list. /// /// An instance of of string containing the custom private endpoints for the cosmos db account. /// - /// This function is optional and is recommended for implementation when a customer has configured one or more private endpoints with a custom DNS + /// This function is optional and is recommended for implementation when a customer has configured one or more endpoints with a custom DNS /// hostname (instead of accountname-region.documents.azure.com) etc. for their Cosmos DB account. /// /// diff --git a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs index 1fe880f800..1a2c3e14a4 100644 --- a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs +++ b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs @@ -88,7 +88,7 @@ public async Task InitializeReaderAsync() AccountProperties databaseAccount = await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( defaultEndpoint: this.serviceEndpoint, locations: this.connectionPolicy.PreferredLocations, - customPrivateEndpoints: this.connectionPolicy.CustomPrivateEndpoints, + accountInitializationCustomEndpoints: this.connectionPolicy.AccountInitializationCustomEndpoints, getDatabaseAccountFn: this.GetDatabaseAccountAsync, cancellationToken: this.cancellationToken); diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs index e9c4469a1c..83a5865a03 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs @@ -117,14 +117,14 @@ public Uri GetHubUri() public static async Task GetDatabaseAccountFromAnyLocationsAsync( Uri defaultEndpoint, IList? locations, - IList customPrivateEndpoints, + IList accountInitializationCustomEndpoints, Func> getDatabaseAccountFn, CancellationToken cancellationToken) { using (GetAccountPropertiesHelper threadSafeGetAccountHelper = new GetAccountPropertiesHelper( defaultEndpoint, locations, - customPrivateEndpoints, + accountInitializationCustomEndpoints, getDatabaseAccountFn, cancellationToken)) { @@ -150,7 +150,7 @@ private class GetAccountPropertiesHelper : IDisposable public GetAccountPropertiesHelper( Uri defaultEndpoint, IList? locations, - IList customPrivateEndpoints, + IList accountInitializationCustomEndpoints, Func> getDatabaseAccountFn, CancellationToken cancellationToken) { @@ -161,7 +161,7 @@ public GetAccountPropertiesHelper( this.ServiceEndpointEnumerator = GetAccountPropertiesHelper.GetServiceEndpointEnumerator( defaultEndpoint, locations, - customPrivateEndpoints); + accountInitializationCustomEndpoints); } public async Task GetAccountPropertiesAsync() @@ -336,20 +336,20 @@ private static bool IsNonRetriableException(Exception exception) /// /// An instance of containing the default global endpoint. /// An instance of containing the preferred location names. - /// An instance of containing the private endpoints. + /// An instance of containing the custom private endpoints. /// An instance of containing the service endpoints. private static IEnumerator GetServiceEndpointEnumerator( Uri defaultEndpoint, IList? locations, - IList customPrivateEndpoints) + IList accountInitializationCustomEndpoints) { // We first iterate through all the private endpoints and add them to the service endpoints list. IList serviceEndpoints = new List(); - if (customPrivateEndpoints != null - && customPrivateEndpoints.Count > 0) + if (accountInitializationCustomEndpoints != null + && accountInitializationCustomEndpoints.Count > 0) { - foreach (string customEndpoint in customPrivateEndpoints) + foreach (string customEndpoint in accountInitializationCustomEndpoints) { // Add all of the custom private endpoints to the service endpoints list. serviceEndpoints.Add( @@ -628,7 +628,7 @@ internal async Task GetDatabaseAccountAsync(bool forceRefresh singleValueInitFunc: () => GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( this.defaultEndpoint, this.connectionPolicy.PreferredLocations, - this.connectionPolicy.CustomPrivateEndpoints, + this.connectionPolicy.AccountInitializationCustomEndpoints, this.GetDatabaseAccountAsync, this.cancellationTokenSource.Token), cancellationToken: this.cancellationTokenSource.Token, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs index 646d3f7d35..91a6fd3f7c 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs @@ -141,7 +141,7 @@ public async Task InitializeReaderAsync_WhenCustomEndpointsProvided_ShouldRetryW if (customEndpointsProvided) { - connectionPolicy.SetCustomEndpoints( + connectionPolicy.SetAccountInitializationCustomEndpoints( new HashSet() { "https://testfed2.documents-test.windows-int.net:443/", @@ -192,7 +192,7 @@ public async Task InitializeReaderAsync_WhenRegionalEndpointsProvided_ShouldThro ConnectionMode = ConnectionMode.Direct, }; - connectionPolicy.SetCustomEndpoints( + connectionPolicy.SetAccountInitializationCustomEndpoints( new HashSet() { "https://testfed2.documents-test.windows-int.net:443/", diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs index c08ce53d07..616d72f812 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs @@ -118,7 +118,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, - customPrivateEndpoints: null, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => throw new Exception("The operation should be canceled and never make the network call."), cancellationTokenSource.Token); @@ -148,7 +148,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, - customPrivateEndpoints: null, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => { count++; @@ -179,7 +179,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, - customPrivateEndpoints: null, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: async (uri) => { count++; @@ -211,7 +211,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, - customPrivateEndpoints: null, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => { count++; @@ -243,7 +243,7 @@ await GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync( "southeastasia", "northcentralus" }, - customPrivateEndpoints: null, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => { count++; @@ -312,7 +312,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "southeastasia", "northcentralus" }, - customPrivateEndpoints: null, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); @@ -335,7 +335,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "southeastasia", "northcentralus" }, - customPrivateEndpoints: null, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); stopwatch.Stop(); @@ -359,7 +359,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "southeastasia", "northcentralus" }, - customPrivateEndpoints: null, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); @@ -381,7 +381,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "southeastasia", "northcentralus" }, - customPrivateEndpoints: null, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); @@ -407,7 +407,7 @@ public async Task GetDatabaseAccountFromAnyLocationsMockTestAsync() "westus6", "westus7", }, - customPrivateEndpoints: null, + accountInitializationCustomEndpoints: null, getDatabaseAccountFn: (uri) => slowPrimaryRegionHelper.RequestHelper(uri), cancellationToken: default); From f177371d23b1e76297a1a330eb5d29414145286c Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Mon, 5 Feb 2024 17:29:47 -0800 Subject: [PATCH 13/16] Code changes to address review comments. --- .../src/ConnectionPolicy.cs | 10 +- .../src/CosmosClientOptions.cs | 8 +- .../src/Fluent/CosmosClientBuilder.cs | 10 +- .../src/Routing/GlobalEndpointManager.cs | 167 ++++++++++-------- .../CosmosClientOptionsUnitTests.cs | 14 +- .../GatewayAccountReaderTests.cs | 16 +- 6 files changed, 131 insertions(+), 94 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs index ba781c01d0..b54c4837c4 100644 --- a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs @@ -29,7 +29,7 @@ internal sealed class ConnectionPolicy private Protocol connectionProtocol; private ObservableCollection preferredLocations; - private ObservableCollection accountInitializationCustomEndpoints; + private ObservableCollection accountInitializationCustomEndpoints; /// /// Initializes a new instance of the class to connect to the Azure Cosmos DB service. @@ -44,7 +44,7 @@ public ConnectionPolicy() this.MediaReadMode = MediaReadMode.Buffered; this.UserAgentContainer = new UserAgentContainer(clientId: 0); this.preferredLocations = new ObservableCollection(); - this.accountInitializationCustomEndpoints = new ObservableCollection(); + this.accountInitializationCustomEndpoints = new ObservableCollection(); this.EnableEndpointDiscovery = true; this.MaxConnectionLimit = defaultMaxConcurrentConnectionLimit; this.RetryOptions = new RetryOptions(); @@ -99,7 +99,7 @@ public void SetPreferredLocations(IReadOnlyList regions) /// An instance of containing the custom DNS endpoints /// provided by the customer. public void SetAccountInitializationCustomEndpoints( - IEnumerable customEndpoints) + IEnumerable customEndpoints) { if (customEndpoints == null) { @@ -107,7 +107,7 @@ public void SetAccountInitializationCustomEndpoints( } this.accountInitializationCustomEndpoints.Clear(); - foreach (string endpoint in customEndpoints) + foreach (Uri endpoint in customEndpoints) { this.accountInitializationCustomEndpoints.Add(endpoint); } @@ -303,7 +303,7 @@ public Collection PreferredLocations /// provided in the customAccountEndpoints list. /// /// - public Collection AccountInitializationCustomEndpoints + public Collection AccountInitializationCustomEndpoints { get { diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs index 1107bef805..507012dc3e 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs @@ -214,7 +214,11 @@ public string ApplicationName /// (){ "custom.p-1.documents.azure.com", "custom.p-2.documents.azure.com" } + /// AccountInitializationCustomEndpoints = new HashSet() + /// { + /// new Uri("custom.p-1.documents.azure.com"), + /// new Uri("custom.p-2.documents.azure.com") + /// } /// }; /// /// CosmosClient client = new CosmosClient("endpoint", "key", clientOptions); @@ -222,7 +226,7 @@ public string ApplicationName /// /// /// High availability on regional outages - public IEnumerable AccountInitializationCustomEndpoints { get; set; } + public IEnumerable AccountInitializationCustomEndpoints { get; set; } /// /// Get or set the maximum number of concurrent connections allowed for the target diff --git a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs index f761b82180..94500b1d09 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs @@ -314,7 +314,7 @@ public CosmosClientBuilder WithApplicationPreferredRegions(IReadOnlyList /// Should the global endpoint become inaccessible, the CosmosClient will attempt to obtain the account information issuing requests to the custom endpoints /// provided in the customAccountEndpoints list. /// - /// An instance of of string containing the custom private endpoints for the cosmos db account. + /// An instance of of Uri containing the custom private endpoints for the cosmos db account. /// /// This function is optional and is recommended for implementation when a customer has configured one or more endpoints with a custom DNS /// hostname (instead of accountname-region.documents.azure.com) etc. for their Cosmos DB account. @@ -326,14 +326,18 @@ public CosmosClientBuilder WithApplicationPreferredRegions(IReadOnlyList /// CosmosClientBuilder cosmosClientBuilder = new CosmosClientBuilder( /// accountEndpoint: "https://testcosmos.documents.azure.com:443/", /// authKeyOrResourceToken: "SuperSecretKey") - /// .WithCustomAccountEndpoints(new HashSet() { "https://region-1.documents-test.windows-int.net:443/", "https://region-2.documents-test.windows-int.net:443/" }); + /// .WithCustomAccountEndpoints(new HashSet() + /// { + /// new Uri("https://region-1.documents-test.windows-int.net:443/"), + /// new Uri("https://region-2.documents-test.windows-int.net:443/") + /// }); /// CosmosClient client = cosmosClientBuilder.Build(); /// ]]> /// /// /// The current . /// - public CosmosClientBuilder WithCustomAccountEndpoints(IEnumerable customAccountEndpoints) + public CosmosClientBuilder WithCustomAccountEndpoints(IEnumerable customAccountEndpoints) { this.clientOptions.AccountInitializationCustomEndpoints = customAccountEndpoints; return this; diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs index 83a5865a03..62b69d159c 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs @@ -117,7 +117,7 @@ public Uri GetHubUri() public static async Task GetDatabaseAccountFromAnyLocationsAsync( Uri defaultEndpoint, IList? locations, - IList accountInitializationCustomEndpoints, + IList? accountInitializationCustomEndpoints, Func> getDatabaseAccountFn, CancellationToken cancellationToken) { @@ -139,7 +139,7 @@ private class GetAccountPropertiesHelper : IDisposable { private readonly CancellationTokenSource CancellationTokenSource; private readonly Uri DefaultEndpoint; - private readonly IEnumerator? Locations; + private readonly bool ShouldVisitGlobalEndpointOnly; private readonly IEnumerator ServiceEndpointEnumerator; private readonly Func> GetDatabaseAccountFn; private readonly List TransientExceptions = new List(); @@ -150,50 +150,52 @@ private class GetAccountPropertiesHelper : IDisposable public GetAccountPropertiesHelper( Uri defaultEndpoint, IList? locations, - IList accountInitializationCustomEndpoints, + IList? accountInitializationCustomEndpoints, Func> getDatabaseAccountFn, CancellationToken cancellationToken) { this.DefaultEndpoint = defaultEndpoint; - this.Locations = locations?.GetEnumerator(); + this.ShouldVisitGlobalEndpointOnly = (locations == null || locations.Count == 0) && (accountInitializationCustomEndpoints == null || accountInitializationCustomEndpoints.Count == 0); this.GetDatabaseAccountFn = getDatabaseAccountFn; this.CancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - this.ServiceEndpointEnumerator = GetAccountPropertiesHelper.GetServiceEndpointEnumerator( - defaultEndpoint, - locations, - accountInitializationCustomEndpoints); + this.ServiceEndpointEnumerator = GetAccountPropertiesHelper + .GetServiceEndpoints( + defaultEndpoint, + locations, + accountInitializationCustomEndpoints) + .GetEnumerator(); } public async Task GetAccountPropertiesAsync() { - // If there are no preferred regions then just wait for the global endpoint results - if (this.Locations == null) + // If there are no preferred regions or private endpoints, then just wait for the global endpoint results + if (this.ShouldVisitGlobalEndpointOnly) { return await this.GetOnlyGlobalEndpointAsync(); } - Task globalEndpointTask = this.GetAndUpdateAccountPropertiesAsync(this.DefaultEndpoint); - - // Start a timer to start secondary requests in parallel. - Task timerTask = Task.Delay(TimeSpan.FromSeconds(5)); - await Task.WhenAny(globalEndpointTask, timerTask); - if (this.AccountProperties != null) - { - return this.AccountProperties; - } - - if (this.NonRetriableException != null) - { - ExceptionDispatchInfo.Capture(this.NonRetriableException).Throw(); - } - - // Start 2 additional tasks to try to get the account information - // from the preferred region list since global account has not succeed yet. - HashSet tasksToWaitOn = new HashSet - { - globalEndpointTask, - this.TryGetAccountPropertiesFromAllLocationsAsync(), - this.TryGetAccountPropertiesFromAllLocationsAsync() + Task globalEndpointTask = this.GetAndUpdateAccountPropertiesAsync(this.DefaultEndpoint); + + // Start a timer to start secondary requests in parallel. + Task timerTask = Task.Delay(TimeSpan.FromSeconds(5)); + await Task.WhenAny(globalEndpointTask, timerTask); + if (this.AccountProperties != null) + { + return this.AccountProperties; + } + + if (this.NonRetriableException != null) + { + ExceptionDispatchInfo.Capture(this.NonRetriableException).Throw(); + } + + // Start 2 additional tasks to try to get the account information + // from the preferred region list since global account has not succeed yet. + HashSet tasksToWaitOn = new HashSet + { + globalEndpointTask, + this.TryGetAccountPropertiesFromAllLocationsAsync(), + this.TryGetAccountPropertiesFromAllLocationsAsync() }; while (tasksToWaitOn.Any()) @@ -227,9 +229,9 @@ public async Task GetAccountPropertiesAsync() private async Task GetOnlyGlobalEndpointAsync() { - if (this.Locations != null) + if (!this.ShouldVisitGlobalEndpointOnly) { - throw new ArgumentException("GetOnlyGlobalEndpointAsync should only be called if there are no other regions"); + throw new ArgumentException("GetOnlyGlobalEndpointAsync should only be called if there are no other private endpoints or regions"); } await this.GetAndUpdateAccountPropertiesAsync(this.DefaultEndpoint); @@ -258,8 +260,7 @@ private async Task GetOnlyGlobalEndpointAsync() } /// - /// This is done in a thread safe way to allow multiple tasks to iterate over the - /// list of locations. + /// This is done in a thread safe way to allow multiple tasks to iterate over the list of service endpoints. /// private async Task TryGetAccountPropertiesFromAllLocationsAsync() { @@ -268,16 +269,49 @@ private async Task TryGetAccountPropertiesFromAllLocationsAsync() return; } - // We first iterate through all the private endpoints to fetch the account information. - // If all the attempt fails to fetch the metadata from the private endpoints, we will - // attempt to retrieve the account information from the regional endpoints constructed - // using the preferred regions list. - while (this.ServiceEndpointEnumerator.MoveNext()) - { + while (this.TryMoveNextServiceEndpointhreadSafe( + out Uri? serviceEndpoint)) + { + if (serviceEndpoint == null) + { + DefaultTrace.TraceCritical("GlobalEndpointManager: serviceEndpoint is null for TryMoveNextServiceEndpointhreadSafe."); + return; + } + await this.GetAndUpdateAccountPropertiesAsync( - endpoint: this.ServiceEndpointEnumerator.Current); + endpoint: serviceEndpoint); + } + } + + /// + /// We first iterate through all the private endpoints to fetch the account information. + /// If all the attempt fails to fetch the metadata from the private endpoints, we will + /// attempt to retrieve the account information from the regional endpoints constructed + /// using the preferred regions list. + /// + /// An instance of that will contain the service endpoint. + /// A boolean flag indicating if the was advanced in a thread safe manner. + private bool TryMoveNextServiceEndpointhreadSafe( + out Uri? serviceEndpoint) + { + if (this.CancellationTokenSource.IsCancellationRequested) + { + serviceEndpoint = null; + return false; } - } + + lock (this.ServiceEndpointEnumerator) + { + if (!this.ServiceEndpointEnumerator.MoveNext()) + { + serviceEndpoint = null; + return false; + } + + serviceEndpoint = this.ServiceEndpointEnumerator.Current; + return true; + } + } private async Task GetAndUpdateAccountPropertiesAsync(Uri endpoint) { @@ -287,7 +321,7 @@ private async Task GetAndUpdateAccountPropertiesAsync(Uri endpoint) { lock (this.TransientExceptions) { - this.TransientExceptions.Add(new OperationCanceledException("GlobalEndpointManager: Get account information canceled")); + this.TransientExceptions.Add(new OperationCanceledException($"GlobalEndpointManager: Get account information canceled for URI: {endpoint}")); } return; @@ -332,46 +366,37 @@ private static bool IsNonRetriableException(Exception exception) } /// - /// Returns an enumerator containing the private and regional service endpoints to iterate over. + /// Returns an instance of containing the private and regional service endpoints to iterate over. /// /// An instance of containing the default global endpoint. - /// An instance of containing the preferred location names. + /// An instance of containing the preferred serviceEndpoint names. /// An instance of containing the custom private endpoints. /// An instance of containing the service endpoints. - private static IEnumerator GetServiceEndpointEnumerator( + private static IEnumerable GetServiceEndpoints( Uri defaultEndpoint, IList? locations, - IList accountInitializationCustomEndpoints) + IList? accountInitializationCustomEndpoints) { - // We first iterate through all the private endpoints and add them to the service endpoints list. - IList serviceEndpoints = new List(); - - if (accountInitializationCustomEndpoints != null - && accountInitializationCustomEndpoints.Count > 0) + // We first iterate over all the private endpoints and yield return them. + if (accountInitializationCustomEndpoints?.Count > 0) { - foreach (string customEndpoint in accountInitializationCustomEndpoints) + foreach (Uri customEndpoint in accountInitializationCustomEndpoints) { - // Add all of the custom private endpoints to the service endpoints list. - serviceEndpoints.Add( - new Uri(customEndpoint)); + // Yield return all of the custom private endpoints first. + yield return customEndpoint; } } - // The next step is to construct and add all of the regional endpoints to the service endpoints list. - // The regional endpoints will be created by appending the preferred region name as a suffix to the - // default global endpoint. - if (locations != null - && locations.Count > 0) + // The next step is to iterate over the preferred locations, construct and yield return the regional endpoints one by one. + // The regional endpoints will be constructed by appending the preferred region name as a suffix to the default global endpoint. + if (locations?.Count > 0) { foreach (string location in locations) { - // Add all of the regional endpoints to the service endpoints list. - serviceEndpoints.Add( - LocationHelper.GetLocationEndpoint(defaultEndpoint, location)); + // Yield return all of the regional endpoints once the private custom endpoints are visited. + yield return LocationHelper.GetLocationEndpoint(defaultEndpoint, location); } } - - return serviceEndpoints.GetEnumerator(); } public void Dispose() @@ -415,7 +440,7 @@ public ReadOnlyDictionary GetAvailableReadEndpointsByLocation() } /// - /// Returns location corresponding to the endpoint + /// Returns serviceEndpoint corresponding to the endpoint /// /// public string GetLocation(Uri endpoint) @@ -555,7 +580,7 @@ private async void StartLocationBackgroundRefreshLoop() return; } - DefaultTrace.TraceCritical("GlobalEndpointManager: StartLocationBackgroundRefreshWithTimer() - Unable to refresh database account from any location. Exception: {0}", ex.ToString()); + DefaultTrace.TraceCritical("GlobalEndpointManager: StartLocationBackgroundRefreshWithTimer() - Unable to refresh database account from any serviceEndpoint. Exception: {0}", ex.ToString()); } // Call itself to create a loop to continuously do background refresh every 5 minutes @@ -574,7 +599,7 @@ private void OnPreferenceChanged(object sender, NotifyCollectionChangedEventArgs } /// - /// Thread safe refresh account and location info. + /// Thread safe refresh account and serviceEndpoint info. /// private async Task RefreshDatabaseAccountInternalAsync(bool forceRefresh) { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs index 2b601db72d..0e04023363 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs @@ -159,7 +159,11 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.IsTrue(clientOptions.EnableAdvancedReplicaSelectionForTcp.Value); IReadOnlyList preferredLocations = new List() { Regions.AustraliaCentral, Regions.AustraliaCentral2 }; - ISet regionalEndpoints = new HashSet() { "https://testfed2.documents-test.windows-int.net:443/", "https://testfed4.documents-test.windows-int.net:443/" }; + ISet regionalEndpoints = new HashSet() + { + new Uri("https://testfed2.documents-test.windows-int.net:443/"), + new Uri("https://testfed4.documents-test.windows-int.net:443/") + }; //Verify Direct Mode settings cosmosClientBuilder = new CosmosClientBuilder( @@ -328,11 +332,11 @@ public void CosmosClientOptions_WhenPartitionLevelFailoverEnabledAndPreferredReg Regions.EastAsia, }) .WithCustomAccountEndpoints( - new HashSet() + new HashSet() { - "https://testfed2.documents-test.windows-int.net:443/", - "https://testfed3.documents-test.windows-int.net:443/", - "https://testfed4.documents-test.windows-int.net:443/", + new Uri("https://testfed2.documents-test.windows-int.net:443/"), + new Uri("https://testfed3.documents-test.windows-int.net:443/"), + new Uri("https://testfed4.documents-test.windows-int.net:443/"), }); CosmosClientOptions clientOptions = cosmosClientBuilder.Build().ClientOptions; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs index 91a6fd3f7c..3ef8e6c996 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs @@ -142,11 +142,11 @@ public async Task InitializeReaderAsync_WhenCustomEndpointsProvided_ShouldRetryW if (customEndpointsProvided) { connectionPolicy.SetAccountInitializationCustomEndpoints( - new HashSet() + new HashSet() { - "https://testfed2.documents-test.windows-int.net:443/", - "https://testfed3.documents-test.windows-int.net:443/", - "https://testfed4.documents-test.windows-int.net:443/", + new Uri("https://testfed2.documents-test.windows-int.net:443/"), + new Uri("https://testfed3.documents-test.windows-int.net:443/"), + new Uri("https://testfed4.documents-test.windows-int.net:443/"), }); } @@ -193,11 +193,11 @@ public async Task InitializeReaderAsync_WhenRegionalEndpointsProvided_ShouldThro }; connectionPolicy.SetAccountInitializationCustomEndpoints( - new HashSet() + new HashSet() { - "https://testfed2.documents-test.windows-int.net:443/", - "https://testfed3.documents-test.windows-int.net:443/", - "https://testfed4.documents-test.windows-int.net:443/", + new Uri("https://testfed2.documents-test.windows-int.net:443/"), + new Uri("https://testfed3.documents-test.windows-int.net:443/"), + new Uri("https://testfed4.documents-test.windows-int.net:443/"), }); GatewayAccountReader accountReader = new GatewayAccountReader( From d4c3526421ae13ffd1210ddeef4d500b84e1c804 Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Mon, 5 Feb 2024 17:45:37 -0800 Subject: [PATCH 14/16] Code changes to update the API contract. --- .../Contracts/DotNetSDKAPI.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json index 40735c888c..d7335051ae 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json @@ -2770,17 +2770,17 @@ ], "MethodInfo": "Microsoft.Azure.Cosmos.CosmosSerializer Serializer;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.CosmosSerializer get_Serializer();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_Serializer(Microsoft.Azure.Cosmos.CosmosSerializer);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Collections.Generic.IEnumerable`1[System.String] AccountInitializationCustomEndpoints": { + "System.Collections.Generic.IEnumerable`1[System.Uri] AccountInitializationCustomEndpoints": { "Type": "Property", "Attributes": [], - "MethodInfo": "System.Collections.Generic.IEnumerable`1[System.String] AccountInitializationCustomEndpoints;CanRead:True;CanWrite:True;System.Collections.Generic.IEnumerable`1[System.String] get_AccountInitializationCustomEndpoints();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_AccountInitializationCustomEndpoints(System.Collections.Generic.IEnumerable`1[System.String]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "System.Collections.Generic.IEnumerable`1[System.Uri] AccountInitializationCustomEndpoints;CanRead:True;CanWrite:True;System.Collections.Generic.IEnumerable`1[System.Uri] get_AccountInitializationCustomEndpoints();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_AccountInitializationCustomEndpoints(System.Collections.Generic.IEnumerable`1[System.Uri]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Collections.Generic.IEnumerable`1[System.String] get_AccountInitializationCustomEndpoints()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "System.Collections.Generic.IEnumerable`1[System.Uri] get_AccountInitializationCustomEndpoints()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ "CompilerGeneratedAttribute" ], - "MethodInfo": "System.Collections.Generic.IEnumerable`1[System.String] get_AccountInitializationCustomEndpoints();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "System.Collections.Generic.IEnumerable`1[System.Uri] get_AccountInitializationCustomEndpoints();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "System.Collections.Generic.IReadOnlyList`1[System.String] ApplicationPreferredRegions": { "Type": "Property", @@ -3005,12 +3005,12 @@ "Attributes": [], "MethodInfo": "[Void .ctor(), Void .ctor()]" }, - "Void set_AccountInitializationCustomEndpoints(System.Collections.Generic.IEnumerable`1[System.String])[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Void set_AccountInitializationCustomEndpoints(System.Collections.Generic.IEnumerable`1[System.Uri])[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ "CompilerGeneratedAttribute" ], - "MethodInfo": "Void set_AccountInitializationCustomEndpoints(System.Collections.Generic.IEnumerable`1[System.String]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "Void set_AccountInitializationCustomEndpoints(System.Collections.Generic.IEnumerable`1[System.Uri]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "Void set_AllowBulkExecution(Boolean)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", @@ -4622,10 +4622,10 @@ "Attributes": [], "MethodInfo": "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithContentResponseOnWrite(Boolean);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithCustomAccountEndpoints(System.Collections.Generic.IEnumerable`1[System.String])": { + "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithCustomAccountEndpoints(System.Collections.Generic.IEnumerable`1[System.Uri])": { "Type": "Method", "Attributes": [], - "MethodInfo": "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithCustomAccountEndpoints(System.Collections.Generic.IEnumerable`1[System.String]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithCustomAccountEndpoints(System.Collections.Generic.IEnumerable`1[System.Uri]);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "Microsoft.Azure.Cosmos.Fluent.CosmosClientBuilder WithCustomSerializer(Microsoft.Azure.Cosmos.CosmosSerializer)": { "Type": "Method", From 65b9b4491d184f8b3eb3bc5f8c9c5f9040f5a2c6 Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Tue, 6 Feb 2024 14:13:31 -0800 Subject: [PATCH 15/16] Cosmetic code changes. --- .../src/Routing/GlobalEndpointManager.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs index 82cdd019ea..e200134997 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs @@ -139,7 +139,7 @@ private class GetAccountPropertiesHelper : IDisposable { private readonly CancellationTokenSource CancellationTokenSource; private readonly Uri DefaultEndpoint; - private readonly bool ShouldVisitGlobalEndpointOnly; + private readonly bool LimitToGlobalEndpointOnly; private readonly IEnumerator ServiceEndpointEnumerator; private readonly Func> GetDatabaseAccountFn; private readonly List TransientExceptions = new List(); @@ -155,7 +155,7 @@ public GetAccountPropertiesHelper( CancellationToken cancellationToken) { this.DefaultEndpoint = defaultEndpoint; - this.ShouldVisitGlobalEndpointOnly = (locations == null || locations.Count == 0) && (accountInitializationCustomEndpoints == null || accountInitializationCustomEndpoints.Count == 0); + this.LimitToGlobalEndpointOnly = (locations == null || locations.Count == 0) && (accountInitializationCustomEndpoints == null || accountInitializationCustomEndpoints.Count == 0); this.GetDatabaseAccountFn = getDatabaseAccountFn; this.CancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); this.ServiceEndpointEnumerator = GetAccountPropertiesHelper @@ -169,7 +169,7 @@ public GetAccountPropertiesHelper( public async Task GetAccountPropertiesAsync() { // If there are no preferred regions or private endpoints, then just wait for the global endpoint results - if (this.ShouldVisitGlobalEndpointOnly) + if (this.LimitToGlobalEndpointOnly) { return await this.GetOnlyGlobalEndpointAsync(); } @@ -229,7 +229,7 @@ public async Task GetAccountPropertiesAsync() private async Task GetOnlyGlobalEndpointAsync() { - if (!this.ShouldVisitGlobalEndpointOnly) + if (!this.LimitToGlobalEndpointOnly) { throw new ArgumentException("GetOnlyGlobalEndpointAsync should only be called if there are no other private endpoints or regions"); } From 3a9d1681aaa84698454941209c1a584050d2aeec Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Thu, 8 Feb 2024 14:19:28 -0800 Subject: [PATCH 16/16] Code changes to address review comments. --- .../src/Routing/GlobalEndpointManager.cs | 5 -- .../CosmosClientOptionsUnitTests.cs | 1 + .../GatewayAccountReaderTests.cs | 65 +++++++++++++++---- 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs index e200134997..2d60e57adb 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs @@ -264,11 +264,6 @@ private async Task GetOnlyGlobalEndpointAsync() /// private async Task TryGetAccountPropertiesFromAllLocationsAsync() { - if (this.CancellationTokenSource.IsCancellationRequested) - { - return; - } - while (this.TryMoveNextServiceEndpointhreadSafe( out Uri? serviceEndpoint)) { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs index 309368bd94..293c3b14ab 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs @@ -216,6 +216,7 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated() Assert.AreEqual(portReuseMode, policy.PortReuseMode); Assert.IsTrue(policy.EnableTcpConnectionEndpointRediscovery); CollectionAssert.AreEqual(preferredLocations.ToArray(), policy.PreferredLocations.ToArray()); + CollectionAssert.AreEqual(regionalEndpoints.ToArray(), policy.AccountInitializationCustomEndpoints.ToArray()); } /// diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs index 3ef8e6c996..41e447ac35 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs @@ -113,7 +113,13 @@ public async Task InitializeReaderAsync_WhenCustomEndpointsProvided_ShouldRetryW "\"defaultConsistencyLevel\": \"Session\"\r\n },\r\n \"systemReplicationPolicy\": {\r\n \"minReplicaSetSize\": 1,\r\n \"maxReplicasetSize\": 4\r\n },\r\n \"readPolicy\": {\r\n \"primaryReadCoefficient\": 1,\r\n \"secondaryReadCoefficient\": 1\r\n },\r\n \"queryEngineConfiguration\": \"{\\\"maxSqlQueryInputLength\\\":262144,\\\"maxJoinsPerSqlQuery\\\":5," + "\\\"maxLogicalAndPerSqlQuery\\\":500,\\\"maxLogicalOrPerSqlQuery\\\":500,\\\"maxUdfRefPerSqlQuery\\\":10,\\\"maxInExpressionItemsCount\\\":16000,\\\"queryMaxInMemorySortDocumentCount\\\":500,\\\"maxQueryRequestTimeoutFraction\\\":0.9,\\\"sqlAllowNonFiniteNumbers\\\":false,\\\"sqlAllowAggregateFunctions\\\":true,\\\"sqlAllowSubQuery\\\":true,\\\"sqlAllowScalarSubQuery\\\":true,\\\"allowNewKeywords\\\":true,\\\"" + "sqlAllowLike\\\":true,\\\"sqlAllowGroupByClause\\\":true,\\\"maxSpatialQueryCells\\\":12,\\\"spatialMaxGeometryPointCount\\\":256,\\\"sqlDisableOptimizationFlags\\\":0,\\\"sqlAllowTop\\\":true,\\\"enableSpatialIndexing\\\":true}\"\r\n}"; - + + Uri globalEndpoint = new("https://testfed1.documents-test.windows-int.net:443/"); + Uri privateEndpoint1 = new ("https://testfed2.documents-test.windows-int.net:443/"); + Uri privateEndpoint2 = new ("https://testfed3.documents-test.windows-int.net:443/"); + Uri privateEndpoint3 = new ("https://testfed4.documents-test.windows-int.net:443/"); + Uri endpointSucceeded = default; + StringContent content = new(accountPropertiesResponse); HttpResponseMessage responseMessage = new() { @@ -122,16 +128,31 @@ public async Task InitializeReaderAsync_WhenCustomEndpointsProvided_ShouldRetryW }; Mock mockHttpClient = new(); + + GatewayAccountReaderTests.SetupMockToThrowException( + mockHttpClient: mockHttpClient, + endpoints: new List() + { + globalEndpoint, + privateEndpoint1, + privateEndpoint2, + }); + mockHttpClient - .SetupSequence(x => x.GetAsync( - It.IsAny(), + .Setup(x => x.GetAsync( + privateEndpoint3, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ThrowsAsync(new Exception("Service is Unavailable at the Moment.")) - .ThrowsAsync(new Exception("Service is Unavailable at the Moment.")) + .Callback(( + Uri serviceEndpoint, + INameValueCollection _, + ResourceType _, + HttpTimeoutPolicy _, + IClientSideRequestStatistics _, + CancellationToken _) => endpointSucceeded = serviceEndpoint) .ReturnsAsync(responseMessage); ConnectionPolicy connectionPolicy = new() @@ -144,14 +165,14 @@ public async Task InitializeReaderAsync_WhenCustomEndpointsProvided_ShouldRetryW connectionPolicy.SetAccountInitializationCustomEndpoints( new HashSet() { - new Uri("https://testfed2.documents-test.windows-int.net:443/"), - new Uri("https://testfed3.documents-test.windows-int.net:443/"), - new Uri("https://testfed4.documents-test.windows-int.net:443/"), + privateEndpoint1, + privateEndpoint2, + privateEndpoint3, }); } GatewayAccountReader accountReader = new GatewayAccountReader( - serviceEndpoint: new Uri("https://testfed1.documents-test.windows-int.net:443/"), + serviceEndpoint: globalEndpoint, cosmosAuthorization: Mock.Of(), connectionPolicy: connectionPolicy, httpClient: mockHttpClient.Object); @@ -163,10 +184,12 @@ public async Task InitializeReaderAsync_WhenCustomEndpointsProvided_ShouldRetryW Assert.IsNotNull(accountProperties); Assert.AreEqual("localhost", accountProperties.Id); Assert.AreEqual("127.0.0.1", accountProperties.ResourceId); + Assert.AreEqual(endpointSucceeded, privateEndpoint3); } else { Exception exception = await Assert.ThrowsExceptionAsync(() => accountReader.InitializeReaderAsync()); + Assert.IsNull(endpointSucceeded); Assert.IsNotNull(exception); Assert.AreEqual("Service is Unavailable at the Moment.", exception.Message); } @@ -195,9 +218,9 @@ public async Task InitializeReaderAsync_WhenRegionalEndpointsProvided_ShouldThro connectionPolicy.SetAccountInitializationCustomEndpoints( new HashSet() { - new Uri("https://testfed2.documents-test.windows-int.net:443/"), - new Uri("https://testfed3.documents-test.windows-int.net:443/"), - new Uri("https://testfed4.documents-test.windows-int.net:443/"), + new ("https://testfed2.documents-test.windows-int.net:443/"), + new ("https://testfed3.documents-test.windows-int.net:443/"), + new ("https://testfed4.documents-test.windows-int.net:443/"), }); GatewayAccountReader accountReader = new GatewayAccountReader( @@ -211,6 +234,24 @@ public async Task InitializeReaderAsync_WhenRegionalEndpointsProvided_ShouldThro Assert.AreEqual("Service is Unavailable at the Moment.", exception.InnerException.Message); } + private static void SetupMockToThrowException( + Mock mockHttpClient, + IList endpoints) + { + foreach(Uri endpoint in endpoints) + { + mockHttpClient + .Setup(x => x.GetAsync( + endpoint, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ThrowsAsync(new Exception("Service is Unavailable at the Moment.")); + } + } + public class CustomMessageHandler : HttpMessageHandler { protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)