Connection limits: Fixes .NET core to honor gateway connection limit

Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs

@@ -11,6 +11,7 @@ namespace Microsoft.Azure.Cosmos
     using System.Linq;
     using System.Net;
     using System.Net.Http;
+    using System.Security.AccessControl;
     using Microsoft.Azure.Cosmos.Fluent;
     using Microsoft.Azure.Documents;
     using Microsoft.Azure.Documents.Client;
@@ -136,9 +137,9 @@ public int GatewayModeMaxConnectionLimit
                     throw new ArgumentOutOfRangeException(nameof(value));
                 }
 
-                if (this.ConnectionMode != ConnectionMode.Gateway)
+                if (this.HttpClientFactory != null && value != ConnectionPolicy.Default.MaxConnectionLimit )
                 {
-                    throw new ArgumentException("Max connection limit is only valid for ConnectionMode.Gateway.");
+                    throw new ArgumentException($"{nameof(this.httpClientFactory)} can not be set along with {nameof(this.GatewayModeMaxConnectionLimit)}. This must be set on the HttpClientHandler.MaxConnectionsPerServer property.");
                 }
 
                 this.gatewayModeMaxConnectionLimit = value;
@@ -468,6 +469,9 @@ public CosmosSerializer Serializer
         /// <para>
         /// Useful in scenarios where the application is using a pool of HttpClient instances to be shared, like ASP.NET Core applications with IHttpClientFactory or Blazor WebAssembly applications.
         /// </para>
+        /// <para>
+        /// For .NET core applications the default GatewayConnectionLimit will be ignored. It must be set on the HttpClientHandler.MaxConnectionsPerServer to limit the number of connections
+        /// </para>
         /// </remarks>
         [JsonIgnore]
         public Func<HttpClient> HttpClientFactory
@@ -480,6 +484,11 @@ public Func<HttpClient> HttpClientFactory
                     throw new ArgumentException($"{nameof(this.HttpClientFactory)} cannot be set along {nameof(this.WebProxy)}");
                 }
 
+                if (this.GatewayModeMaxConnectionLimit != ConnectionPolicy.Default.MaxConnectionLimit)
+                {
+                    throw new ArgumentException($"{nameof(this.httpClientFactory)} can not be set along with {nameof(this.GatewayModeMaxConnectionLimit)}. This must be set on the HttpClientHandler.MaxConnectionsPerServer property.");
+                }
+
                 this.httpClientFactory = value;
             }
         }
@@ -623,7 +632,7 @@ internal ConnectionPolicy GetConnectionPolicy()
                 EnableEndpointDiscovery = !this.LimitToEndpoint,
                 PortReuseMode = this.portReuseMode,
                 EnableTcpConnectionEndpointRediscovery = this.EnableTcpConnectionEndpointRediscovery,
-                HttpClientFactory = this.httpClientFactory
+                HttpClientFactory = this.httpClientFactory,
             };
 
             if (this.ApplicationRegion != null)

Microsoft.Azure.Cosmos/src/Resource/ClientContextCore.cs

@@ -464,14 +464,16 @@ private bool IsBulkOperationSupported(
 
         private static HttpClientHandler CreateHttpClientHandler(CosmosClientOptions clientOptions)
         {
-            if (clientOptions == null || clientOptions.WebProxy == null)
+            if (clientOptions == null)
             {
-                return null;
+                throw new ArgumentNullException(nameof(clientOptions));
             }
-
+
+            // https://docs.microsoft.com/en-us/archive/blogs/timomta/controlling-the-number-of-outgoing-connections-from-httpclient-net-core-or-full-framework
             HttpClientHandler httpClientHandler = new HttpClientHandler
             {
-                Proxy = clientOptions.WebProxy
+                Proxy = clientOptions.WebProxy,
+                MaxConnectionsPerServer = clientOptions.GatewayModeMaxConnectionLimit
             };
 
             return httpClientHandler;

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ClientTests.cs

@@ -5,9 +5,14 @@
 namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
 {
     using System;
+    using System.Collections.Generic;
+    using System.Diagnostics;
     using System.IO;
+    using System.Linq;
     using System.Net;
     using System.Net.Http;
+    using System.Net.NetworkInformation;
+    using System.Reflection;
     using System.Threading.Tasks;
     using Microsoft.Azure.Cosmos.Query.Core;
     using Microsoft.Azure.Cosmos.Services.Management.Tests.LinqProviderTests;
@@ -30,13 +35,13 @@ public async Task ResourceResponseStreamingTest()
 
                 Database db = (await client.CreateDatabaseAsync(new Database() { Id = Guid.NewGuid().ToString() })).Resource;
                 DocumentCollection coll = await TestCommon.CreateCollectionAsync(client, db, new DocumentCollection()
-                        {
-                            Id = Guid.NewGuid().ToString(),
-                            PartitionKey = new PartitionKeyDefinition()
-                            {
-                                Paths = new System.Collections.ObjectModel.Collection<string>() { "/id" }
-                            }
-                        });
+                {
+                    Id = Guid.NewGuid().ToString(),
+                    PartitionKey = new PartitionKeyDefinition()
+                    {
+                        Paths = new System.Collections.ObjectModel.Collection<string>() { "/id" }
+                    }
+                });
                 ResourceResponse<Document> doc = await client.CreateDocumentAsync(coll.SelfLink, new Document() { Id = Guid.NewGuid().ToString() });
 
                 Assert.AreEqual(doc.ResponseStream.Position, 0);
@@ -281,7 +286,8 @@ public async Task VerifyNegativeWebProxySettings()
                 }
             );
 
-            await Assert.ThrowsExceptionAsync<HttpRequestException>(async () => {
+            await Assert.ThrowsExceptionAsync<HttpRequestException>(async () =>
+            {
                 DatabaseResponse databaseResponse = await cosmosClient.CreateDatabaseAsync(Guid.NewGuid().ToString());
             });
 
@@ -298,7 +304,8 @@ await Assert.ThrowsExceptionAsync<HttpRequestException>(async () => {
                 }
             );
 
-            await Assert.ThrowsExceptionAsync<HttpRequestException>(async () => {
+            await Assert.ThrowsExceptionAsync<HttpRequestException>(async () =>
+            {
                 DatabaseResponse databaseResponse = await cosmosClient.CreateDatabaseAsync(Guid.NewGuid().ToString());
             });
         }
@@ -335,12 +342,97 @@ public async Task HttpClientFactorySmokeTest()
             }
             finally
             {
-                if (database!= null)
+                if (database != null)
                 {
                     await database.DeleteAsync();
                 }
             }
         }
+
+        [TestMethod]
+        public async Task HttpClientConnectionLimitTest()
+        {
+            int gatewayConnectionLimit = 1;
+
+            IReadOnlyList<string> excludeConnections = GetActiveConnections();
+            CosmosClient cosmosClient = new CosmosClient(
+                ConfigurationManager.AppSettings["GatewayEndpoint"],
+                ConfigurationManager.AppSettings["MasterKey"],
+                new CosmosClientOptions
+                {
+                    ApplicationName = "test",
+                    GatewayModeMaxConnectionLimit = gatewayConnectionLimit,
+                    ConnectionMode = ConnectionMode.Gateway,
+                    ConnectionProtocol = Protocol.Https
+                }
+            );
+
+            DelegatingHandler handler = (DelegatingHandler)cosmosClient.DocumentClient.httpMessageHandler;
+            HttpClientHandler httpClientHandler = (HttpClientHandler)handler.InnerHandler;
+            Assert.AreEqual(gatewayConnectionLimit, httpClientHandler.MaxConnectionsPerServer);
+
+            Cosmos.Database database = await cosmosClient.CreateDatabaseAsync(Guid.NewGuid().ToString());
+            Container container = await database.CreateContainerAsync(
+                "TestConnections",
+                "/pk",
+                throughput: 20000);
+
+            List<Task> creates = new List<Task>();
+            for (int i = 0; i < 100; i++)
+            {
+                creates.Add(container.CreateItemAsync<dynamic>(new { id = Guid.NewGuid().ToString(), pk = Guid.NewGuid().ToString() }));
+            }
+
+            await Task.WhenAll(creates);
+
+            // Verify the handler still exists after client warm up
+            //Assert.AreEqual(gatewayConnectionLimit, httpClientHandler.MaxConnectionsPerServer);
+            IReadOnlyList<string> afterConnections = GetActiveConnections();
+
+            // Clean up the database and container
+            await database.DeleteAsync();
+
+            int connectionDiff = afterConnections.Count - excludeConnections.Count;
+            Assert.IsTrue(connectionDiff <= gatewayConnectionLimit, $"Connection before : {excludeConnections.Count}, after {afterConnections.Count}");
+        }
+
+        public static IReadOnlyList<string> GetActiveConnections()
+        {
+            string testPid = Process.GetCurrentProcess().Id.ToString();
+            using (Process p = new Process())
+            {
+                ProcessStartInfo ps = new ProcessStartInfo();
+                ps.Arguments = "-a -n -o";
+                ps.FileName = "netstat.exe";
+                ps.UseShellExecute = false;
+                ps.WindowStyle = ProcessWindowStyle.Hidden;
+                ps.RedirectStandardInput = true;
+                ps.RedirectStandardOutput = true;
+                ps.RedirectStandardError = true;
+
+                p.StartInfo = ps;
+                p.Start();
+
+                StreamReader stdOutput = p.StandardOutput;
+                StreamReader stdError = p.StandardError;
+
+                string content = stdOutput.ReadToEnd() + stdError.ReadToEnd();
+                string exitStatus = p.ExitCode.ToString();
+
+                if (exitStatus != "0")
+                {
+                    // Command Errored. Handle Here If Need Be
+                }
+
+                List<string> connections = content.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries)
+                    .Select(x => x.Trim())
+                    .Where(x => x.EndsWith(testPid)).ToList();
+
+                Assert.IsTrue(connections.Count > 0);
+
+                return connections;
+            }
+        }
     }
 
     internal static class StringHelper

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs

@@ -300,6 +300,30 @@ public void VerifyHttpClientHandlerSettingsThrowIfNotUsedInGatewayMode()
             Assert.ThrowsException<ArgumentException>(() => { cosmosClientOptions.WebProxy = new TestWebProxy(); });
         }
 
+        [TestMethod]
+        public void VerifyHttpClientFactoryBlockedWithConnectionLimit()
+        {
+            CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()
+            {
+                GatewayModeMaxConnectionLimit = 42
+            };
+
+            Assert.ThrowsException<ArgumentException>(() =>
+            {
+                cosmosClientOptions.HttpClientFactory = () => new HttpClient();
+            });
+
+            cosmosClientOptions = new CosmosClientOptions()
+            {
+                HttpClientFactory = () => new HttpClient()
+            };
+
+            Assert.ThrowsException<ArgumentException>(() =>
+            {
+                cosmosClientOptions.GatewayModeMaxConnectionLimit = 42;
+            });
+        }
+
         [TestMethod]
         public void VerifyHttpClientHandlerIsSet()
         {