Implement automatic rate-limit handling (#512)

* Implement automatic rate-limit handling

* Move random instance to class level

* Use ConcurrentRandom

* Try fixing CI

src/Auth0.Core/ConcurrentRandom.cs

@@ -0,0 +1,23 @@
+using System;
+
+namespace Auth0.Core
+{
+    internal sealed class ConcurrentRandom
+    {
+        private static readonly Random SRandom = new Random();
+
+        /// <summary>
+        /// Returns a random integer that is within a specified range.
+        /// </summary>
+        /// <param name="minValue">The inclusive lower bound of the random number returned.</param>
+        /// <param name="maxValue">The exclusive upper bound of the random number returned. <paramref name="maxValue" /> must be greater than or equal to <paramref name="minValue" />.</param>
+        /// <returns>A 32-bit signed integer greater than or equal to <paramref name="minValue" /> and less than <paramref name="maxValue" />; that is, the range of return values includes <paramref name="minValue" /> but not <paramref name="maxValue" />. If <paramref name="minValue" /> equals <paramref name="maxValue" />, <paramref name="minValue" /> is returned.</returns>
+        public int Next(int minValue, int maxValue)
+        {
+            lock (SRandom)
+            {
+                return SRandom.Next(minValue, maxValue);
+            }
+        }
+    }
+}

src/Auth0.ManagementApi/HttpClientManagementConnection.cs

@@ -6,7 +6,9 @@
 using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
+using Auth0.Core;
 
 namespace Auth0.ManagementApi
 {
@@ -18,17 +20,27 @@ public class HttpClientManagementConnection : IManagementConnection, IDisposable
         static readonly JsonSerializerSettings jsonSerializerSettings = new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore, DateParseHandling = DateParseHandling.DateTime };
 
         readonly HttpClient httpClient;
+        readonly HttpClientManagementConnectionOptions options;
         bool ownHttpClient;
 
+        readonly ConcurrentRandom random = new ConcurrentRandom();
+        readonly int MAX_REQUEST_RETRY_JITTER = 250;
+        readonly int MAX_REQUEST_RETRY_DELAY = 500;
+        readonly int MIN_REQUEST_RETRY_DELAY = 100;
+        readonly int DEFAULT_NUMBER_RETRIES = 3;
+        readonly int MAX_NUMBER_RETRIES = 10;
+        readonly int BASE_DELAY = 100;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="HttpClientManagementConnection"/> class.
         /// </summary>
         /// <param name="httpClient">Optional <see cref="HttpClient"/> to use. If not specified one will
         /// be created and be used for all requests made by this instance.</param>
-        public HttpClientManagementConnection(HttpClient httpClient = null)
+        public HttpClientManagementConnection(HttpClient httpClient = null, HttpClientManagementConnectionOptions options = null)
         {
             ownHttpClient = httpClient == null;
             this.httpClient = httpClient ?? new HttpClient();
+            this.options = options ?? new HttpClientManagementConnectionOptions();
         }
 
         /// <summary>
@@ -37,14 +49,25 @@ public HttpClientManagementConnection(HttpClient httpClient = null)
         /// <param name="handler"><see cref="HttpMessageHandler"/> to use with the managed 
         /// <see cref="HttpClient"/> that will be created and used for all requests made
         /// by this instance.</param>
-        public HttpClientManagementConnection(HttpMessageHandler handler)
-            : this(new HttpClient(handler ?? new HttpClientHandler()))
+        public HttpClientManagementConnection(HttpMessageHandler handler, HttpClientManagementConnectionOptions options = null)
+            : this(new HttpClient(handler ?? new HttpClientHandler()), options)
         {
             ownHttpClient = true;
         }
 
         /// <inheritdoc />
         public async Task<T> GetAsync<T>(Uri uri, IDictionary<string, string> headers, JsonConverter[] converters = null)
+        {
+            return await Retry(async () => await GetAsyncInternal<T>(uri, headers, converters)).ConfigureAwait(false);
+        }
+
+        /// <inheritdoc />
+        public async Task<T> SendAsync<T>(HttpMethod method, Uri uri, object body, IDictionary<string, string> headers, IList<FileUploadParameter> files = null)
+        {
+            return await Retry(async () => await SendAsyncInternal<T>(method, uri, body, headers, files)).ConfigureAwait(false);
+        }
+
+        private async Task<T> GetAsyncInternal<T>(Uri uri, IDictionary<string, string> headers, JsonConverter[] converters = null)
         {
             using (var request = new HttpRequestMessage(HttpMethod.Get, uri))
             {
@@ -53,8 +76,7 @@ public async Task<T> GetAsync<T>(Uri uri, IDictionary<string, string> headers, J
             }
         }
 
-        /// <inheritdoc />
-        public async Task<T> SendAsync<T>(HttpMethod method, Uri uri, object body, IDictionary<string, string> headers, IList<FileUploadParameter> files = null)
+        private async Task<T> SendAsyncInternal<T>(HttpMethod method, Uri uri, object body, IDictionary<string, string> headers, IList<FileUploadParameter> files = null)
         {
             using (var request = new HttpRequestMessage(method, uri) { Content = BuildMessageContent(body, files) })
             {
@@ -166,5 +188,44 @@ private static string SerializeFormBodyValue(object value)
             if (value is bool boolean) return boolean ? "true" : "false";
             return Uri.EscapeDataString(value.ToString());
         }
+
+        private async Task<TResult> Retry<TResult>(Func<Task<TResult>> retryable)
+        {
+            int? configuredNrOfTries = options.NumberOfHttpRetries;
+            var nrOfTries = 0;
+            var nrOfTriesToAttempt = Math.Min(MAX_NUMBER_RETRIES, configuredNrOfTries ?? DEFAULT_NUMBER_RETRIES);
+
+            while (true)
+            {
+                try
+                {
+                    nrOfTries++;
+
+                    return await retryable();
+                }
+                catch (Exception ex)
+                {
+                    if (!(ex is RateLimitApiException) || nrOfTries >= nrOfTriesToAttempt)
+                    {
+                        throw;
+                    }
+                }
+
+                // Use an exponential back-off with the formula:
+                // max(MIN_REQUEST_RETRY_DELAY, min(MAX_REQUEST_RETRY_DELAY, (BASE_DELAY * (2 ** attempt - 1)) + random_between(0, MAX_REQUEST_RETRY_JITTER)))
+                //
+                // ✔ Each attempt increases base delay by (100ms * (2 ** attempt - 1))
+                // ✔ Randomizes jitter, adding up to MAX_REQUEST_RETRY_JITTER (250ms)
+                // ✔ Never less than MIN_REQUEST_RETRY_DELAY (100ms)
+                // ✔ Never more than MAX_REQUEST_RETRY_DELAY (500ms)
+
+                var wait = Convert.ToInt32(BASE_DELAY * Math.Pow(2, nrOfTries - 1));
+                wait = random.Next(wait + 1, wait + MAX_REQUEST_RETRY_JITTER);
+                wait = Math.Min(wait, MAX_REQUEST_RETRY_DELAY);
+                wait = Math.Max(wait, MIN_REQUEST_RETRY_DELAY);
+
+                await Task.Delay(wait);
+            }
+        }
     }
 }

src/Auth0.ManagementApi/HttpClientManagementConnectionOptions.cs

@@ -0,0 +1,13 @@
+namespace Auth0.ManagementApi
+{
+    public class HttpClientManagementConnectionOptions
+    {
+        /// <summary>
+        ///  Set the maximum number of consecutive retries for Management API requests that fail due to rate-limits being reached.
+        ///  By default, rate-limited requests will be retries a maximum of three times.To disable retries on rate-limit
+        ///  errors, set this value to zero.
+        /// </summary>
+        /// <remarks>Must be a number between zero (do not retry) and ten.</remarks>
+        public int? NumberOfHttpRetries { get; set; }
+    }
+}

tests/Auth0.AuthenticationApi.IntegrationTests/AccessTokenTests.cs

@@ -21,7 +21,7 @@ public async Task InitializeAsync()
         {
             string token = await GenerateManagementApiToken();
 
-            _managementApiClient = new TestManagementApiClient(token, GetVariable("AUTH0_MANAGEMENT_API_URL"));
+            _managementApiClient = new ManagementApiClient(token, GetVariable("AUTH0_MANAGEMENT_API_URL"));
 
             // We will need a connection to add the users to...
             _connection = await _managementApiClient.Connections.CreateAsync(new ConnectionCreateRequest

tests/Auth0.AuthenticationApi.IntegrationTests/Auth0.AuthenticationApi.IntegrationTests.csproj

@@ -34,4 +34,7 @@
   <ItemGroup>
     <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
   </ItemGroup>
+  <ItemGroup>
+    <Folder Include="Testing\" />
+  </ItemGroup>
 </Project>

tests/Auth0.AuthenticationApi.IntegrationTests/AuthenticationTests.cs

@@ -23,7 +23,7 @@ public async Task InitializeAsync()
         {
             string token = await GenerateManagementApiToken();
 
-            _managementApiClient = new TestManagementApiClient(token, GetVariable("AUTH0_MANAGEMENT_API_URL"));
+            _managementApiClient = new ManagementApiClient(token, GetVariable("AUTH0_MANAGEMENT_API_URL"));
 
             var tenantSettings = await _managementApiClient.TenantSettings.GetAsync();
 
@@ -69,7 +69,7 @@ public async Task InitializeAsync()
                 Password = Password
             });
 
-            _authenticationApiClient = new AuthenticationApiClient(GetVariable("AUTH0_AUTHENTICATION_API_URL"));
+            _authenticationApiClient = new TestAuthenticationApiClient(GetVariable("AUTH0_AUTHENTICATION_API_URL"));
         }
 
         public async Task DisposeAsync()

tests/Auth0.AuthenticationApi.IntegrationTests/DatabaseConnectionTests.cs

@@ -20,7 +20,7 @@ public async Task InitializeAsync()
         {
             string token = await GenerateManagementApiToken();
 
-            _managementApiClient = new TestManagementApiClient(token, GetVariable("AUTH0_MANAGEMENT_API_URL"));
+            _managementApiClient = new ManagementApiClient(token, GetVariable("AUTH0_MANAGEMENT_API_URL"));
 
             // We will need a connection to add the users to...
             _connection = await _managementApiClient.Connections.CreateAsync(new ConnectionCreateRequest

tests/Auth0.AuthenticationApi.IntegrationTests/Tokens/IdTokenValidatorIntegrationTests.cs

@@ -21,7 +21,7 @@ public async Task InitializeAsync()
         {
             string token = await GenerateManagementApiToken();
 
-            _managementApiClient = new TestManagementApiClient(token, GetVariable("AUTH0_MANAGEMENT_API_URL"));
+            _managementApiClient = new ManagementApiClient(token, GetVariable("AUTH0_MANAGEMENT_API_URL"));
 
             var tenantSettings = await _managementApiClient.TenantSettings.GetAsync();
 
@@ -71,7 +71,7 @@ public async Task Passes_Token_Validation_RS256()
             var clientSecret = GetVariable("AUTH0_CLIENT_SECRET");
 
             // Arrange
-            using (var authenticationApiClient = new AuthenticationApiClient(authUrl))
+            using (var authenticationApiClient = new TestAuthenticationApiClient(authUrl))
             {
                 // Act
                 var authenticationResponse = await authenticationApiClient.GetTokenAsync(new ResourceOwnerTokenRequest
@@ -98,7 +98,7 @@ public async Task Passes_Token_Validation_HS256()
             var clientSecret = GetVariable("AUTH0_HS256_CLIENT_SECRET");
 
             // Arrange
-            using (var authenticationApiClient = new AuthenticationApiClient(authUrl))
+            using (var authenticationApiClient = new TestAuthenticationApiClient(authUrl))
             {
                 // Act
                 var authenticationResponse = await authenticationApiClient.GetTokenAsync(new ResourceOwnerTokenRequest
@@ -122,7 +122,7 @@ public async Task Passes_Token_Validation_HS256()
         public async Task Passes_Token_Validation_With_CNAME()
         {
             // Arrange
-            using (var authenticationApiClient = new AuthenticationApiClient(GetVariable("AUTH0_AUTHENTICATION_API_URL")))
+            using (var authenticationApiClient = new TestAuthenticationApiClient(GetVariable("AUTH0_AUTHENTICATION_API_URL")))
             {
                 // Act
                 var authenticationResponse = await authenticationApiClient.GetTokenAsync(new ResourceOwnerTokenRequest
@@ -145,7 +145,7 @@ public async Task Passes_Token_Validation_With_CNAME()
         public async Task Fails_Token_Validation_With_Incorrect_Domain()
         {
             // Arrange
-            using (var authenticationApiClient = new AuthenticationApiClient(GetVariable("AUTH0_AUTHENTICATION_API_URL")))
+            using (var authenticationApiClient = new TestAuthenticationApiClient(GetVariable("AUTH0_AUTHENTICATION_API_URL")))
             {
                 // Act
                 var authenticationResponse = await authenticationApiClient.GetTokenAsync(new ResourceOwnerTokenRequest
@@ -171,7 +171,7 @@ public async Task Fails_Token_Validation_With_Incorrect_Domain()
         public async Task Fails_Token_Validation_With_Incorrect_Audience()
         {
             // Arrange
-            using (var authenticationApiClient = new AuthenticationApiClient(GetVariable("AUTH0_AUTHENTICATION_API_URL")))
+            using (var authenticationApiClient = new TestAuthenticationApiClient(GetVariable("AUTH0_AUTHENTICATION_API_URL")))
             {
                 // Act
                 var authenticationResponse = await authenticationApiClient.GetTokenAsync(new ResourceOwnerTokenRequest

tests/Auth0.Core.UnitTests/Auth0.Core.UnitTests.csproj

@@ -8,6 +8,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
+    <PackageReference Include="Moq" Version="4.14.7" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.1">
       <PrivateAssets>all</PrivateAssets>
@@ -19,6 +20,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\src\Auth0.Core\Auth0.Core.csproj" />
+    <ProjectReference Include="..\..\src\Auth0.ManagementApi\Auth0.ManagementApi.csproj" />
   </ItemGroup>
 
   <ItemGroup>