Availability: Fixes retry behavior on HttpException

Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs

@@ -68,10 +68,13 @@ public async Task<ShouldRetryResult> ShouldRetryAsync(
 
  this.retryContext = null;
  // Received Connection error (HttpRequestException), initiate the endpoint rediscovery
- if (exception is HttpRequestException httpException)
+ if (exception is HttpRequestException _)
  {
  DefaultTrace.TraceWarning("Endpoint not reachable. Refresh cache and retry");
- return await this.ShouldRetryOnEndpointFailureAsync(this.isReadRequest, false);
+ return await this.ShouldRetryOnEndpointFailureAsync(
+ isReadRequest: this.isReadRequest,
+ forceRefresh: false,
+ retryOnPreferredLocations: true);
  }
 
  DocumentClientException clientException = exception as DocumentClientException;
@@ -146,7 +149,7 @@ public void OnBeforeSendRequest(DocumentServiceRequest request)
  if (this.retryContext != null)
  {
  // set location-based routing directive based on request retry context
- request.RequestContext.RouteToLocation(this.retryContext.RetryCount, this.retryContext.RetryRequestOnPreferredLocations);
+ request.RequestContext.RouteToLocation(this.retryContext.RetryLocationIndex, this.retryContext.RetryRequestOnPreferredLocations);
  }
 
  // Resolve the endpoint for the request and pin the resolution to the resolved endpoint
@@ -171,7 +174,10 @@ private async Task<ShouldRetryResult> ShouldRetryInternalAsync(
  && subStatusCode == SubStatusCodes.WriteForbidden)
  {
  DefaultTrace.TraceWarning("Endpoint not writable. Refresh cache and retry");
- return await this.ShouldRetryOnEndpointFailureAsync(false, true);
+ return await this.ShouldRetryOnEndpointFailureAsync(
+ isReadRequest: false,
+ forceRefresh: true,
+ retryOnPreferredLocations: false);
  }
 
  // Regional endpoint is not available yet for reads (e.g. add/ online of region is in progress)
@@ -180,7 +186,10 @@ private async Task<ShouldRetryResult> ShouldRetryInternalAsync(
  && (this.isReadRequest || this.canUseMultipleWriteLocations))
  {
  DefaultTrace.TraceWarning("Endpoint not available for reads. Refresh cache and retry");
- return await this.ShouldRetryOnEndpointFailureAsync(true, false);
+ return await this.ShouldRetryOnEndpointFailureAsync(
+ isReadRequest: this.isReadRequest,
+ forceRefresh: false,
+ retryOnPreferredLocations: false);
  }
 
  if (statusCode == HttpStatusCode.NotFound
@@ -199,7 +208,10 @@ private async Task<ShouldRetryResult> ShouldRetryInternalAsync(
  return null;
  }
 
- private async Task<ShouldRetryResult> ShouldRetryOnEndpointFailureAsync(bool isReadRequest, bool forceRefresh)
+ private async Task<ShouldRetryResult> ShouldRetryOnEndpointFailureAsync(
+ bool isReadRequest,
+ bool forceRefresh,
+ bool retryOnPreferredLocations)
  {
  if (!this.enableEndpointDiscovery || this.failoverRetryCount > MaxRetryCount)
  {
@@ -239,10 +251,16 @@ private async Task<ShouldRetryResult> ShouldRetryOnEndpointFailureAsync(bool isR
 
  await this.globalEndpointManager.RefreshLocationAsync(null, forceRefresh);
 
+ int retryLocationIndex = this.failoverRetryCount; // Used to generate a round-robin effect
+ if (retryOnPreferredLocations)
+ {
+ retryLocationIndex = 0; // When the endpoint is marked as unavailable, it is moved to the bottom of the preferrence list
+ }
+
  this.retryContext = new RetryContext
  {
- RetryCount = this.failoverRetryCount,
- RetryRequestOnPreferredLocations = false
+ RetryLocationIndex = retryLocationIndex,
+ RetryRequestOnPreferredLocations = retryOnPreferredLocations,
  };
 
  return ShouldRetryResult.RetryAfter(retryDelay);
@@ -273,7 +291,7 @@ private ShouldRetryResult ShouldRetryOnSessionNotAvailable()
  {
  this.retryContext = new RetryContext()
  {
- RetryCount = this.sessionTokenRetryCount - 1,
+ RetryLocationIndex = this.sessionTokenRetryCount - 1,
  RetryRequestOnPreferredLocations = this.sessionTokenRetryCount > 1
  };
 
@@ -292,7 +310,7 @@ private ShouldRetryResult ShouldRetryOnSessionNotAvailable()
  {
  this.retryContext = new RetryContext
  {
- RetryCount = this.sessionTokenRetryCount - 1,
+ RetryLocationIndex = this.sessionTokenRetryCount - 1,
  RetryRequestOnPreferredLocations = false
  };
 
@@ -336,7 +354,7 @@ private ShouldRetryResult ShouldRetryOnServiceUnavailable()
  // RetryCount is used as zero-based index
  this.retryContext = new RetryContext()
  {
- RetryCount = this.serviceUnavailableRetryCount,
+ RetryLocationIndex = this.serviceUnavailableRetryCount,
  RetryRequestOnPreferredLocations = true
  };
 
@@ -345,7 +363,7 @@ private ShouldRetryResult ShouldRetryOnServiceUnavailable()
 
  private sealed class RetryContext
  {
- public int RetryCount { get; set; }
+ public int RetryLocationIndex { get; set; }
  public bool RetryRequestOnPreferredLocations { get; set; }
  }
  }

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GlobalEndpointManagerTest.cs

@@ -70,7 +70,7 @@ public async Task EndpointFailureMockTest()
  Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], globalEndpointManager.WriteEndpoints[0]);
 
  //Sleep a second for the unavailable endpoint entry to expire and background refresh timer to kick in
- Thread.Sleep(2000);
+ Thread.Sleep(3000);
  await globalEndpointManager.RefreshLocationAsync(null);
  Assert.AreEqual(globalEndpointManager.ReadEndpoints[0], new Uri(readLocation1.Endpoint));
  }