[automatic failover] Implement max number of failover attempts (#4293)

* - implement max failover attempt
- add tests

* - fix user receive the intended exception

* -clean+format

* - java doc for exceptions

* format

* - more tests on excaption types in max failover attempts mechanism

* format

* fix failing timing in test

* disable health checks

* rename to switchToHealthyCluster

* format

Author: atakavci

src/main/java/redis/clients/jedis/MultiClusterClientConfig.java

@@ -161,6 +161,12 @@ public static interface StrategySupplier {
   /** Default grace period in milliseconds to keep clusters disabled after they become unhealthy. */
   private static final long GRACE_PERIOD_DEFAULT = 10000;
 
+  /** Default maximum number of failover attempts. */
+  private static final int MAX_NUM_FAILOVER_ATTEMPTS_DEFAULT = 10;
+
+  /** Default delay in milliseconds between failover attempts. */
+  private static final int DELAY_IN_BETWEEN_FAILOVER_ATTEMPTS_DEFAULT = 12000;
+
   /** Array of cluster configurations defining the available Redis endpoints and their settings. */
   private final ClusterConfig[] clusterConfigs;
 
@@ -485,6 +491,34 @@ public static interface StrategySupplier {
    */
   private boolean fastFailover;
 
+  /**
+   * Maximum number of failover attempts.
+   * <p>
+   * This setting controls how many times the system will attempt to failover to a different cluster
+   * before giving up. For example, if set to 3, the system will make 1 initial attempt plus 2
+   * failover attempts for a total of 3 attempts.
+   * </p>
+   * <p>
+   * <strong>Default:</strong> {@value #MAX_NUM_FAILOVER_ATTEMPTS_DEFAULT}
+   * </p>
+   * @see #getMaxNumFailoverAttempts()
+   */
+  private int maxNumFailoverAttempts;
+
+  /**
+   * Delay in milliseconds between failover attempts.
+   * <p>
+   * This setting controls how long the system will wait before attempting to failover to a
+   * different cluster. For example, if set to 1000, the system will wait 1 second before attempting
+   * to failover to a different cluster.
+   * </p>
+   * <p>
+   * <strong>Default:</strong> {@value #DELAY_IN_BETWEEN_FAILOVER_ATTEMPTS_DEFAULT} milliseconds
+   * </p>
+   * @see #getDelayInBetweenFailoverAttempts()
+   */
+  private int delayInBetweenFailoverAttempts;
+
   /**
    * Constructs a new MultiClusterClientConfig with the specified cluster configurations.
    * <p>
@@ -679,6 +713,25 @@ public long getGracePeriod() {
     return gracePeriod;
   }
 
+  /**
+   * Returns the maximum number of failover attempts.
+   * @return maximum number of failover attempts
+   * @see #maxNumFailoverAttempts
+   */
+  public int getMaxNumFailoverAttempts() {
+    return maxNumFailoverAttempts;
+
+  }
+
+  /**
+   * Returns the delay in milliseconds between failover attempts.
+   * @return delay in milliseconds between failover attempts
+   * @see #delayInBetweenFailoverAttempts
+   */
+  public int getDelayInBetweenFailoverAttempts() {
+    return delayInBetweenFailoverAttempts;
+  }
+
   /**
    * Returns whether connections are forcefully terminated during failover.
    * @return true if fast failover is enabled, false for graceful failover
@@ -1090,6 +1143,12 @@ public static class Builder {
     /** Whether to forcefully terminate connections during failover. */
     private boolean fastFailover = false;
 
+    /** Maximum number of failover attempts. */
+    private int maxNumFailoverAttempts = MAX_NUM_FAILOVER_ATTEMPTS_DEFAULT;
+
+    /** Delay in milliseconds between failover attempts. */
+    private int delayInBetweenFailoverAttempts = DELAY_IN_BETWEEN_FAILOVER_ATTEMPTS_DEFAULT;
+
     /**
      * Constructs a new Builder with the specified cluster configurations.
      * @param clusterConfigs array of cluster configurations defining available Redis endpoints
@@ -1539,6 +1598,42 @@ public Builder fastFailover(boolean fastFailover) {
       return this;
     }
 
+    /**
+     * Sets the maximum number of failover attempts.
+     * <p>
+     * This setting controls how many times the system will attempt to failover to a different
+     * cluster before giving up. For example, if set to 3, the system will make 1 initial attempt
+     * plus 2 failover attempts for a total of 3 attempts.
+     * </p>
+     * <p>
+     * <strong>Default:</strong> {@value #MAX_NUM_FAILOVER_ATTEMPTS_DEFAULT}
+     * </p>
+     * @param maxNumFailoverAttempts maximum number of failover attempts
+     * @return this builder instance for method chaining
+     */
+    public Builder maxNumFailoverAttempts(int maxNumFailoverAttempts) {
+      this.maxNumFailoverAttempts = maxNumFailoverAttempts;
+      return this;
+    }
+
+    /**
+     * Sets the delay in milliseconds between failover attempts.
+     * <p>
+     * This setting controls how long the system will wait before attempting to failover to a
+     * different cluster. For example, if set to 1000, the system will wait 1 second before
+     * attempting to failover to a different cluster.
+     * </p>
+     * <p>
+     * <strong>Default:</strong> {@value #DELAY_IN_BETWEEN_FAILOVER_ATTEMPTS_DEFAULT} milliseconds
+     * </p>
+     * @param delayInBetweenFailoverAttempts delay in milliseconds between failover attempts
+     * @return this builder instance for method chaining
+     */
+    public Builder delayInBetweenFailoverAttempts(int delayInBetweenFailoverAttempts) {
+      this.delayInBetweenFailoverAttempts = delayInBetweenFailoverAttempts;
+      return this;
+    }
+
     /**
      * Builds and returns a new MultiClusterClientConfig instance with all configured settings.
      * <p>
@@ -1576,6 +1671,8 @@ public MultiClusterClientConfig build() {
       config.failbackCheckInterval = this.failbackCheckInterval;
       config.gracePeriod = this.gracePeriod;
       config.fastFailover = this.fastFailover;
+      config.maxNumFailoverAttempts = this.maxNumFailoverAttempts;
+      config.delayInBetweenFailoverAttempts = this.delayInBetweenFailoverAttempts;
 
       return config;
     }

src/main/java/redis/clients/jedis/mcf/CircuitBreakerCommandExecutor.java

@@ -7,6 +7,7 @@
 import redis.clients.jedis.CommandObject;
 import redis.clients.jedis.Connection;
 import redis.clients.jedis.annots.Experimental;
+import redis.clients.jedis.exceptions.JedisConnectionException;
 import redis.clients.jedis.executors.CommandExecutor;
 import redis.clients.jedis.mcf.MultiClusterPooledConnectionProvider.Cluster;
 
@@ -46,7 +47,14 @@ public <T> T executeCommand(CommandObject<T> commandObject) {
    * Functional interface wrapped in retry and circuit breaker logic to handle happy path scenarios
    */
   private <T> T handleExecuteCommand(CommandObject<T> commandObject, Cluster cluster) {
-    try (Connection connection = cluster.getConnection()) {
+    Connection connection;
+    try {
+      connection = cluster.getConnection();
+    } catch (JedisConnectionException e) {
+      provider.assertOperability();
+      throw e;
+    }
+    try {
       return connection.executeCommand(commandObject);
     } catch (Exception e) {
       if (cluster.retryOnFailover() && !isActiveCluster(cluster)
@@ -56,6 +64,8 @@ && isCircuitBreakerTrackedException(e, cluster.getCircuitBreaker())) {
       }
 
       throw e;
+    } finally {
+      connection.close();
     }
   }
 

src/main/java/redis/clients/jedis/mcf/CircuitBreakerFailoverBase.java

@@ -63,19 +63,16 @@ protected void clusterFailover(Cluster cluster) {
         // Iterating the active cluster will allow subsequent calls to the executeCommand() to use
         // the next
         // cluster's connection pool - according to the configuration's prioritization/order/weight
-        provider.iterateActiveCluster(SwitchReason.CIRCUIT_BREAKER);
+        provider.switchToHealthyCluster(SwitchReason.CIRCUIT_BREAKER, cluster);
       }
       // this check relies on the fact that many failover attempts can hit with the same CB,
       // only the first one will trigger a failover, and make the CB FORCED_OPEN.
       // when the rest reaches here, the active cluster is already the next one, and should be
       // different than
       // active CB. If its the same one and there are no more clusters to failover to, then throw an
       // exception
-      else if (cluster == provider.getCluster() && !provider.canIterateOnceMore()) {
-        throw new JedisConnectionException(
-            "Cluster/database endpoint could not failover since the MultiClusterClientConfig was not "
-                + "provided with an additional cluster/database endpoint according to its prioritized sequence. "
-                + "If applicable, consider failing back OR restarting with an available cluster/database endpoint");
+      else if (cluster == provider.getCluster()) {
+        provider.switchToHealthyCluster(SwitchReason.CIRCUIT_BREAKER, cluster);
       }
       // Ignore exceptions since we are already in a failure state
     } finally {

src/main/java/redis/clients/jedis/mcf/JedisFailoverException.java

@@ -0,0 +1,66 @@
+package redis.clients.jedis.mcf;
+
+import redis.clients.jedis.exceptions.JedisConnectionException;
+
+/**
+ * Exception thrown when a failover attempt fails due to lack of available/healthy clusters.
+ * <p>
+ * This exception itself is not thrown, see the child exceptions for more details.
+ * </p>
+ * @see JedisFailoverException.JedisPermanentlyNotAvailableException
+ * @see JedisFailoverException.JedisTemporarilyNotAvailableException
+ */
+public class JedisFailoverException extends JedisConnectionException {
+  private static final String MESSAGE = "Cluster/database endpoint could not failover since the MultiClusterClientConfig was not "
+      + "provided with an additional cluster/database endpoint according to its prioritized sequence. "
+      + "If applicable, consider falling back OR restarting with an available cluster/database endpoint";
+
+  public JedisFailoverException(String s) {
+    super(s);
+  }
+
+  public JedisFailoverException() {
+    super(MESSAGE);
+  }
+
+  /**
+   * Exception thrown when a failover attempt fails due to lack of available/healthy clusters, and
+   * the max number of failover attempts has been exceeded. And there is still no healthy cluster.
+   * <p>
+   * See the configuration properties
+   * {@link redis.clients.jedis.MultiClusterClientConfig#maxNumFailoverAttempts} and
+   * {@link redis.clients.jedis.MultiClusterClientConfig#delayInBetweenFailoverAttempts} for more
+   * details.
+   */
+  public static class JedisPermanentlyNotAvailableException extends JedisFailoverException {
+    public JedisPermanentlyNotAvailableException(String s) {
+      super(s);
+    }
+
+    public JedisPermanentlyNotAvailableException() {
+      super();
+    }
+  }
+
+  /**
+   * Exception thrown when a failover attempt fails due to lack of available/healthy clusters, but
+   * the max number of failover attempts has not been exceeded yet. Though there is no healthy
+   * cluster including the selected/current one, given configuration suggests that it should be a
+   * temporary condition and it is possible that there will be a healthy cluster available.
+   * <p>
+   * See the configuration properties
+   * {@link redis.clients.jedis.MultiClusterClientConfig#maxNumFailoverAttempts} and
+   * {@link redis.clients.jedis.MultiClusterClientConfig#delayInBetweenFailoverAttempts} for more
+   * details.
+   */
+  public static class JedisTemporarilyNotAvailableException extends JedisFailoverException {
+
+    public JedisTemporarilyNotAvailableException(String s) {
+      super(s);
+    }
+
+    public JedisTemporarilyNotAvailableException() {
+      super();
+    }
+  }
+}