Skip to content

Commit af7f1b3

Browse files
dkropachevdkropachev
committed
Introduce exponential retry backoff policy
Make driver wait before retry. It is needed to mitigate retry storms that can happen in certain cases.
1 parent 3b4609b commit af7f1b3

File tree

18 files changed

+824
-18
lines changed

18 files changed

+824
-18
lines changed

‎core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,20 @@ public enum DefaultDriverOption implements DriverOption {
190190
*/
191191
RETRY_POLICY_CLASS("advanced.retry-policy.class"),
192192

193+
// BACKOFF_RETRY_POLICY is a collection of sub-properties
194+
BACKOFF_RETRY_POLICY("advanced.backoff-retry-policy"),
195+
196+
/**
197+
* The class of the backoff retry policy.
198+
*
199+
* <p>Value-type: {@link String}
200+
*/
201+
BACKOFF_RETRY_POLICY_CLASS("advanced.backoff-retry-policy.class"),
202+
203+
BACKOFF_RETRY_MAX_BACKOFF_MS("advanced.backoff-retry-policy.max-backoff-ms"),
204+
BACKOFF_RETRY_BASE_BACKOFF_MS("advanced.backoff-retry-policy.base-backoff-ms"),
205+
BACKOFF_RETRY_JITTER_RATIO("advanced.backoff-retry-policy.jitter-ratio"),
206+
193207
// SPECULATIVE_EXECUTION_POLICY is a collection of sub-properties
194208
SPECULATIVE_EXECUTION_POLICY("advanced.speculative-execution-policy"),
195209
/**
@@ -537,7 +551,6 @@ public enum DefaultDriverOption implements DriverOption {
537551
* <p>Value-type: {@link java.time.Duration Duration}
538552
*/
539553
METRICS_NODE_CQL_MESSAGES_INTERVAL("advanced.metrics.node.cql-messages.refresh-interval"),
540-
541554
/**
542555
* Whether or not to disable the Nagle algorithm.
543556
*

‎core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,10 @@ protected static void fillWithDriverDefaults(OptionsMap map) {
281281
map.put(TypedDriverOption.RECONNECTION_BASE_DELAY, Duration.ofSeconds(1));
282282
map.put(TypedDriverOption.RECONNECTION_MAX_DELAY, Duration.ofSeconds(60));
283283
map.put(TypedDriverOption.RETRY_POLICY_CLASS, "DefaultRetryPolicy");
284+
map.put(TypedDriverOption.BACKOFF_RETRY_POLICY_CLASS, "NoBackoffPolicy");
285+
map.put(TypedDriverOption.BACKOFF_RETRY_BASE_BACKOFF_MS, 100);
286+
map.put(TypedDriverOption.BACKOFF_RETRY_MAX_BACKOFF_MS, 10000);
287+
map.put(TypedDriverOption.BACKOFF_RETRY_JITTER_RATIO, 0.1);
284288
map.put(TypedDriverOption.SPECULATIVE_EXECUTION_POLICY_CLASS, "NoSpeculativeExecutionPolicy");
285289
map.put(TypedDriverOption.TIMESTAMP_GENERATOR_CLASS, "AtomicTimestampGenerator");
286290
map.put(TypedDriverOption.TIMESTAMP_GENERATOR_DRIFT_WARNING_THRESHOLD, Duration.ofSeconds(1));

‎core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,20 @@ public String toString() {
199199
/** The class of the retry policy. */
200200
public static final TypedDriverOption<String> RETRY_POLICY_CLASS =
201201
new TypedDriverOption<>(DefaultDriverOption.RETRY_POLICY_CLASS, GenericType.STRING);
202+
/** The class of the retry policy. */
203+
public static final TypedDriverOption<String> BACKOFF_RETRY_POLICY_CLASS =
204+
new TypedDriverOption<>(DefaultDriverOption.BACKOFF_RETRY_POLICY_CLASS, GenericType.STRING);
205+
/** The class of the retry policy. */
206+
public static final TypedDriverOption<Integer> BACKOFF_RETRY_BASE_BACKOFF_MS =
207+
new TypedDriverOption<>(
208+
DefaultDriverOption.BACKOFF_RETRY_BASE_BACKOFF_MS, GenericType.INTEGER);
209+
/** The class of the retry policy. */
210+
public static final TypedDriverOption<Integer> BACKOFF_RETRY_MAX_BACKOFF_MS =
211+
new TypedDriverOption<>(
212+
DefaultDriverOption.BACKOFF_RETRY_MAX_BACKOFF_MS, GenericType.INTEGER);
213+
/** The class of the retry policy. */
214+
public static final TypedDriverOption<Double> BACKOFF_RETRY_JITTER_RATIO =
215+
new TypedDriverOption<>(DefaultDriverOption.BACKOFF_RETRY_JITTER_RATIO, GenericType.DOUBLE);
202216
/** The class of the speculative execution policy. */
203217
public static final TypedDriverOption<String> SPECULATIVE_EXECUTION_POLICY_CLASS =
204218
new TypedDriverOption<>(

‎core/src/main/java/com/datastax/oss/driver/api/core/context/DriverContext.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import com.datastax.oss.driver.api.core.loadbalancing.LoadBalancingPolicy;
2828
import com.datastax.oss.driver.api.core.metadata.NodeStateListener;
2929
import com.datastax.oss.driver.api.core.metadata.schema.SchemaChangeListener;
30+
import com.datastax.oss.driver.api.core.retry.BackoffRetryPolicy;
3031
import com.datastax.oss.driver.api.core.retry.RetryPolicy;
3132
import com.datastax.oss.driver.api.core.session.Session;
3233
import com.datastax.oss.driver.api.core.session.throttling.RequestThrottler;
@@ -95,6 +96,26 @@ default RetryPolicy getRetryPolicy(@NonNull String profileName) {
9596
return (policy != null) ? policy : getRetryPolicies().get(DriverExecutionProfile.DEFAULT_NAME);
9697
}
9798

99+
/**
100+
* @return The driver's retry policies, keyed by profile name; the returned map is guaranteed to
101+
* never be {@code null} and to always contain an entry for the {@value
102+
* DriverExecutionProfile#DEFAULT_NAME} profile.
103+
*/
104+
@NonNull
105+
Map<String, BackoffRetryPolicy> getBackoffRetryPolicies();
106+
107+
/**
108+
* @param profileName the profile name; never {@code null}.
109+
* @return The driver's retry policy for the given profile; never {@code null}.
110+
*/
111+
@NonNull
112+
default BackoffRetryPolicy getBackoffRetryPolicy(@NonNull String profileName) {
113+
BackoffRetryPolicy policy = getBackoffRetryPolicies().get(profileName);
114+
return (policy != null)
115+
? policy
116+
: getBackoffRetryPolicies().get(DriverExecutionProfile.DEFAULT_NAME);
117+
}
118+
98119
/**
99120
* @return The driver's speculative execution policies, keyed by profile name; the returned map is
100121
* guaranteed to never be {@code null} and to always contain an entry for the {@value

‎core/src/main/java/com/datastax/oss/driver/api/core/retry/BackoffRetryPolicy.java‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package com.datastax.oss.driver.api.core.retry;
19+
20+
import com.datastax.oss.driver.api.core.ConsistencyLevel;
21+
import com.datastax.oss.driver.api.core.connection.ClosedConnectionException;
22+
import com.datastax.oss.driver.api.core.connection.HeartbeatException;
23+
import com.datastax.oss.driver.api.core.loadbalancing.LoadBalancingPolicy;
24+
import com.datastax.oss.driver.api.core.servererrors.BootstrappingException;
25+
import com.datastax.oss.driver.api.core.servererrors.CoordinatorException;
26+
import com.datastax.oss.driver.api.core.servererrors.FunctionFailureException;
27+
import com.datastax.oss.driver.api.core.servererrors.OverloadedException;
28+
import com.datastax.oss.driver.api.core.servererrors.ProtocolError;
29+
import com.datastax.oss.driver.api.core.servererrors.QueryValidationException;
30+
import com.datastax.oss.driver.api.core.servererrors.ReadFailureException;
31+
import com.datastax.oss.driver.api.core.servererrors.ReadTimeoutException;
32+
import com.datastax.oss.driver.api.core.servererrors.ServerError;
33+
import com.datastax.oss.driver.api.core.servererrors.TruncateException;
34+
import com.datastax.oss.driver.api.core.servererrors.WriteFailureException;
35+
import com.datastax.oss.driver.api.core.servererrors.WriteType;
36+
import com.datastax.oss.driver.api.core.session.Request;
37+
import edu.umd.cs.findbugs.annotations.NonNull;
38+
39+
/**
40+
* Defines the behavior to adopt when a request fails.
41+
*
42+
* <p>For each request, the driver gets a "query plan" (a list of coordinators to try) from the
43+
* {@link LoadBalancingPolicy}, and tries each node in sequence. This policy is invoked if the
44+
* request to that node fails.
45+
*
46+
* <p>The methods of this interface are invoked on I/O threads, therefore <b>implementations should
47+
* never block</b>. In particular, don't call {@link Thread#sleep(long)} to retry after a delay:
48+
* this would prevent asynchronous processing of other requests, and very negatively impact
49+
* throughput. If the application needs to back off and retry later, this should be implemented in
50+
* client code, not in this policy.
51+
*/
52+
public interface BackoffRetryPolicy extends AutoCloseable {
53+
/**
54+
* Whether to retry when the server replied with a {@code READ_TIMEOUT} error; this indicates a
55+
* <b>server-side</b> timeout during a read query, i.e. some replicas did not reply to the
56+
* coordinator in time.
57+
*
58+
* @param request the request that timed out.
59+
* @param cl the requested consistency level.
60+
* @param blockFor the minimum number of replica acknowledgements/responses that were required to
61+
* fulfill the operation.
62+
* @param received the number of replica that had acknowledged/responded to the operation before
63+
* it failed.
64+
* @param dataPresent whether the actual data was amongst the received replica responses. See
65+
* {@link ReadTimeoutException#wasDataPresent()}.
66+
* @param retryCount how many times the retry policy has been invoked already for this request
67+
* (not counting the current invocation).
68+
*/
69+
int onReadTimeoutBackoffMs(
70+
@NonNull Request request,
71+
@NonNull ConsistencyLevel cl,
72+
int blockFor,
73+
int received,
74+
boolean dataPresent,
75+
int retryCount,
76+
RetryVerdict verdict);
77+
78+
/**
79+
* Whether to retry when the server replied with a {@code WRITE_TIMEOUT} error; this indicates a
80+
* <b>server-side</b> timeout during a write query, i.e. some replicas did not reply to the
81+
* coordinator in time.
82+
*
83+
* <p>Note that this method will only be invoked for {@link Request#isIdempotent()} idempotent}
84+
* requests: when a write times out, it is impossible to determine with 100% certainty whether the
85+
* mutation was applied or not, so the write is never safe to retry; the driver will rethrow the
86+
* error directly, without invoking the retry policy.
87+
*
88+
* @param request the request that timed out.
89+
* @param cl the requested consistency level.
90+
* @param writeType the type of the write for which the timeout was raised.
91+
* @param blockFor the minimum number of replica acknowledgements/responses that were required to
92+
* fulfill the operation.
93+
* @param received the number of replica that had acknowledged/responded to the operation before
94+
* it failed.
95+
* @param retryCount how many times the retry policy has been invoked already for this request
96+
* (not counting the current invocation).
97+
*/
98+
int onWriteTimeoutBackoffMs(
99+
@NonNull Request request,
100+
@NonNull ConsistencyLevel cl,
101+
@NonNull WriteType writeType,
102+
int blockFor,
103+
int received,
104+
int retryCount,
105+
RetryVerdict verdict);
106+
107+
/**
108+
* Whether to retry when the server replied with an {@code UNAVAILABLE} error; this indicates that
109+
* the coordinator determined that there were not enough replicas alive to perform a query with
110+
* the requested consistency level.
111+
*
112+
* @param request the request that timed out.
113+
* @param cl the requested consistency level.
114+
* @param required the number of replica acknowledgements/responses required to perform the
115+
* operation (with its required consistency level).
116+
* @param alive the number of replicas that were known to be alive by the coordinator node when it
117+
* tried to execute the operation.
118+
* @param retryCount how many times the retry policy has been invoked already for this request
119+
* (not counting the current invocation).
120+
*/
121+
int onUnavailableBackoffMs(
122+
@NonNull Request request,
123+
@NonNull ConsistencyLevel cl,
124+
int required,
125+
int alive,
126+
int retryCount,
127+
RetryVerdict verdict);
128+
129+
/**
130+
* Whether to retry when a request was aborted before we could get a response from the server.
131+
*
132+
* <p>This can happen in two cases: if the connection was closed due to an external event (this
133+
* will manifest as a {@link ClosedConnectionException}, or {@link HeartbeatException} for a
134+
* heartbeat failure); or if there was an unexpected error while decoding the response (this can
135+
* only be a driver bug).
136+
*
137+
* <p>Note that this method will only be invoked for {@linkplain Request#isIdempotent()
138+
* idempotent} requests: when execution was aborted before getting a response, it is impossible to
139+
* determine with 100% certainty whether a mutation was applied or not, so a write is never safe
140+
* to retry; the driver will rethrow the error directly, without invoking the retry policy.
141+
*
142+
* @param request the request that was aborted.
143+
* @param error the error.
144+
* @param retryCount how many times the retry policy has been invoked already for this request
145+
* (not counting the current invocation).
146+
*/
147+
int onRequestAbortedBackoffMs(
148+
@NonNull Request request, @NonNull Throwable error, int retryCount, RetryVerdict verdict);
149+
150+
/**
151+
* Whether to retry when the server replied with a recoverable error (other than {@code
152+
* READ_TIMEOUT}, {@code WRITE_TIMEOUT} or {@code UNAVAILABLE}).
153+
*
154+
* <p>This can happen for the following errors: {@link OverloadedException}, {@link ServerError},
155+
* {@link TruncateException}, {@link ReadFailureException}, {@link WriteFailureException}.
156+
*
157+
* <p>The following errors are handled internally by the driver, and therefore will <b>never</b>
158+
* be encountered in this method:
159+
*
160+
* <ul>
161+
* <li>{@link BootstrappingException}: always retried on the next node;
162+
* <li>{@link QueryValidationException} (and its subclasses), {@link FunctionFailureException}
163+
* and {@link ProtocolError}: always rethrown.
164+
* </ul>
165+
*
166+
* <p>Note that this method will only be invoked for {@link Request#isIdempotent()} idempotent}
167+
* requests: when execution was aborted before getting a response, it is impossible to determine
168+
* with 100% certainty whether a mutation was applied or not, so a write is never safe to retry;
169+
* the driver will rethrow the error directly, without invoking the retry policy.
170+
*
171+
* @param request the request that failed.
172+
* @param error the error.
173+
* @param retryCount how many times the retry policy has been invoked already for this request
174+
* (not counting the current invocation).
175+
*/
176+
int onErrorResponseBackoff(
177+
@NonNull Request request,
178+
@NonNull CoordinatorException error,
179+
int retryCount,
180+
RetryVerdict verdict);
181+
182+
/** Called when the cluster that this policy is associated with closes. */
183+
@Override
184+
void close();
185+
}

‎core/src/main/java/com/datastax/oss/driver/api/core/retry/BackoffRetryVerdict.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package com.datastax.oss.driver.api.core.retry;
19+
20+
import java.time.Duration;
21+
22+
/**
23+
* The verdict returned by a {@link RetryPolicy} determining what to do when a request failed. A
24+
* verdict contains a {@link RetryDecision} indicating if a retry should be attempted at all and
25+
* where, with what delay, and a method that allows the original request to be modified before the
26+
* retry.
27+
*/
28+
public interface BackoffRetryVerdict extends RetryVerdict {
29+
30+
/** @return a delay that request needs to take before retrying. */
31+
Duration getRetryBackoff();
32+
}

‎core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import com.datastax.oss.driver.api.core.metadata.Node;
3939
import com.datastax.oss.driver.api.core.metadata.NodeStateListener;
4040
import com.datastax.oss.driver.api.core.metadata.schema.SchemaChangeListener;
41+
import com.datastax.oss.driver.api.core.retry.BackoffRetryPolicy;
4142
import com.datastax.oss.driver.api.core.retry.RetryPolicy;
4243
import com.datastax.oss.driver.api.core.session.ProgrammaticArguments;
4344
import com.datastax.oss.driver.api.core.session.throttling.RequestThrottler;
@@ -150,6 +151,8 @@ public class DefaultDriverContext implements InternalDriverContext {
150151
new LazyReference<>("reconnectionPolicy", this::buildReconnectionPolicy, cycleDetector);
151152
private final LazyReference<Map<String, RetryPolicy>> retryPoliciesRef =
152153
new LazyReference<>("retryPolicies", this::buildRetryPolicies, cycleDetector);
154+
private final LazyReference<Map<String, BackoffRetryPolicy>> backoffRetryPoliciesRef =
155+
new LazyReference<>("backoffRetryPolicies", this::buildBackoffRetryPolicies, cycleDetector);
153156
private final LazyReference<Map<String, SpeculativeExecutionPolicy>>
154157
speculativeExecutionPoliciesRef =
155158
new LazyReference<>(
@@ -367,6 +370,15 @@ protected Map<String, RetryPolicy> buildRetryPolicies() {
367370
"com.datastax.oss.driver.internal.core.retry");
368371
}
369372

373+
protected Map<String, BackoffRetryPolicy> buildBackoffRetryPolicies() {
374+
return Reflection.buildFromConfigProfiles(
375+
this,
376+
DefaultDriverOption.BACKOFF_RETRY_POLICY_CLASS,
377+
DefaultDriverOption.BACKOFF_RETRY_POLICY,
378+
BackoffRetryPolicy.class,
379+
"com.datastax.oss.driver.internal.core.retry");
380+
}
381+
370382
protected Map<String, SpeculativeExecutionPolicy> buildSpeculativeExecutionPolicies() {
371383
return Reflection.buildFromConfigProfiles(
372384
this,
@@ -768,6 +780,12 @@ public Map<String, RetryPolicy> getRetryPolicies() {
768780
return retryPoliciesRef.get();
769781
}
770782

783+
@NonNull
784+
@Override
785+
public Map<String, BackoffRetryPolicy> getBackoffRetryPolicies() {
786+
return backoffRetryPoliciesRef.get();
787+
}
788+
771789
@NonNull
772790
@Override
773791
public Map<String, SpeculativeExecutionPolicy> getSpeculativeExecutionPolicies() {

‎core/src/main/java/com/datastax/oss/driver/internal/core/cql/Conversions.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import com.datastax.oss.driver.api.core.metadata.schema.ColumnMetadata;
4747
import com.datastax.oss.driver.api.core.metadata.schema.RelationMetadata;
4848
import com.datastax.oss.driver.api.core.metadata.token.Partitioner;
49+
import com.datastax.oss.driver.api.core.retry.BackoffRetryPolicy;
4950
import com.datastax.oss.driver.api.core.retry.RetryPolicy;
5051
import com.datastax.oss.driver.api.core.servererrors.AlreadyExistsException;
5152
import com.datastax.oss.driver.api.core.servererrors.BootstrappingException;
@@ -606,6 +607,11 @@ public static RetryPolicy resolveRetryPolicy(
606607
return context.getRetryPolicy(executionProfile.getName());
607608
}
608609

610+
public static BackoffRetryPolicy resolveBackoffRetryPolicy(
611+
InternalDriverContext context, DriverExecutionProfile executionProfile) {
612+
return context.getBackoffRetryPolicy(executionProfile.getName());
613+
}
614+
609615
/**
610616
* Use {@link #resolveSpeculativeExecutionPolicy(InternalDriverContext, DriverExecutionProfile)}
611617
* instead.

0 commit comments

Comments
 (0)