Skip to content

Commit

Permalink
implement generic Retryable from config
Browse files Browse the repository at this point in the history
  • Loading branch information
aktaskaan committed Dec 8, 2023
1 parent 3308169 commit 46b067b
Show file tree
Hide file tree
Showing 10 changed files with 419 additions and 58 deletions.
Original file line number Diff line number Diff line change
@@ -1,37 +1,12 @@
package uk.gov.hmcts.befta.player;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.rholder.retry.RetryException;
import com.github.rholder.retry.Retryer;
import com.github.rholder.retry.RetryerBuilder;
import com.github.rholder.retry.StopStrategies;
import com.github.rholder.retry.WaitStrategies;
import com.google.common.base.Predicates;
import org.apache.commons.collections4.map.HashedMap;
import org.apache.http.impl.EnglishReasonPhraseCatalog;
import org.aspectj.util.FileUtil;
import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.fasterxml.jackson.databind.ObjectMapper;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.ConnectException;
import java.text.DecimalFormat;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import io.cucumber.java.Before;
import io.cucumber.java.Scenario;
import io.cucumber.java.en.Given;
Expand All @@ -43,6 +18,12 @@
import io.restassured.specification.QueryableRequestSpecification;
import io.restassured.specification.RequestSpecification;
import io.restassured.specification.SpecificationQuerier;
import org.apache.commons.collections4.map.HashedMap;
import org.apache.http.impl.EnglishReasonPhraseCatalog;
import org.aspectj.util.FileUtil;
import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import uk.gov.hmcts.befta.AuthenticationRetryConfiguration;
import uk.gov.hmcts.befta.BeftaMain;
import uk.gov.hmcts.befta.TestAutomationConfig;
Expand All @@ -58,15 +39,32 @@
import uk.gov.hmcts.befta.exception.UnconfirmedApiCallException;
import uk.gov.hmcts.befta.exception.UnconfirmedDataSpecException;
import uk.gov.hmcts.befta.factory.BeftaScenarioContextFactory;
import uk.gov.hmcts.befta.featuretoggle.ScenarioFeatureToggleInfo;
import uk.gov.hmcts.befta.featuretoggle.FeatureToggleService;
import uk.gov.hmcts.befta.featuretoggle.ScenarioFeatureToggleInfo;
import uk.gov.hmcts.befta.util.BeftaUtils;
import uk.gov.hmcts.befta.util.EnvironmentVariableUtils;
import uk.gov.hmcts.befta.util.JsonUtils;
import uk.gov.hmcts.befta.util.MapVerificationResult;
import uk.gov.hmcts.befta.util.MapVerifier;
import uk.gov.hmcts.befta.util.Retryable;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.ConnectException;
import java.text.DecimalFormat;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class DefaultBackEndFunctionalTestScenarioPlayer implements BackEndFunctionalTestAutomationDSL {

static final String HTTP_S_REGEX = "^(http|https):.*";
Expand Down Expand Up @@ -358,6 +356,7 @@ public void submitTheRequestToCallAnOperationOfAProduct(String operation, String
submitTheRequestToCallAnOperationOfAProduct(this.scenarioContext, operation, productName);
}

@SuppressWarnings("UnstableApiUsage")
private void submitTheRequestToCallAnOperationOfAProduct(BackEndFunctionalTestScenarioContext scenarioContext,
String operationName, String productName) throws IOException {
boolean isCorrectOperation = scenarioContext.getTestData().meetsOperationOfProduct(productName, operationName);
Expand All @@ -378,16 +377,30 @@ private void submitTheRequestToCallAnOperationOfAProduct(BackEndFunctionalTestSc
}

Retryable retryable = scenarioContext.getRetryConfiguration();
logger.info("Applying retry policy with the following configuration:"
+ " Max attempts: {}, Retry on status codes: {},"
+ " Delay between retries: {}ms.", retryable.getMaxAttempts(), retryable.getStatusCodes()
, retryable.getDelay());

Retryer<Response> retryer = RetryerBuilder.<Response>newBuilder()
.retryIfResult(res -> retryable.getStatusCodes().contains(res.getStatusCode()))
.withStopStrategy(StopStrategies.stopAfterAttempt(retryable.getMaxAttempts()))
.withWaitStrategy(WaitStrategies.fixedWait(retryable.getDelay(), TimeUnit.MILLISECONDS))
.build();
Retryer<Response> retryer;

logger.info("Calling: {} {}", testData.getMethod(), uri);
if (retryable.getNonRetryableHttpMethods().contains("*") || retryable.getNonRetryableHttpMethods()
.contains(testData.getMethod())) {
logger.info("Applying no-retry policy...");
retryer = RetryerBuilder.<Response>newBuilder().build();
} else {
logger.info("Applying active retry policy...");

retryer = RetryerBuilder.<Response>newBuilder()
.withRetryListener(retryable.getRetryListener())
.retryIfException(e -> {
boolean isRetryableException = retryable.getRetryableExceptions().contains(e.getClass());
Throwable cause = e.getCause();
boolean isRetryableCause = cause != null && retryable.getRetryableExceptions()
.contains(cause.getClass());
return isRetryableException || isRetryableCause;
})
.retryIfResult(res -> retryable.getStatusCodes().contains(res.getStatusCode()))
.withStopStrategy(StopStrategies.stopAfterAttempt(retryable.getMaxAttempts()))
.withWaitStrategy(WaitStrategies.fixedWait(retryable.getDelay(), TimeUnit.MILLISECONDS))
.build();
}

Response response = executeHttpRequestWithRetry(theRequest, testData.getMethod(), uri, retryer);

Expand Down
2 changes: 1 addition & 1 deletion src/main/java/uk/gov/hmcts/befta/util/BeftaUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ public static Retryable getRetryableTag(Scenario scenario) {
.collect(Collectors.joining());

if (retryInput.isEmpty()) {
return Retryable.N0_RETRYABLE;
return Retryable.RETRYABLE_FROM_CONFIG;
}

// default 1000ms
Expand Down
162 changes: 159 additions & 3 deletions src/main/java/uk/gov/hmcts/befta/util/Retryable.java
Original file line number Diff line number Diff line change
@@ -1,21 +1,177 @@
package uk.gov.hmcts.befta.util;

import com.github.rholder.retry.Attempt;
import com.github.rholder.retry.RetryListener;
import io.restassured.internal.RestAssuredResponseImpl;
import lombok.Builder;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;

import java.io.InputStream;
import java.util.Collections;
import java.util.HashSet;
import java.util.Properties;
import java.util.Set;

@Slf4j
@Builder
@Getter
@SuppressWarnings("UnstableApiUsage")
public class Retryable {
private static final String BEFTA_RETRY_MAX_ATTEMPTS_ENV_VAR = "BEFTA_RETRY_MAX_ATTEMPTS";
private static final String BEFTA_RETRY_MAX_ATTEMPTS_PROPERTY = "befta.retry.maxAttempts";
private static final String BEFTA_RETRY_STATUS_CODES_ENV_VAR = "BEFTA_RETRY_STATUS_CODES";
private static final String BEFTA_RETRY_STATUS_CODES_PROPERTY = "befta.retry.statusCodes";
private static final String BEFTA_RETRY_MAX_DELAY_ENV_VAR = "BEFTA_RETRY_MAX_DELAY";
private static final String BEFTA_RETRY_MAX_DELAY_PROPERTY = "befta.retry.maxDelay";
private static final String BEFTA_RETRY_RETRYABLE_EXCEPTIONS_ENV_VAR = "BEFTA_RETRY_RETRYABLE_EXCEPTIONS";
private static final String BEFTA_RETRY_RETRYABLE_EXCEPTIONS_PROPERTY = "befta.retry.retryableExceptions";
private static final String BEFTA_RETRY_NON_RETRYABLE_HTTP_METHODS_ENV_VAR =
"BEFTA_RETRY_NON_RETRYABLE_HTTP_METHODS";
private static final String BEFTA_RETRY_NON_RETRYABLE_HTTP_METHODS_PROPERTY = "befta.retry.nonRetryableHttpMethods";
private static final String BEFTA_RETRY_ENABLE_LISTENER_ENV_VAR = "BEFTA_RETRY_ENABLE_LISTENER";
private static final String BEFTA_RETRY_ENABLE_LISTENER_PROPERTY = "befta.retry.enable.listener";

public static Retryable N0_RETRYABLE = Retryable.builder().build();
private static final int DEFAULT_MAX_ATTEMPTS = 1;
private static final String DEFAULT_STATUS_CODES = "";
private static final int DEFAULT_MAX_DELAY = 0;
private static final String DEFAULT_RETRYABLE_EXCEPTIONS = "";
private static final String DEFAULT_NON_RETRYABLE_HTTP_METHODS = "";

private static final String resourceName = "retry-config.properties";

public static final Retryable DEFAULT_RETRYABLE = Retryable.builder().build();
public static final Retryable RETRYABLE_FROM_CONFIG = createFromConfiguration(resourceName);

@Builder.Default
private int maxAttempts = 1;
private int maxAttempts = DEFAULT_MAX_ATTEMPTS;
@Builder.Default
private Set<Integer> statusCodes = new HashSet<>();
@Builder.Default
private int delay = 0;
private int delay = DEFAULT_MAX_DELAY;
@Builder.Default
private Set<Class<? extends Exception>> retryableExceptions = new HashSet<>();
@Builder.Default
private Set<String> nonRetryableHttpMethods = new HashSet<>(Collections.singleton("*"));
private RetryListener retryListener;

public static Retryable createFromConfiguration(String configFile) {
if (configFile == null || configFile.isEmpty()) {
return DEFAULT_RETRYABLE;
}

try (InputStream input = Retryable.class.getClassLoader().getResourceAsStream(configFile)) {
if (input == null) {
return DEFAULT_RETRYABLE;
}

Properties properties = new Properties();
properties.load(input);

String disableListener = getEnvironmentOrProperty(properties, BEFTA_RETRY_ENABLE_LISTENER_ENV_VAR,
BEFTA_RETRY_ENABLE_LISTENER_PROPERTY, "true");

Retryable retryable = Retryable.builder()
.maxAttempts(Integer.parseInt(getEnvironmentOrProperty(properties, BEFTA_RETRY_MAX_ATTEMPTS_ENV_VAR,
BEFTA_RETRY_MAX_ATTEMPTS_PROPERTY, String.valueOf(DEFAULT_MAX_ATTEMPTS))))
.statusCodes(parseStatusCodes(getEnvironmentOrProperty(properties, BEFTA_RETRY_STATUS_CODES_ENV_VAR,
BEFTA_RETRY_STATUS_CODES_PROPERTY, DEFAULT_STATUS_CODES)))
.delay(Integer.parseInt(getEnvironmentOrProperty(properties, BEFTA_RETRY_MAX_DELAY_ENV_VAR,
BEFTA_RETRY_MAX_DELAY_PROPERTY, String.valueOf(DEFAULT_MAX_DELAY))))
.retryableExceptions(parseRetryableExceptions(getEnvironmentOrProperty(properties,
BEFTA_RETRY_RETRYABLE_EXCEPTIONS_ENV_VAR, BEFTA_RETRY_RETRYABLE_EXCEPTIONS_PROPERTY,
DEFAULT_RETRYABLE_EXCEPTIONS)))
.nonRetryableHttpMethods(parseHttpMethods(getEnvironmentOrProperty(properties,
BEFTA_RETRY_NON_RETRYABLE_HTTP_METHODS_ENV_VAR,
BEFTA_RETRY_NON_RETRYABLE_HTTP_METHODS_PROPERTY,
DEFAULT_NON_RETRYABLE_HTTP_METHODS)))
.retryListener(setRetryListener(Boolean.parseBoolean(disableListener)))
.build();

log.info("Creating retry policy with the following configuration:\n"
+ " Max attempts: {}\n"
+ " Retry on status codes: {}\n"
+ " Retry on exceptions: {}\n"
+ " No retry on http methods: {}\n"
+ " Delay between retries: {}ms.",
retryable.getMaxAttempts(), retryable.getStatusCodes(), retryable.getRetryableExceptions(),
retryable.getNonRetryableHttpMethods(), retryable.getDelay());

return retryable;
} catch (Exception e) {
return DEFAULT_RETRYABLE;
}
}

public static RetryListener setRetryListener(boolean isEnableListener) {
log.info("Initializing retry listener...");
return new RetryListener() {
@Override
public <V> void onRetry(Attempt<V> attempt) {
if (isEnableListener) {
StringBuilder logMessage = new StringBuilder();
logMessage.append(String.format("Retry attempt %d. ", attempt.getAttemptNumber()));

if (attempt.hasException()) {
logMessage.append(String.format("exception: '%s'. ", attempt.getExceptionCause()));
} else if (attempt.getResult() instanceof RestAssuredResponseImpl) {
RestAssuredResponseImpl result = (RestAssuredResponseImpl) attempt.getResult();
logMessage.append(String.format("result: '%s'. ", result.response().getStatusLine().trim()));
}

logMessage.append(String.format("%d ms delay since the first attempt",
attempt.getDelaySinceFirstAttempt()));

log.info(logMessage.toString());
}
}
};
}

private static String getEnvironmentOrProperty(Properties properties, String envVarName, String propertyName,
String defaultValue) {
String envVarValue = System.getenv(envVarName);
return envVarValue != null ? envVarValue : properties.getProperty(propertyName, defaultValue);
}

private static Set<String> parseHttpMethods(final String httpMethods) {
Set<String> methods = new HashSet<>();
if (!httpMethods.isEmpty()) {
String[] methodsArray = httpMethods.split(",");
for (String method : methodsArray) {
methods.add(method.trim());
}
}
return methods;
}

private static Set<Integer> parseStatusCodes(final String statusCodes) {
Set<Integer> codes = new HashSet<>();
if (!statusCodes.isEmpty()) {
String[] codesArray = statusCodes.split(",");
for (String code : codesArray) {
codes.add(Integer.parseInt(code.trim()));
}
}
return codes;
}

@SuppressWarnings("unchecked")
private static Set<Class<? extends Exception>> parseRetryableExceptions(final String exceptions) {
Set<Class<? extends Exception>> exceptionSet = new HashSet<>();
if (!exceptions.isEmpty()) {
String[] exceptionsArray = exceptions.split(",");
for (String exceptionClassName : exceptionsArray) {
try {
Class<?> exceptionClass = Class.forName(exceptionClassName.trim());
if (Exception.class.isAssignableFrom(exceptionClass)) {
exceptionSet.add((Class<? extends Exception>) exceptionClass);
}
} catch (Exception e) {
// Ignore exception if the class is not found
}
}
}
return exceptionSet;
}
}
6 changes: 6 additions & 0 deletions src/main/resources/retry-config.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
befta.retry.maxAttempts=1
befta.retry.maxDelay=1000
befta.retry.statusCodes=500,502,503,504
befta.retry.retryableExceptions=java.net.SocketException,javax.net.ssl.SSLException,java.net.ConnectException
befta.retry.nonRetryableHttpMethods=*
befta.retry.enable.listener=true
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ public void shouldMapRetryableToDefaultRetryConfigurationWhenBlankCommaUsed() {
}

@Test
public void shouldThrowExceptionWhenStatusCodesMissingInRetryableTag() {
public void shouldRetrieveDefaultRetryWhenThereIsNoRetryableTag() {
final Collection<String> tags = new ArrayList<String>() {
private static final long serialVersionUID = 1L;
{
Expand All @@ -211,9 +211,12 @@ public void shouldThrowExceptionWhenStatusCodesMissingInRetryableTag() {

Retryable result = contextUnderTest.getRetryableTag();
assertAll(
() -> assertEquals(0, result.getDelay()),
() -> assertEquals(1000, result.getDelay()),
() -> assertEquals(1, result.getMaxAttempts()),
() -> assertEquals(ImmutableSet.of(), result.getStatusCodes())
() -> assertEquals(ImmutableSet.of(500,502,503,504), result.getStatusCodes()),
() -> assertEquals(ImmutableSet.of(java.net.ConnectException.class,java.net.SocketException.class,
javax.net.ssl.SSLException.class),
result.getRetryableExceptions())
);
}

Expand Down
Loading

0 comments on commit 46b067b

Please sign in to comment.