diff --git a/riptide-failsafe/README.md b/riptide-failsafe/README.md index c2cd87318..596b300e2 100644 --- a/riptide-failsafe/README.md +++ b/riptide-failsafe/README.md @@ -52,8 +52,8 @@ The failsafe plugin will not perform retries nor apply circuit breakers unless t Http.builder() .plugin(new FailsafePlugin(Executors.newScheduledThreadPool(20)) .withRetryPolicy(new RetryPolicy() - .retryOn(SocketTimeoutException.class) .withDelay(25, TimeUnit.MILLISECONDS) + .withDelay(new RetryAfterDelayFunction(clock)) .withMaxRetries(4)) .withCircuitBreaker(new CircuitBreaker() .withFailureThreshold(3, 10) @@ -63,8 +63,32 @@ Http.builder() ``` Please visit the [Failsafe readme](https://github.com/jhalterman/failsafe#readme) in order to see possible -configurations. Make sure you **check out -[zalando/failsafe-actuator](https://github.com/zalando/failsafe-actuator)** for a seemless integration of +configurations. + +**Beware** when using `retryOn` to retry conditionally on certain exception types. +You'll need to register `RetryException` in order for the `retry()` route to work: + +```java +new RetryPolicy() + .retryOn(SocketTimeoutException.class) + .retryOn(RetryException.class); +``` + +As of Failsafe version 1.1.0, it's now supported to dynamically compute delays using a custom function. +Riptide: Failsafe offers a special implementation that understands +[`Retry-After` (RFC 7231, section 7.1.3)](https://tools.ietf.org/html/rfc7231#section-7.1.3): + +```java +Http.builder() + .plugin(new FailsafePlugin(Executors.newScheduledThreadPool(20)) + .withRetryPolicy(new RetryPolicy() + .withDelay(25, TimeUnit.MILLISECONDS) + .withDelay(new RetryAfterDelayFunction(clock))) + .build(); +``` + +Make sure you **check out +[zalando/failsafe-actuator](https://github.com/zalando/failsafe-actuator)** for a seamless integration of Failsafe and Spring Boot: ```java @@ -74,7 +98,6 @@ private CircuitBreaker breaker; Http.builder() .plugin(new FailsafePlugin(Executors.newScheduledThreadPool(20)) .withRetryPolicy(new RetryPolicy() - .retryOn(SocketTimeoutException.class) .withDelay(25, TimeUnit.MILLISECONDS) .withMaxRetries(4)) .withCircuitBreaker(breaker)) diff --git a/riptide-failsafe/pom.xml b/riptide-failsafe/pom.xml index d9d25963c..44d871027 100644 --- a/riptide-failsafe/pom.xml +++ b/riptide-failsafe/pom.xml @@ -16,7 +16,7 @@ Client side response routing - 1.0.4 + 1.1.0 diff --git a/riptide-failsafe/src/main/java/org/zalando/riptide/failsafe/FailsafePlugin.java b/riptide-failsafe/src/main/java/org/zalando/riptide/failsafe/FailsafePlugin.java index 9b54eaeea..b44a0db00 100644 --- a/riptide-failsafe/src/main/java/org/zalando/riptide/failsafe/FailsafePlugin.java +++ b/riptide-failsafe/src/main/java/org/zalando/riptide/failsafe/FailsafePlugin.java @@ -4,7 +4,6 @@ import net.jodah.failsafe.Failsafe; import net.jodah.failsafe.RetryPolicy; import org.springframework.http.client.ClientHttpResponse; -import org.zalando.riptide.CancelableCompletableFuture; import org.zalando.riptide.Plugin; import org.zalando.riptide.RequestArguments; import org.zalando.riptide.RequestExecution; @@ -36,11 +35,7 @@ public FailsafePlugin(final ScheduledExecutorService scheduler) { } public FailsafePlugin withRetryPolicy(final RetryPolicy retryPolicy) { - return new FailsafePlugin(scheduler, withRetryExceptionSupport(retryPolicy), circuitBreaker); - } - - private RetryPolicy withRetryExceptionSupport(final RetryPolicy retryPolicy) { - return new RetryPolicy(retryPolicy).retryOn(RetryException.class); + return new FailsafePlugin(scheduler, retryPolicy, circuitBreaker); } public FailsafePlugin withCircuitBreaker(final CircuitBreaker circuitBreaker) { diff --git a/riptide-failsafe/src/main/java/org/zalando/riptide/failsafe/RetryAfterDelayFunction.java b/riptide-failsafe/src/main/java/org/zalando/riptide/failsafe/RetryAfterDelayFunction.java new file mode 100644 index 000000000..78ba52a05 --- /dev/null +++ b/riptide-failsafe/src/main/java/org/zalando/riptide/failsafe/RetryAfterDelayFunction.java @@ -0,0 +1,90 @@ +package org.zalando.riptide.failsafe; + +import lombok.extern.slf4j.Slf4j; +import net.jodah.failsafe.ExecutionContext; +import net.jodah.failsafe.RetryPolicy.DelayFunction; +import net.jodah.failsafe.util.Duration; +import org.zalando.riptide.HttpResponseException; + +import javax.annotation.Nullable; +import java.time.Clock; +import java.time.Instant; +import java.time.format.DateTimeParseException; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +import static java.lang.Long.parseLong; +import static java.time.Duration.between; +import static java.time.Instant.now; +import static java.time.format.DateTimeFormatter.RFC_1123_DATE_TIME; + +/** + * @see RFC 7231, section 7.1.3: Retry-After + */ +@Slf4j +public final class RetryAfterDelayFunction implements DelayFunction { + + private final Pattern digit = Pattern.compile("\\d"); + + private final Clock clock; + + public RetryAfterDelayFunction(final Clock clock) { + this.clock = clock; + } + + @Override + public Duration computeDelay(final Object result, final Throwable failure, final ExecutionContext context) { + return failure instanceof HttpResponseException ? computeDelay((HttpResponseException) failure) : null; + } + + @Nullable + private Duration computeDelay(final HttpResponseException failure) { + @Nullable final String retryAfter = failure.getResponseHeaders().getFirst("Retry-After"); + return retryAfter == null ? null : toDuration(parseDelay(retryAfter)); + } + + /** + * The value of this field can be either an HTTP-date or a number of seconds to delay after the response + * is received. + * + * Retry-After = HTTP-date / delay-seconds + * + * @param retryAfter non-null header value + * @return the parsed delay in seconds + */ + @Nullable + private Long parseDelay(final String retryAfter) { + return onlyDigits(retryAfter) ? + parseSeconds(retryAfter) : + secondsUntil(parseDate(retryAfter)); + } + + private boolean onlyDigits(final String s) { + return digit.matcher(s).matches(); + } + + private Long parseSeconds(final String retryAfter) { + return parseLong(retryAfter); + } + + @Nullable + private Instant parseDate(final String retryAfter) { + try { + return Instant.from(RFC_1123_DATE_TIME.parse(retryAfter)); + } catch (final DateTimeParseException e) { + log.warn("Received invalid 'Retry-After' header [{}]; will ignore it", retryAfter); + return null; + } + } + + @Nullable + private Long secondsUntil(@Nullable final Instant end) { + return end == null ? null : between(now(clock), end).getSeconds(); + } + + @Nullable + private Duration toDuration(@Nullable final Long seconds) { + return seconds == null ? null : new Duration(seconds, TimeUnit.SECONDS); + } + +} diff --git a/riptide-failsafe/src/main/java/org/zalando/riptide/failsafe/RetryException.java b/riptide-failsafe/src/main/java/org/zalando/riptide/failsafe/RetryException.java index e22073b12..0e40d77a6 100644 --- a/riptide-failsafe/src/main/java/org/zalando/riptide/failsafe/RetryException.java +++ b/riptide-failsafe/src/main/java/org/zalando/riptide/failsafe/RetryException.java @@ -5,7 +5,7 @@ import java.io.IOException; -final class RetryException extends HttpResponseException { +public final class RetryException extends HttpResponseException { RetryException(final ClientHttpResponse response) throws IOException { super("Retrying response", response); diff --git a/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/FailsafePluginTest.java b/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/FailsafePluginTest.java index 24ab700c4..72bcf6f77 100644 --- a/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/FailsafePluginTest.java +++ b/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/FailsafePluginTest.java @@ -54,7 +54,6 @@ public class FailsafePluginTest { .converter(createJsonConverter()) .plugin(new FailsafePlugin(newSingleThreadScheduledExecutor()) .withRetryPolicy(new RetryPolicy() - .retryOn(SocketTimeoutException.class) .withDelay(500, MILLISECONDS) .withMaxRetries(4)) .withCircuitBreaker(new CircuitBreaker() @@ -104,7 +103,7 @@ public void shouldRetryUnsuccessfully() throws Throwable { } @Test - public void shouldRetryOnDemand() throws Throwable { + public void shouldRetryOnDemand() { driver.addExpectation(onRequestTo("/baz"), giveEmptyResponse().withStatus(503)); driver.addExpectation(onRequestTo("/baz"), giveEmptyResponse()); diff --git a/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/RetryAfterDelayFunctionTest.java b/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/RetryAfterDelayFunctionTest.java new file mode 100644 index 000000000..e9ad44262 --- /dev/null +++ b/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/RetryAfterDelayFunctionTest.java @@ -0,0 +1,149 @@ +package org.zalando.riptide.failsafe; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.restdriver.clientdriver.ClientDriverRule; +import net.jodah.failsafe.CircuitBreaker; +import net.jodah.failsafe.RetryPolicy; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.springframework.http.HttpStatus; +import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.scheduling.concurrent.ConcurrentTaskExecutor; +import org.zalando.riptide.Http; +import org.zalando.riptide.httpclient.RestAsyncClientHttpRequestFactory; + +import java.io.IOException; +import java.time.Clock; +import java.util.concurrent.TimeUnit; + +import static com.github.restdriver.clientdriver.RestClientDriver.giveEmptyResponse; +import static com.github.restdriver.clientdriver.RestClientDriver.onRequestTo; +import static java.time.Instant.parse; +import static java.time.ZoneOffset.UTC; +import static java.util.concurrent.Executors.newSingleThreadExecutor; +import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.springframework.http.HttpStatus.Series.SUCCESSFUL; +import static org.zalando.riptide.Bindings.anySeries; +import static org.zalando.riptide.Bindings.on; +import static org.zalando.riptide.Navigators.series; +import static org.zalando.riptide.Navigators.status; +import static org.zalando.riptide.PassRoute.pass; +import static org.zalando.riptide.failsafe.RetryRoute.retry; + +public class RetryAfterDelayFunctionTest { + + @Rule + public final ClientDriverRule driver = new ClientDriverRule(); + + private final CloseableHttpClient client = HttpClientBuilder.create() + .setDefaultRequestConfig(RequestConfig.custom() + .setSocketTimeout(1000) + .build()) + .build(); + + private final Clock clock = Clock.fixed(parse("2018-04-11T22:34:27Z"), UTC); + + private final Http unit = Http.builder() + .baseUrl(driver.getBaseUrl()) + .requestFactory(new RestAsyncClientHttpRequestFactory(client, + new ConcurrentTaskExecutor(newSingleThreadExecutor()))) + .converter(createJsonConverter()) + .plugin(new FailsafePlugin(newSingleThreadScheduledExecutor()) + .withRetryPolicy(new RetryPolicy() + .withDelay(2, SECONDS) + .withDelay(new RetryAfterDelayFunction(clock)) + .withMaxRetries(4)) + .withCircuitBreaker(new CircuitBreaker() + .withFailureThreshold(3, 10) + .withSuccessThreshold(5) + .withDelay(1, TimeUnit.MINUTES))) + .build(); + + private static MappingJackson2HttpMessageConverter createJsonConverter() { + final MappingJackson2HttpMessageConverter converter = new MappingJackson2HttpMessageConverter(); + converter.setObjectMapper(createObjectMapper()); + return converter; + } + + private static ObjectMapper createObjectMapper() { + return new ObjectMapper().findAndRegisterModules() + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + } + + @After + public void tearDown() throws IOException { + client.close(); + } + + @Test + public void shouldRetryWithoutDynamicDelay() { + driver.addExpectation(onRequestTo("/baz"), giveEmptyResponse().withStatus(503)); + driver.addExpectation(onRequestTo("/baz"), giveEmptyResponse()); + + unit.get("/baz") + .dispatch(series(), + on(SUCCESSFUL).call(pass()), + anySeries().dispatch(status(), + on(HttpStatus.SERVICE_UNAVAILABLE).call(retry()))) + .join(); + } + + @Test + public void shouldIgnoreDynamicDelayOnInvalidFormat() { + driver.addExpectation(onRequestTo("/baz"), giveEmptyResponse().withStatus(503) + .withHeader("Retry-After", "2018-04-11T22:34:28Z")); // should've been HTTP date + driver.addExpectation(onRequestTo("/baz"), giveEmptyResponse()); + + unit.get("/baz") + .dispatch(series(), + on(SUCCESSFUL).call(pass()), + anySeries().dispatch(status(), + on(HttpStatus.SERVICE_UNAVAILABLE).call(retry()))) + .join(); + } + + @Test(timeout = 1500) + public void shouldRetryOnDemandWithDynamicDelay() { + driver.addExpectation(onRequestTo("/baz"), giveEmptyResponse().withStatus(503) + .withHeader("Retry-After", "1")); + driver.addExpectation(onRequestTo("/baz"), giveEmptyResponse()); + + unit.get("/baz") + .dispatch(series(), + on(SUCCESSFUL).call(pass()), + anySeries().dispatch(status(), + on(HttpStatus.SERVICE_UNAVAILABLE).call(retry()))) + .join(); + } + + @Test(timeout = 1500) + public void shouldRetryWithDynamicDelay() { + driver.addExpectation(onRequestTo("/baz"), giveEmptyResponse().withStatus(503) + .withHeader("Retry-After", "1")); + driver.addExpectation(onRequestTo("/baz"), giveEmptyResponse()); + + unit.get("/baz") + .dispatch(series(), + on(SUCCESSFUL).call(pass())) + .join(); + } + + @Test(timeout = 1500) + public void shouldRetryWithDynamicDelayDate() { + driver.addExpectation(onRequestTo("/baz"), giveEmptyResponse().withStatus(503) + .withHeader("Retry-After", "Wed, 11 Apr 2018 22:34:28 GMT")); + driver.addExpectation(onRequestTo("/baz"), giveEmptyResponse()); + + unit.get("/baz") + .dispatch(series(), + on(SUCCESSFUL).call(pass())) + .join(); + } + +} diff --git a/riptide-spring-boot-starter/src/main/java/org/zalando/riptide/spring/AccessTokensFactoryBean.java b/riptide-spring-boot-starter/src/main/java/org/zalando/riptide/spring/AccessTokensFactoryBean.java index 02817c763..969155cbf 100644 --- a/riptide-spring-boot-starter/src/main/java/org/zalando/riptide/spring/AccessTokensFactoryBean.java +++ b/riptide-spring-boot-starter/src/main/java/org/zalando/riptide/spring/AccessTokensFactoryBean.java @@ -22,7 +22,6 @@ final class AccessTokensFactoryBean extends AbstractFactoryBean { private AccessTokensBuilder builder; AccessTokensFactoryBean(final RiptideSettings settings) { - final Defaults defaults = settings.getDefaults(); final GlobalOAuth oAuth = settings.getOauth(); final URI accessTokenUrl = getAccessTokenUrl(oAuth); @@ -74,7 +73,7 @@ private URI getAccessTokenUrl(final GlobalOAuth oauth) { } @Override - protected AccessTokens createInstance() throws Exception { + protected AccessTokens createInstance() { return builder.start(); } @@ -84,7 +83,7 @@ public Class getObjectType() { } @Override - protected void destroyInstance(final AccessTokens tokens) throws Exception { + protected void destroyInstance(final AccessTokens tokens) { tokens.stop(); } diff --git a/riptide-spring-boot-starter/src/main/java/org/zalando/riptide/spring/CircuitBreakerFactoryBean.java b/riptide-spring-boot-starter/src/main/java/org/zalando/riptide/spring/CircuitBreakerFactoryBean.java index dd60f86a0..cf34f69f6 100644 --- a/riptide-spring-boot-starter/src/main/java/org/zalando/riptide/spring/CircuitBreakerFactoryBean.java +++ b/riptide-spring-boot-starter/src/main/java/org/zalando/riptide/spring/CircuitBreakerFactoryBean.java @@ -25,7 +25,7 @@ public void setConfiguration(final RiptideSettings.CircuitBreaker config) { } @Override - public CircuitBreaker getObject() throws Exception { + public CircuitBreaker getObject() { return circuitBreaker; } diff --git a/riptide-spring-boot-starter/src/main/java/org/zalando/riptide/spring/RetryPolicyFactoryBean.java b/riptide-spring-boot-starter/src/main/java/org/zalando/riptide/spring/RetryPolicyFactoryBean.java index f2489b246..8a5f1f336 100644 --- a/riptide-spring-boot-starter/src/main/java/org/zalando/riptide/spring/RetryPolicyFactoryBean.java +++ b/riptide-spring-boot-starter/src/main/java/org/zalando/riptide/spring/RetryPolicyFactoryBean.java @@ -2,6 +2,8 @@ import net.jodah.failsafe.RetryPolicy; import org.springframework.beans.factory.FactoryBean; +import org.zalando.riptide.failsafe.RetryAfterDelayFunction; +import org.zalando.riptide.failsafe.RetryException; import org.zalando.riptide.faults.TransientFaultException; import org.zalando.riptide.spring.RiptideSettings.Retry; @@ -9,6 +11,7 @@ import java.util.Optional; import java.util.concurrent.TimeUnit; +import static java.time.Clock.systemUTC; import static java.util.concurrent.TimeUnit.MILLISECONDS; final class RetryPolicyFactoryBean implements FactoryBean { @@ -45,6 +48,8 @@ public void setConfiguration(final Retry config) { .ifPresent(jitter -> jitter.applyTo(retryPolicy::withJitter)); retryPolicy.retryOn(TransientFaultException.class); + retryPolicy.retryOn(RetryException.class); + retryPolicy.withDelay(new RetryAfterDelayFunction(systemUTC())); } @Override