From e1902c4a4a2c0d50b57e52e677d3edd820b6a068 Mon Sep 17 00:00:00 2001 From: Alexandre Sieira Date: Fri, 3 Mar 2017 21:42:59 -0300 Subject: [PATCH] RETRY options to ignore statuses and use try MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Based on https://github.com/hadley/httr/issues/404 and some observations of my own I humbly submit this pull request that adds two new arguments to `RETRY`. `safe_statuses` is an integer vector of HTTP status codes that should not trigger a retry, overruling the result of a call to `http_error`. `use_try` is meant to handle the cases where curl signals an error (i.e., with `stop`) for transient situations such as a DNS lookup failure or a connection error. If this is set to `TRUE` then `RETRY` will call `try(request_perform(..))` and retry the operation if an error is signaled. Both parameters have sensible default values so that this change doesn’t introduce breaking changes and RETRY behaves as it did before if they are not overridden. --- R/retry.R | 39 +++++++++++++++++++++++++++++++++------ man/RETRY.Rd | 16 +++++++++++++--- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/R/retry.R b/R/retry.R index c26663c2..8dc27364 100644 --- a/R/retry.R +++ b/R/retry.R @@ -16,7 +16,13 @@ #' \code{pause_cap} seconds. #' @param quiet If \code{FALSE}, will print a message displaying how long #' until the next request. -#' @return The last response. Note that if the request doesn't succeed after +#' @param use_try If \code{TRUE}, will wrap the request execution in a call to +#' \code{\link[base]{try}()} in order to catch lower level errors in the +#' request. +#' @param safe_statuses Integer vector of status codes that should be considered +#' safe and not cause a retry. +#' @return The last response or, if \code{use_try} was \code{TRUE}, an invisible +#' object of class \code{"try-error"}. Note that if the request doesn't succeed after #' \code{times} times this will be a failed request, i.e. you still need #' to use \code{\link{stop_for_status}()}. #' @export @@ -25,29 +31,50 @@ #' RETRY("GET", "http://httpbin.org/status/200") #' # Never succeeds #' RETRY("GET", "http://httpbin.org/status/500") +#' # Returns immediately with a 404 response +#' RETRY("GET", "http://httpbin.org/status/404", safe_statuses = 400L) RETRY <- function(verb, url = NULL, config = list(), ..., body = NULL, encode = c("multipart", "form", "json", "raw"), times = 3, pause_base = 1, pause_cap = 60, - handle = NULL, quiet = FALSE) { + handle = NULL, quiet = FALSE, use_try = FALSE, safe_statuses = integer()) { stopifnot(is.numeric(times), length(times) == 1L) stopifnot(is.numeric(pause_base), length(pause_base) == 1L) stopifnot(is.numeric(pause_cap), length(pause_cap) == 1L) + stopifnot(is.logical(use_try), length(use_try) == 1L, !is.na(use_try)) + stopifnot(is.integer(safe_statuses)) hu <- handle_url(handle, url, ...) req <- request_build(verb, hu$url, body_config(body, match.arg(encode)), config, ...) - resp <- request_perform(req, hu$handle$handle) + resp <- if (use_try) { + try(request_perform(req, hu$handle$handle)) + } else { + request_perform(req, hu$handle$handle) + } i <- 1 - while (i < times && http_error(resp)) { - backoff_full_jitter(i, status_code(resp), pause_base, pause_cap, quiet = quiet) + while (i < times && should_retry(resp, safe_statuses)) { + status = if ("try-error" %in% class(resp)) { + attr(resp, "condition")$message + } else { + status_code(resp) + } + backoff_full_jitter(i, status, pause_base, pause_cap, quiet = quiet) i <- i + 1 - resp <- request_perform(req, hu$handle$handle) + resp <- if (use_try) { + try(request_perform(req, hu$handle$handle)) + } else { + request_perform(req, hu$handle$handle) + } } resp } +should_retry <- function(resp, safe_statuses) { + "try-error" %in% class(resp) || (!status_code(resp) %in% safe_statuses && http_error(resp)) +} + backoff_full_jitter <- function(i, status, pause_base = 1, pause_cap = 60, quiet = FALSE) { length <- ceiling(stats::runif(1, max = min(pause_cap, pause_base * (2 ^ i)))) if (!quiet) { diff --git a/man/RETRY.Rd b/man/RETRY.Rd index 301b761c..3c559623 100644 --- a/man/RETRY.Rd +++ b/man/RETRY.Rd @@ -6,7 +6,8 @@ \usage{ RETRY(verb, url = NULL, config = list(), ..., body = NULL, encode = c("multipart", "form", "json", "raw"), times = 3, - pause_base = 1, pause_cap = 60, handle = NULL, quiet = FALSE) + pause_base = 1, pause_cap = 60, handle = NULL, quiet = FALSE, + use_try = FALSE, safe_statuses = integer()) } \arguments{ \item{verb}{Name of verb to use.} @@ -67,9 +68,17 @@ details.} \item{quiet}{If \code{FALSE}, will print a message displaying how long until the next request.} + +\item{use_try}{If \code{TRUE}, will wrap the request execution in a call to +\code{\link[base]{try}()} in order to catch lower level errors in the +request.} + +\item{safe_statuses}{Integer vector of status codes that should be considered +safe and not cause a retry.} } \value{ -The last response. Note that if the request doesn't succeed after +The last response or, if \code{use_try} was \code{TRUE}, an invisible + object of class \code{"try-error"}. Note that if the request doesn't succeed after \code{times} times this will be a failed request, i.e. you still need to use \code{\link{stop_for_status}()}. } @@ -85,5 +94,6 @@ backoff with jitter, using the approach outlined in RETRY("GET", "http://httpbin.org/status/200") # Never succeeds RETRY("GET", "http://httpbin.org/status/500") +# Returns immediately with a 404 response +RETRY("GET", "http://httpbin.org/status/404", safe_statuses = 400L) } -