diff --git a/core/google/api/core/retry.py b/core/google/api/core/retry.py index b5a550faa584..fe85ce48cf1b 100644 --- a/core/google/api/core/retry.py +++ b/core/google/api/core/retry.py @@ -12,9 +12,52 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Helpers for retrying functions with exponential back-off.""" +"""Helpers for retrying functions with exponential back-off. + +The :cls:`Retry` decorator can be used to retry functions that raise exceptions +using exponential backoff. Because a exponential sleep algorithm is used, +the retry is limited by a `deadline`. The deadline is the maxmimum amount of +time a method can block. This is used instead of total number of retries +because it is difficult to ascertain the amount of time a function can block +when using total number of retries and exponential backoff. + +By default, this decorator will retry transient +API errors (see :func:`if_transient_error`). For example: + +.. code-block:: python + + @retry.Retry() + def call_flaky_rpc(): + return client.flaky_rpc() + + # Will retry flaky_rpc() if it raises transient API errors. + result = call_flaky_rpc() + +You can pass a custom predicate to retry on different exceptions, such as +waiting for an eventually consistent item to be available: + +.. code-block:: python + + @retry.Retry(predicate=if_exception_type(exceptions.NotFound)) + def check_if_exists(): + return client.does_thing_exist() + + is_available = check_if_exists() + +Some client library methods apply retry automatically. These methods can accept +a ``retry`` parameter that allows you to configure the behavior: + +.. code-block:: python + + my_retry = retry.Retry(deadline=60) + result = client.some_method(retry=my_retry) + +""" + +from __future__ import unicode_literals import datetime +import functools import logging import random import time @@ -25,7 +68,10 @@ from google.api.core.helpers import datetime_helpers _LOGGER = logging.getLogger(__name__) -_DEFAULT_MAX_JITTER = 0.2 +_DEFAULT_INITIAL_DELAY = 1.0 +_DEFAULT_MAXIMUM_DELAY = 60.0 +_DEFAULT_DELAY_MULTIPLIER = 2.0 +_DEFAULT_DEADLINE = 60.0 * 2.0 def if_exception_type(*exception_types): @@ -38,10 +84,10 @@ def if_exception_type(*exception_types): Callable[Exception]: A predicate that returns True if the provided exception is of the given type(s). """ - def inner(exception): + def if_exception_type_predicate(exception): """Bound predicate for checking an exception type.""" return isinstance(exception, exception_types) - return inner + return if_exception_type_predicate # pylint: disable=invalid-name @@ -64,7 +110,7 @@ def inner(exception): def exponential_sleep_generator( - initial, maximum, multiplier=2, jitter=_DEFAULT_MAX_JITTER): + initial, maximum, multiplier=_DEFAULT_DELAY_MULTIPLIER): """Generates sleep intervals based on the exponential back-off algorithm. This implements the `Truncated Exponential Back-off`_ algorithm. @@ -77,16 +123,16 @@ def exponential_sleep_generator( be greater than 0. maximum (float): The maximum about of time to delay. multiplier (float): The multiplier applied to the delay. - jitter (float): The maximum about of randomness to apply to the delay. Yields: float: successive sleep intervals. """ delay = initial while True: - yield delay - delay = min( - delay * multiplier + random.uniform(0, jitter), maximum) + # Introduce jitter by yielding a delay that is uniformly distributed + # to average out to the delay time. + yield min(random.uniform(0.0, delay * 2.0), maximum) + delay = delay * multiplier def retry_target(target, predicate, sleep_generator, deadline): @@ -146,3 +192,120 @@ def retry_target(target, predicate, sleep_generator, deadline): time.sleep(sleep) raise ValueError('Sleep generator stopped yielding sleep values.') + + +@six.python_2_unicode_compatible +class Retry(object): + """Exponential retry decorator. + + This class is a decorator used to add exponential back-off retry behavior + to an RPC call. + + Although the default behavior is to retry transient API errors, a + different predicate can be provided to retry other exceptions. + + Args: + predicate (Callable[Exception]): A callable that should return ``True`` + if the given exception is retryable. + initial (float): The minimum about of time to delay in seconds. This + must be greater than 0. + maximum (float): The maximum about of time to delay in seconds. + multiplier (float): The multiplier applied to the delay. + deadline (float): How long to keep retrying in seconds. + """ + def __init__( + self, + predicate=if_transient_error, + initial=_DEFAULT_INITIAL_DELAY, + maximum=_DEFAULT_MAXIMUM_DELAY, + multiplier=_DEFAULT_DELAY_MULTIPLIER, + deadline=_DEFAULT_DEADLINE): + self._predicate = predicate + self._initial = initial + self._multiplier = multiplier + self._maximum = maximum + self._deadline = deadline + + def __call__(self, func): + """Wrap a callable with retry behavior. + + Args: + func (Callable): The callable to add retry behavior to. + + Returns: + Callable: A callable that will invoke ``func`` with retry + behavior. + """ + @six.wraps(func) + def retry_wrapped_func(*args, **kwargs): + """A wrapper that calls target function with retry.""" + target = functools.partial(func, *args, **kwargs) + sleep_generator = exponential_sleep_generator( + self._initial, self._maximum, multiplier=self._multiplier) + return retry_target( + target, + self._predicate, + sleep_generator, + self._deadline) + + return retry_wrapped_func + + def with_deadline(self, deadline): + """Return a copy of this retry with the given deadline. + + Args: + deadline (float): How long to keep retrying. + + Returns: + Retry: A new retry instance with the given deadline. + """ + return Retry( + predicate=self._predicate, + initial=self._initial, + maximum=self._maximum, + multiplier=self._multiplier, + deadline=deadline) + + def with_predicate(self, predicate): + """Return a copy of this retry with the given predicate. + + Args: + predicate (Callable[Exception]): A callable that should return + ``True`` if the given exception is retryable. + + Returns: + Retry: A new retry instance with the given predicate. + """ + return Retry( + predicate=predicate, + initial=self._initial, + maximum=self._maximum, + multiplier=self._multiplier, + deadline=self._deadline) + + def with_delay( + self, initial=None, maximum=None, multiplier=None): + """Return a copy of this retry with the given delay options. + + Args: + initial (float): The minimum about of time to delay. This must + be greater than 0. + maximum (float): The maximum about of time to delay. + multiplier (float): The multiplier applied to the delay. + + Returns: + Retry: A new retry instance with the given predicate. + """ + return Retry( + predicate=self._predicate, + initial=initial if initial is not None else self._initial, + maximum=maximum if maximum is not None else self._maximum, + multiplier=multiplier if maximum is not None else self._multiplier, + deadline=self._deadline) + + def __str__(self): + return ( + ''.format( + self._predicate, self._initial, self._maximum, + self._multiplier, self._deadline)) diff --git a/core/setup.py b/core/setup.py index 96d7567b9de6..c45f7dd24ac2 100644 --- a/core/setup.py +++ b/core/setup.py @@ -60,6 +60,10 @@ 'tenacity >= 4.0.0, <5.0.0dev' ] +EXTRAS_REQUIREMENTS = { + ':python_version<"3.2"': ['futures >= 3.0.0'], +} + setup( name='google-cloud-core', version='0.26.0', @@ -72,5 +76,6 @@ ], packages=find_packages(exclude=('tests*',)), install_requires=REQUIREMENTS, + extras_require=EXTRAS_REQUIREMENTS, **SETUP_BASE ) diff --git a/core/tests/unit/api_core/test_retry.py b/core/tests/unit/api_core/test_retry.py index 5ad5612482dc..71569137b94f 100644 --- a/core/tests/unit/api_core/test_retry.py +++ b/core/tests/unit/api_core/test_retry.py @@ -14,6 +14,7 @@ import datetime import itertools +import re import mock import pytest @@ -43,29 +44,22 @@ def test_if_transient_error(): assert not retry.if_transient_error(exceptions.InvalidArgument('')) -def test_exponential_sleep_generator_base_2(): +# Make uniform return half of its maximum, which will be the calculated +# sleep time. +@mock.patch('random.uniform', autospec=True, side_effect=lambda m, n: n/2.0) +def test_exponential_sleep_generator_base_2(uniform): gen = retry.exponential_sleep_generator( - 1, 60, 2, jitter=0.0) + 1, 60, multiplier=2) result = list(itertools.islice(gen, 8)) assert result == [1, 2, 4, 8, 16, 32, 60, 60] -@mock.patch('random.uniform') -def test_exponential_sleep_generator_jitter(uniform): - uniform.return_value = 1 - gen = retry.exponential_sleep_generator( - 1, 60, 2, jitter=2.2) - - result = list(itertools.islice(gen, 7)) - assert result == [1, 3, 7, 15, 31, 60, 60] - uniform.assert_called_with(0.0, 2.2) - - -@mock.patch('time.sleep') +@mock.patch('time.sleep', autospec=True) @mock.patch( 'google.api.core.helpers.datetime_helpers.utcnow', - return_value=datetime.datetime.min) + return_value=datetime.datetime.min, + autospec=True) def test_retry_target_success(utcnow, sleep): predicate = retry.if_exception_type(ValueError) call_count = [0] @@ -83,10 +77,11 @@ def target(): sleep.assert_has_calls([mock.call(0), mock.call(1)]) -@mock.patch('time.sleep') +@mock.patch('time.sleep', autospec=True) @mock.patch( 'google.api.core.helpers.datetime_helpers.utcnow', - return_value=datetime.datetime.min) + return_value=datetime.datetime.min, + autospec=True) def test_retry_target_non_retryable_error(utcnow, sleep): predicate = retry.if_exception_type(ValueError) exception = TypeError() @@ -99,9 +94,9 @@ def test_retry_target_non_retryable_error(utcnow, sleep): sleep.assert_not_called() -@mock.patch('time.sleep') +@mock.patch('time.sleep', autospec=True) @mock.patch( - 'google.api.core.helpers.datetime_helpers.utcnow') + 'google.api.core.helpers.datetime_helpers.utcnow', autospec=True) def test_retry_target_deadline_exceeded(utcnow, sleep): predicate = retry.if_exception_type(ValueError) exception = ValueError('meep') @@ -127,3 +122,99 @@ def test_retry_target_bad_sleep_generator(): with pytest.raises(ValueError, match='Sleep generator'): retry.retry_target( mock.sentinel.target, mock.sentinel.predicate, [], None) + + +class TestRetry(object): + def test_constructor_defaults(self): + retry_ = retry.Retry() + assert retry_._predicate == retry.if_transient_error + assert retry_._initial == 1 + assert retry_._maximum == 60 + assert retry_._multiplier == 2 + assert retry_._deadline == 120 + + def test_constructor_options(self): + retry_ = retry.Retry( + predicate=mock.sentinel.predicate, + initial=1, + maximum=2, + multiplier=3, + deadline=4) + assert retry_._predicate == mock.sentinel.predicate + assert retry_._initial == 1 + assert retry_._maximum == 2 + assert retry_._multiplier == 3 + assert retry_._deadline == 4 + + def test_with_deadline(self): + retry_ = retry.Retry() + new_retry = retry_.with_deadline(42) + assert retry_ is not new_retry + assert new_retry._deadline == 42 + + def test_with_predicate(self): + retry_ = retry.Retry() + new_retry = retry_.with_predicate(mock.sentinel.predicate) + assert retry_ is not new_retry + assert new_retry._predicate == mock.sentinel.predicate + + def test_with_delay_noop(self): + retry_ = retry.Retry() + new_retry = retry_.with_delay() + assert retry_ is not new_retry + assert new_retry._initial == retry_._initial + assert new_retry._maximum == retry_._maximum + assert new_retry._multiplier == retry_._multiplier + + def test_with_delay(self): + retry_ = retry.Retry() + new_retry = retry_.with_delay( + initial=1, maximum=2, multiplier=3) + assert retry_ is not new_retry + assert new_retry._initial == 1 + assert new_retry._maximum == 2 + assert new_retry._multiplier == 3 + + def test___str__(self): + retry_ = retry.Retry() + assert re.match(( + r', ' + r'initial=1.0, maximum=60.0, multiplier=2.0, deadline=120.0>'), + str(retry_)) + + @mock.patch('time.sleep', autospec=True) + def test___call___and_execute_success(self, sleep): + retry_ = retry.Retry() + target = mock.Mock(spec=['__call__'], return_value=42) + # __name__ is needed by functools.partial. + target.__name__ = 'target' + + decorated = retry_(target) + target.assert_not_called() + + result = decorated('meep') + + assert result == 42 + target.assert_called_once_with('meep') + sleep.assert_not_called() + + # Make uniform return half of its maximum, which will be the calculated + # sleep time. + @mock.patch( + 'random.uniform', autospec=True, side_effect=lambda m, n: n/2.0) + @mock.patch('time.sleep', autospec=True) + def test___call___and_execute_retry(self, sleep, uniform): + retry_ = retry.Retry(predicate=retry.if_exception_type(ValueError)) + target = mock.Mock(spec=['__call__'], side_effect=[ValueError(), 42]) + # __name__ is needed by functools.partial. + target.__name__ = 'target' + + decorated = retry_(target) + target.assert_not_called() + + result = decorated('meep') + + assert result == 42 + assert target.call_count == 2 + target.assert_has_calls([mock.call('meep'), mock.call('meep')]) + sleep.assert_called_once_with(retry_._initial)