diff options
author | uzhas <uzhas@ydb.tech> | 2024-06-19 20:06:32 +0300 |
---|---|---|
committer | uzhas <uzhas@ydb.tech> | 2024-06-19 20:16:54 +0300 |
commit | 93108b981df25d1adbac948754f5955075eedcc8 (patch) | |
tree | be8163ba74d6ce0f360424d777702174e354db3c /contrib | |
parent | 174edaf3b2e14c1164dcbe616906a6b8f62e96ae (diff) | |
download | ydb-93108b981df25d1adbac948754f5955075eedcc8.tar.gz |
add retry py lib
e9b0291ce8188afa295b944f29ce6efbb38bcd92
Diffstat (limited to 'contrib')
27 files changed, 1538 insertions, 0 deletions
diff --git a/contrib/python/retry/py2/.dist-info/METADATA b/contrib/python/retry/py2/.dist-info/METADATA new file mode 100644 index 0000000000..c5986d2fea --- /dev/null +++ b/contrib/python/retry/py2/.dist-info/METADATA @@ -0,0 +1,184 @@ +Metadata-Version: 2.0 +Name: retry +Version: 0.9.2 +Summary: Easy to use retry decorator. +Home-page: https://github.com/invl/retry +Author: invl +Author-email: invlpg@gmail.com +License: Apache License 2.0 +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Natural Language :: English +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development +Requires-Dist: decorator (>=3.4.2) +Requires-Dist: py (<2.0.0,>=1.4.26) + +retry +===== + +.. image:: https://pypip.in/d/retry/badge.png + :target: https://pypi.python.org/pypi/retry/ + +.. image:: https://pypip.in/v/retry/badge.png + :target: https://pypi.python.org/pypi/retry/ + +.. image:: https://pypip.in/license/retry/badge.png + :target: https://pypi.python.org/pypi/retry/ + + +Easy to use retry decorator. + + +Features +-------- + +- No external dependency (stdlib only). +- (Optionally) Preserve function signatures (`pip install decorator`). +- Original traceback, easy to debug. + + +Installation +------------ + +.. code-block:: bash + + $ pip install retry + + +API +--- + +retry decorator +^^^^^^^^^^^^^^^ + +.. code:: python + + def retry(exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1, jitter=0, logger=logging_logger): + """Return a retry decorator. + + :param exceptions: an exception or a tuple of exceptions to catch. default: Exception. + :param tries: the maximum number of attempts. default: -1 (infinite). + :param delay: initial delay between attempts. default: 0. + :param max_delay: the maximum value of delay. default: None (no limit). + :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff). + :param jitter: extra seconds added to delay between attempts. default: 0. + fixed if a number, random if a range tuple (min, max) + :param logger: logger.warning(fmt, error, delay) will be called on failed attempts. + default: retry.logging_logger. if None, logging is disabled. + """ + +Various retrying logic can be achieved by combination of arguments. + + +Examples +"""""""" + +.. code:: python + + from retry import retry + +.. code:: python + + @retry() + def make_trouble(): + '''Retry until succeed''' + +.. code:: python + + @retry(ZeroDivisionError, tries=3, delay=2) + def make_trouble(): + '''Retry on ZeroDivisionError, raise error after 3 attempts, sleep 2 seconds between attempts.''' + +.. code:: python + + @retry((ValueError, TypeError), delay=1, backoff=2) + def make_trouble(): + '''Retry on ValueError or TypeError, sleep 1, 2, 4, 8, ... seconds between attempts.''' + +.. code:: python + + @retry((ValueError, TypeError), delay=1, backoff=2, max_delay=4) + def make_trouble(): + '''Retry on ValueError or TypeError, sleep 1, 2, 4, 4, ... seconds between attempts.''' + +.. code:: python + + @retry(ValueError, delay=1, jitter=1) + def make_trouble(): + '''Retry on ValueError, sleep 1, 2, 3, 4, ... seconds between attempts.''' + +.. code:: python + + # If you enable logging, you can get warnings like 'ValueError, retrying in + # 1 seconds' + if __name__ == '__main__': + import logging + logging.basicConfig() + make_trouble() + +retry_call +^^^^^^^^^^ + +.. code:: python + + def retry_call(f, fargs=None, fkwargs=None, exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1, + jitter=0, + logger=logging_logger): + """ + Calls a function and re-executes it if it failed. + + :param f: the function to execute. + :param fargs: the positional arguments of the function to execute. + :param fkwargs: the named arguments of the function to execute. + :param exceptions: an exception or a tuple of exceptions to catch. default: Exception. + :param tries: the maximum number of attempts. default: -1 (infinite). + :param delay: initial delay between attempts. default: 0. + :param max_delay: the maximum value of delay. default: None (no limit). + :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff). + :param jitter: extra seconds added to delay between attempts. default: 0. + fixed if a number, random if a range tuple (min, max) + :param logger: logger.warning(fmt, error, delay) will be called on failed attempts. + default: retry.logging_logger. if None, logging is disabled. + :returns: the result of the f function. + """ + +This is very similar to the decorator, except that it takes a function and its arguments as parameters. The use case behind it is to be able to dynamically adjust the retry arguments. + +.. code:: python + + import requests + + from retry.api import retry_call + + + def make_trouble(service, info=None): + if not info: + info = '' + r = requests.get(service + info) + return r.text + + + def what_is_my_ip(approach=None): + if approach == "optimistic": + tries = 1 + elif approach == "conservative": + tries = 3 + else: + # skeptical + tries = -1 + result = retry_call(make_trouble, fargs=["http://ipinfo.io/"], fkwargs={"info": "ip"}, tries=tries) + print(result) + + what_is_my_ip("conservative") + + + diff --git a/contrib/python/retry/py2/.dist-info/top_level.txt b/contrib/python/retry/py2/.dist-info/top_level.txt new file mode 100644 index 0000000000..77428f7b73 --- /dev/null +++ b/contrib/python/retry/py2/.dist-info/top_level.txt @@ -0,0 +1 @@ +retry diff --git a/contrib/python/retry/py2/AUTHORS b/contrib/python/retry/py2/AUTHORS new file mode 100644 index 0000000000..573ff53561 --- /dev/null +++ b/contrib/python/retry/py2/AUTHORS @@ -0,0 +1,8 @@ +Richard O'Dwyer <richard@richard.do> +Rémy <remy.greinhofer@gmail.com> +Rémy <rgreinho@users.noreply.github.com> +Rémy Greinhofer <remy.greinhofer@livelovely.com> +invl <invlpg@gmail.com> +invlpg <invlpg@gmail.com> +invlpg@gmail.com <invlpg@gmail.com> +williara <ray@hudl.com> diff --git a/contrib/python/retry/py2/LICENSE b/contrib/python/retry/py2/LICENSE new file mode 100644 index 0000000000..3a2cc3260e --- /dev/null +++ b/contrib/python/retry/py2/LICENSE @@ -0,0 +1,13 @@ +Copyright 2014 invl + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/contrib/python/retry/py2/README.rst b/contrib/python/retry/py2/README.rst new file mode 100644 index 0000000000..e8f78b15e8 --- /dev/null +++ b/contrib/python/retry/py2/README.rst @@ -0,0 +1,160 @@ +retry +===== + +.. image:: https://pypip.in/d/retry/badge.png + :target: https://pypi.python.org/pypi/retry/ + +.. image:: https://pypip.in/v/retry/badge.png + :target: https://pypi.python.org/pypi/retry/ + +.. image:: https://pypip.in/license/retry/badge.png + :target: https://pypi.python.org/pypi/retry/ + + +Easy to use retry decorator. + + +Features +-------- + +- No external dependency (stdlib only). +- (Optionally) Preserve function signatures (`pip install decorator`). +- Original traceback, easy to debug. + + +Installation +------------ + +.. code-block:: bash + + $ pip install retry + + +API +--- + +retry decorator +^^^^^^^^^^^^^^^ + +.. code:: python + + def retry(exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1, jitter=0, logger=logging_logger): + """Return a retry decorator. + + :param exceptions: an exception or a tuple of exceptions to catch. default: Exception. + :param tries: the maximum number of attempts. default: -1 (infinite). + :param delay: initial delay between attempts. default: 0. + :param max_delay: the maximum value of delay. default: None (no limit). + :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff). + :param jitter: extra seconds added to delay between attempts. default: 0. + fixed if a number, random if a range tuple (min, max) + :param logger: logger.warning(fmt, error, delay) will be called on failed attempts. + default: retry.logging_logger. if None, logging is disabled. + """ + +Various retrying logic can be achieved by combination of arguments. + + +Examples +"""""""" + +.. code:: python + + from retry import retry + +.. code:: python + + @retry() + def make_trouble(): + '''Retry until succeed''' + +.. code:: python + + @retry(ZeroDivisionError, tries=3, delay=2) + def make_trouble(): + '''Retry on ZeroDivisionError, raise error after 3 attempts, sleep 2 seconds between attempts.''' + +.. code:: python + + @retry((ValueError, TypeError), delay=1, backoff=2) + def make_trouble(): + '''Retry on ValueError or TypeError, sleep 1, 2, 4, 8, ... seconds between attempts.''' + +.. code:: python + + @retry((ValueError, TypeError), delay=1, backoff=2, max_delay=4) + def make_trouble(): + '''Retry on ValueError or TypeError, sleep 1, 2, 4, 4, ... seconds between attempts.''' + +.. code:: python + + @retry(ValueError, delay=1, jitter=1) + def make_trouble(): + '''Retry on ValueError, sleep 1, 2, 3, 4, ... seconds between attempts.''' + +.. code:: python + + # If you enable logging, you can get warnings like 'ValueError, retrying in + # 1 seconds' + if __name__ == '__main__': + import logging + logging.basicConfig() + make_trouble() + +retry_call +^^^^^^^^^^ + +.. code:: python + + def retry_call(f, fargs=None, fkwargs=None, exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1, + jitter=0, + logger=logging_logger): + """ + Calls a function and re-executes it if it failed. + + :param f: the function to execute. + :param fargs: the positional arguments of the function to execute. + :param fkwargs: the named arguments of the function to execute. + :param exceptions: an exception or a tuple of exceptions to catch. default: Exception. + :param tries: the maximum number of attempts. default: -1 (infinite). + :param delay: initial delay between attempts. default: 0. + :param max_delay: the maximum value of delay. default: None (no limit). + :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff). + :param jitter: extra seconds added to delay between attempts. default: 0. + fixed if a number, random if a range tuple (min, max) + :param logger: logger.warning(fmt, error, delay) will be called on failed attempts. + default: retry.logging_logger. if None, logging is disabled. + :returns: the result of the f function. + """ + +This is very similar to the decorator, except that it takes a function and its arguments as parameters. The use case behind it is to be able to dynamically adjust the retry arguments. + +.. code:: python + + import requests + + from retry.api import retry_call + + + def make_trouble(service, info=None): + if not info: + info = '' + r = requests.get(service + info) + return r.text + + + def what_is_my_ip(approach=None): + if approach == "optimistic": + tries = 1 + elif approach == "conservative": + tries = 3 + else: + # skeptical + tries = -1 + result = retry_call(make_trouble, fargs=["http://ipinfo.io/"], fkwargs={"info": "ip"}, tries=tries) + print(result) + + what_is_my_ip("conservative") + + + diff --git a/contrib/python/retry/py2/patches/01-return-retry_call.patch b/contrib/python/retry/py2/patches/01-return-retry_call.patch new file mode 100644 index 0000000000..a683760a33 --- /dev/null +++ b/contrib/python/retry/py2/patches/01-return-retry_call.patch @@ -0,0 +1,13 @@ +--- contrib/python/retry/py2/retry/__init__.py (index) ++++ contrib/python/retry/py2/retry/__init__.py (working tree) +@@ -1,8 +1,8 @@ +-__all__ = ['retry'] ++__all__ = ['retry', 'retry_call'] + + import logging + +-from .api import retry ++from .api import retry, retry_call + + + # Set default logging handler to avoid "No handler found" warnings. diff --git a/contrib/python/retry/py2/retry/__init__.py b/contrib/python/retry/py2/retry/__init__.py new file mode 100644 index 0000000000..1dd5705356 --- /dev/null +++ b/contrib/python/retry/py2/retry/__init__.py @@ -0,0 +1,18 @@ +__all__ = ['retry', 'retry_call'] + +import logging + +from .api import retry, retry_call + + +# Set default logging handler to avoid "No handler found" warnings. +try: # Python 2.7+ + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + + def emit(self, record): + pass + +log = logging.getLogger(__name__) +log.addHandler(NullHandler()) diff --git a/contrib/python/retry/py2/retry/api.py b/contrib/python/retry/py2/retry/api.py new file mode 100644 index 0000000000..245e4e4c08 --- /dev/null +++ b/contrib/python/retry/py2/retry/api.py @@ -0,0 +1,101 @@ +import logging +import random +import time + +from functools import partial + +from retry.compat import decorator + + +logging_logger = logging.getLogger(__name__) + + +def __retry_internal(f, exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1, jitter=0, + logger=logging_logger): + """ + Executes a function and retries it if it failed. + + :param f: the function to execute. + :param exceptions: an exception or a tuple of exceptions to catch. default: Exception. + :param tries: the maximum number of attempts. default: -1 (infinite). + :param delay: initial delay between attempts. default: 0. + :param max_delay: the maximum value of delay. default: None (no limit). + :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff). + :param jitter: extra seconds added to delay between attempts. default: 0. + fixed if a number, random if a range tuple (min, max) + :param logger: logger.warning(fmt, error, delay) will be called on failed attempts. + default: retry.logging_logger. if None, logging is disabled. + :returns: the result of the f function. + """ + _tries, _delay = tries, delay + while _tries: + try: + return f() + except exceptions as e: + _tries -= 1 + if not _tries: + raise + + if logger is not None: + logger.warning('%s, retrying in %s seconds...', e, _delay) + + time.sleep(_delay) + _delay *= backoff + + if isinstance(jitter, tuple): + _delay += random.uniform(*jitter) + else: + _delay += jitter + + if max_delay is not None: + _delay = min(_delay, max_delay) + + +def retry(exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1, jitter=0, logger=logging_logger): + """Returns a retry decorator. + + :param exceptions: an exception or a tuple of exceptions to catch. default: Exception. + :param tries: the maximum number of attempts. default: -1 (infinite). + :param delay: initial delay between attempts. default: 0. + :param max_delay: the maximum value of delay. default: None (no limit). + :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff). + :param jitter: extra seconds added to delay between attempts. default: 0. + fixed if a number, random if a range tuple (min, max) + :param logger: logger.warning(fmt, error, delay) will be called on failed attempts. + default: retry.logging_logger. if None, logging is disabled. + :returns: a retry decorator. + """ + + @decorator + def retry_decorator(f, *fargs, **fkwargs): + args = fargs if fargs else list() + kwargs = fkwargs if fkwargs else dict() + return __retry_internal(partial(f, *args, **kwargs), exceptions, tries, delay, max_delay, backoff, jitter, + logger) + + return retry_decorator + + +def retry_call(f, fargs=None, fkwargs=None, exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1, + jitter=0, + logger=logging_logger): + """ + Calls a function and re-executes it if it failed. + + :param f: the function to execute. + :param fargs: the positional arguments of the function to execute. + :param fkwargs: the named arguments of the function to execute. + :param exceptions: an exception or a tuple of exceptions to catch. default: Exception. + :param tries: the maximum number of attempts. default: -1 (infinite). + :param delay: initial delay between attempts. default: 0. + :param max_delay: the maximum value of delay. default: None (no limit). + :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff). + :param jitter: extra seconds added to delay between attempts. default: 0. + fixed if a number, random if a range tuple (min, max) + :param logger: logger.warning(fmt, error, delay) will be called on failed attempts. + default: retry.logging_logger. if None, logging is disabled. + :returns: the result of the f function. + """ + args = fargs if fargs else list() + kwargs = fkwargs if fkwargs else dict() + return __retry_internal(partial(f, *args, **kwargs), exceptions, tries, delay, max_delay, backoff, jitter, logger) diff --git a/contrib/python/retry/py2/retry/compat.py b/contrib/python/retry/py2/retry/compat.py new file mode 100644 index 0000000000..f39510d171 --- /dev/null +++ b/contrib/python/retry/py2/retry/compat.py @@ -0,0 +1,18 @@ +import functools + + +try: + from decorator import decorator +except ImportError: + def decorator(caller): + """ Turns caller into a decorator. + Unlike decorator module, function signature is not preserved. + + :param caller: caller(f, *args, **kwargs) + """ + def decor(f): + @functools.wraps(f) + def wrapper(*args, **kwargs): + return caller(f, *args, **kwargs) + return wrapper + return decor diff --git a/contrib/python/retry/py2/retry/tests/__init__.py b/contrib/python/retry/py2/retry/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/contrib/python/retry/py2/retry/tests/__init__.py diff --git a/contrib/python/retry/py2/retry/tests/test_retry.py b/contrib/python/retry/py2/retry/tests/test_retry.py new file mode 100644 index 0000000000..64f45cd89d --- /dev/null +++ b/contrib/python/retry/py2/retry/tests/test_retry.py @@ -0,0 +1,185 @@ +try: + from unittest.mock import create_autospec +except ImportError: + from mock import create_autospec + +try: + from unittest.mock import MagicMock +except ImportError: + from mock import MagicMock + +import time + +import pytest + +from retry.api import retry_call +from retry.api import retry + + +def test_retry(monkeypatch): + mock_sleep_time = [0] + + def mock_sleep(seconds): + mock_sleep_time[0] += seconds + + monkeypatch.setattr(time, 'sleep', mock_sleep) + + hit = [0] + + tries = 5 + delay = 1 + backoff = 2 + + @retry(tries=tries, delay=delay, backoff=backoff) + def f(): + hit[0] += 1 + 1 / 0 + + with pytest.raises(ZeroDivisionError): + f() + assert hit[0] == tries + assert mock_sleep_time[0] == sum( + delay * backoff ** i for i in range(tries - 1)) + + +def test_tries_inf(): + hit = [0] + target = 10 + + @retry(tries=float('inf')) + def f(): + hit[0] += 1 + if hit[0] == target: + return target + else: + raise ValueError + assert f() == target + + +def test_tries_minus1(): + hit = [0] + target = 10 + + @retry(tries=-1) + def f(): + hit[0] += 1 + if hit[0] == target: + return target + else: + raise ValueError + assert f() == target + + +def test_max_delay(monkeypatch): + mock_sleep_time = [0] + + def mock_sleep(seconds): + mock_sleep_time[0] += seconds + + monkeypatch.setattr(time, 'sleep', mock_sleep) + + hit = [0] + + tries = 5 + delay = 1 + backoff = 2 + max_delay = delay # Never increase delay + + @retry(tries=tries, delay=delay, max_delay=max_delay, backoff=backoff) + def f(): + hit[0] += 1 + 1 / 0 + + with pytest.raises(ZeroDivisionError): + f() + assert hit[0] == tries + assert mock_sleep_time[0] == delay * (tries - 1) + + +def test_fixed_jitter(monkeypatch): + mock_sleep_time = [0] + + def mock_sleep(seconds): + mock_sleep_time[0] += seconds + + monkeypatch.setattr(time, 'sleep', mock_sleep) + + hit = [0] + + tries = 10 + jitter = 1 + + @retry(tries=tries, jitter=jitter) + def f(): + hit[0] += 1 + 1 / 0 + + with pytest.raises(ZeroDivisionError): + f() + assert hit[0] == tries + assert mock_sleep_time[0] == sum(range(tries - 1)) + + +def test_retry_call(): + f_mock = MagicMock(side_effect=RuntimeError) + tries = 2 + try: + retry_call(f_mock, exceptions=RuntimeError, tries=tries) + except RuntimeError: + pass + + assert f_mock.call_count == tries + + +def test_retry_call_2(): + side_effect = [RuntimeError, RuntimeError, 3] + f_mock = MagicMock(side_effect=side_effect) + tries = 5 + result = None + try: + result = retry_call(f_mock, exceptions=RuntimeError, tries=tries) + except RuntimeError: + pass + + assert result == 3 + assert f_mock.call_count == len(side_effect) + + +def test_retry_call_with_args(): + + def f(value=0): + if value < 0: + return value + else: + raise RuntimeError + + return_value = -1 + result = None + f_mock = MagicMock(spec=f, return_value=return_value) + try: + result = retry_call(f_mock, fargs=[return_value]) + except RuntimeError: + pass + + assert result == return_value + assert f_mock.call_count == 1 + + +def test_retry_call_with_kwargs(): + + def f(value=0): + if value < 0: + return value + else: + raise RuntimeError + + kwargs = {'value': -1} + result = None + f_mock = MagicMock(spec=f, return_value=kwargs['value']) + try: + result = retry_call(f_mock, fkwargs=kwargs) + except RuntimeError: + pass + + assert result == kwargs['value'] + assert f_mock.call_count == 1 diff --git a/contrib/python/retry/py2/tests/ya.make b/contrib/python/retry/py2/tests/ya.make new file mode 100644 index 0000000000..381b71c8c4 --- /dev/null +++ b/contrib/python/retry/py2/tests/ya.make @@ -0,0 +1,23 @@ +PY2TEST() + +SUBSCRIBER(g:python-contrib) + +PEERDIR( + contrib/python/retry +) + +IF (PYTHON2) + PEERDIR( + contrib/python/mock + ) +ENDIF() + +SRCDIR(contrib/python/retry/py2/retry/tests) + +TEST_SRCS( + test_retry.py +) + +NO_LINT() + +END() diff --git a/contrib/python/retry/py2/ya.make b/contrib/python/retry/py2/ya.make new file mode 100644 index 0000000000..97cf460201 --- /dev/null +++ b/contrib/python/retry/py2/ya.make @@ -0,0 +1,35 @@ +# Generated by devtools/yamaker (pypi). + +PY2_LIBRARY() + +SUBSCRIBER(g:python-contrib) + +VERSION(0.9.2) + +LICENSE(Apache-2.0) + +PEERDIR( + contrib/python/decorator + contrib/python/py +) + +NO_LINT() + +PY_SRCS( + TOP_LEVEL + retry/__init__.py + retry/api.py + retry/compat.py +) + +RESOURCE_FILES( + PREFIX contrib/python/retry/py2/ + .dist-info/METADATA + .dist-info/top_level.txt +) + +END() + +RECURSE_FOR_TESTS( + tests +) diff --git a/contrib/python/retry/py3/.dist-info/METADATA b/contrib/python/retry/py3/.dist-info/METADATA new file mode 100644 index 0000000000..c5986d2fea --- /dev/null +++ b/contrib/python/retry/py3/.dist-info/METADATA @@ -0,0 +1,184 @@ +Metadata-Version: 2.0 +Name: retry +Version: 0.9.2 +Summary: Easy to use retry decorator. +Home-page: https://github.com/invl/retry +Author: invl +Author-email: invlpg@gmail.com +License: Apache License 2.0 +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Natural Language :: English +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development +Requires-Dist: decorator (>=3.4.2) +Requires-Dist: py (<2.0.0,>=1.4.26) + +retry +===== + +.. image:: https://pypip.in/d/retry/badge.png + :target: https://pypi.python.org/pypi/retry/ + +.. image:: https://pypip.in/v/retry/badge.png + :target: https://pypi.python.org/pypi/retry/ + +.. image:: https://pypip.in/license/retry/badge.png + :target: https://pypi.python.org/pypi/retry/ + + +Easy to use retry decorator. + + +Features +-------- + +- No external dependency (stdlib only). +- (Optionally) Preserve function signatures (`pip install decorator`). +- Original traceback, easy to debug. + + +Installation +------------ + +.. code-block:: bash + + $ pip install retry + + +API +--- + +retry decorator +^^^^^^^^^^^^^^^ + +.. code:: python + + def retry(exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1, jitter=0, logger=logging_logger): + """Return a retry decorator. + + :param exceptions: an exception or a tuple of exceptions to catch. default: Exception. + :param tries: the maximum number of attempts. default: -1 (infinite). + :param delay: initial delay between attempts. default: 0. + :param max_delay: the maximum value of delay. default: None (no limit). + :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff). + :param jitter: extra seconds added to delay between attempts. default: 0. + fixed if a number, random if a range tuple (min, max) + :param logger: logger.warning(fmt, error, delay) will be called on failed attempts. + default: retry.logging_logger. if None, logging is disabled. + """ + +Various retrying logic can be achieved by combination of arguments. + + +Examples +"""""""" + +.. code:: python + + from retry import retry + +.. code:: python + + @retry() + def make_trouble(): + '''Retry until succeed''' + +.. code:: python + + @retry(ZeroDivisionError, tries=3, delay=2) + def make_trouble(): + '''Retry on ZeroDivisionError, raise error after 3 attempts, sleep 2 seconds between attempts.''' + +.. code:: python + + @retry((ValueError, TypeError), delay=1, backoff=2) + def make_trouble(): + '''Retry on ValueError or TypeError, sleep 1, 2, 4, 8, ... seconds between attempts.''' + +.. code:: python + + @retry((ValueError, TypeError), delay=1, backoff=2, max_delay=4) + def make_trouble(): + '''Retry on ValueError or TypeError, sleep 1, 2, 4, 4, ... seconds between attempts.''' + +.. code:: python + + @retry(ValueError, delay=1, jitter=1) + def make_trouble(): + '''Retry on ValueError, sleep 1, 2, 3, 4, ... seconds between attempts.''' + +.. code:: python + + # If you enable logging, you can get warnings like 'ValueError, retrying in + # 1 seconds' + if __name__ == '__main__': + import logging + logging.basicConfig() + make_trouble() + +retry_call +^^^^^^^^^^ + +.. code:: python + + def retry_call(f, fargs=None, fkwargs=None, exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1, + jitter=0, + logger=logging_logger): + """ + Calls a function and re-executes it if it failed. + + :param f: the function to execute. + :param fargs: the positional arguments of the function to execute. + :param fkwargs: the named arguments of the function to execute. + :param exceptions: an exception or a tuple of exceptions to catch. default: Exception. + :param tries: the maximum number of attempts. default: -1 (infinite). + :param delay: initial delay between attempts. default: 0. + :param max_delay: the maximum value of delay. default: None (no limit). + :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff). + :param jitter: extra seconds added to delay between attempts. default: 0. + fixed if a number, random if a range tuple (min, max) + :param logger: logger.warning(fmt, error, delay) will be called on failed attempts. + default: retry.logging_logger. if None, logging is disabled. + :returns: the result of the f function. + """ + +This is very similar to the decorator, except that it takes a function and its arguments as parameters. The use case behind it is to be able to dynamically adjust the retry arguments. + +.. code:: python + + import requests + + from retry.api import retry_call + + + def make_trouble(service, info=None): + if not info: + info = '' + r = requests.get(service + info) + return r.text + + + def what_is_my_ip(approach=None): + if approach == "optimistic": + tries = 1 + elif approach == "conservative": + tries = 3 + else: + # skeptical + tries = -1 + result = retry_call(make_trouble, fargs=["http://ipinfo.io/"], fkwargs={"info": "ip"}, tries=tries) + print(result) + + what_is_my_ip("conservative") + + + diff --git a/contrib/python/retry/py3/.dist-info/top_level.txt b/contrib/python/retry/py3/.dist-info/top_level.txt new file mode 100644 index 0000000000..77428f7b73 --- /dev/null +++ b/contrib/python/retry/py3/.dist-info/top_level.txt @@ -0,0 +1 @@ +retry diff --git a/contrib/python/retry/py3/AUTHORS b/contrib/python/retry/py3/AUTHORS new file mode 100644 index 0000000000..573ff53561 --- /dev/null +++ b/contrib/python/retry/py3/AUTHORS @@ -0,0 +1,8 @@ +Richard O'Dwyer <richard@richard.do> +Rémy <remy.greinhofer@gmail.com> +Rémy <rgreinho@users.noreply.github.com> +Rémy Greinhofer <remy.greinhofer@livelovely.com> +invl <invlpg@gmail.com> +invlpg <invlpg@gmail.com> +invlpg@gmail.com <invlpg@gmail.com> +williara <ray@hudl.com> diff --git a/contrib/python/retry/py3/LICENSE b/contrib/python/retry/py3/LICENSE new file mode 100644 index 0000000000..3a2cc3260e --- /dev/null +++ b/contrib/python/retry/py3/LICENSE @@ -0,0 +1,13 @@ +Copyright 2014 invl + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/contrib/python/retry/py3/README.rst b/contrib/python/retry/py3/README.rst new file mode 100644 index 0000000000..e8f78b15e8 --- /dev/null +++ b/contrib/python/retry/py3/README.rst @@ -0,0 +1,160 @@ +retry +===== + +.. image:: https://pypip.in/d/retry/badge.png + :target: https://pypi.python.org/pypi/retry/ + +.. image:: https://pypip.in/v/retry/badge.png + :target: https://pypi.python.org/pypi/retry/ + +.. image:: https://pypip.in/license/retry/badge.png + :target: https://pypi.python.org/pypi/retry/ + + +Easy to use retry decorator. + + +Features +-------- + +- No external dependency (stdlib only). +- (Optionally) Preserve function signatures (`pip install decorator`). +- Original traceback, easy to debug. + + +Installation +------------ + +.. code-block:: bash + + $ pip install retry + + +API +--- + +retry decorator +^^^^^^^^^^^^^^^ + +.. code:: python + + def retry(exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1, jitter=0, logger=logging_logger): + """Return a retry decorator. + + :param exceptions: an exception or a tuple of exceptions to catch. default: Exception. + :param tries: the maximum number of attempts. default: -1 (infinite). + :param delay: initial delay between attempts. default: 0. + :param max_delay: the maximum value of delay. default: None (no limit). + :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff). + :param jitter: extra seconds added to delay between attempts. default: 0. + fixed if a number, random if a range tuple (min, max) + :param logger: logger.warning(fmt, error, delay) will be called on failed attempts. + default: retry.logging_logger. if None, logging is disabled. + """ + +Various retrying logic can be achieved by combination of arguments. + + +Examples +"""""""" + +.. code:: python + + from retry import retry + +.. code:: python + + @retry() + def make_trouble(): + '''Retry until succeed''' + +.. code:: python + + @retry(ZeroDivisionError, tries=3, delay=2) + def make_trouble(): + '''Retry on ZeroDivisionError, raise error after 3 attempts, sleep 2 seconds between attempts.''' + +.. code:: python + + @retry((ValueError, TypeError), delay=1, backoff=2) + def make_trouble(): + '''Retry on ValueError or TypeError, sleep 1, 2, 4, 8, ... seconds between attempts.''' + +.. code:: python + + @retry((ValueError, TypeError), delay=1, backoff=2, max_delay=4) + def make_trouble(): + '''Retry on ValueError or TypeError, sleep 1, 2, 4, 4, ... seconds between attempts.''' + +.. code:: python + + @retry(ValueError, delay=1, jitter=1) + def make_trouble(): + '''Retry on ValueError, sleep 1, 2, 3, 4, ... seconds between attempts.''' + +.. code:: python + + # If you enable logging, you can get warnings like 'ValueError, retrying in + # 1 seconds' + if __name__ == '__main__': + import logging + logging.basicConfig() + make_trouble() + +retry_call +^^^^^^^^^^ + +.. code:: python + + def retry_call(f, fargs=None, fkwargs=None, exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1, + jitter=0, + logger=logging_logger): + """ + Calls a function and re-executes it if it failed. + + :param f: the function to execute. + :param fargs: the positional arguments of the function to execute. + :param fkwargs: the named arguments of the function to execute. + :param exceptions: an exception or a tuple of exceptions to catch. default: Exception. + :param tries: the maximum number of attempts. default: -1 (infinite). + :param delay: initial delay between attempts. default: 0. + :param max_delay: the maximum value of delay. default: None (no limit). + :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff). + :param jitter: extra seconds added to delay between attempts. default: 0. + fixed if a number, random if a range tuple (min, max) + :param logger: logger.warning(fmt, error, delay) will be called on failed attempts. + default: retry.logging_logger. if None, logging is disabled. + :returns: the result of the f function. + """ + +This is very similar to the decorator, except that it takes a function and its arguments as parameters. The use case behind it is to be able to dynamically adjust the retry arguments. + +.. code:: python + + import requests + + from retry.api import retry_call + + + def make_trouble(service, info=None): + if not info: + info = '' + r = requests.get(service + info) + return r.text + + + def what_is_my_ip(approach=None): + if approach == "optimistic": + tries = 1 + elif approach == "conservative": + tries = 3 + else: + # skeptical + tries = -1 + result = retry_call(make_trouble, fargs=["http://ipinfo.io/"], fkwargs={"info": "ip"}, tries=tries) + print(result) + + what_is_my_ip("conservative") + + + diff --git a/contrib/python/retry/py3/patches/01-return-retry_call.patch b/contrib/python/retry/py3/patches/01-return-retry_call.patch new file mode 100644 index 0000000000..b849f59947 --- /dev/null +++ b/contrib/python/retry/py3/patches/01-return-retry_call.patch @@ -0,0 +1,13 @@ +--- contrib/python/retry/py3/retry/__init__.py (index) ++++ contrib/python/retry/py3/retry/__init__.py (working tree) +@@ -1,8 +1,8 @@ +-__all__ = ['retry'] ++__all__ = ['retry', 'retry_call'] + + import logging + +-from .api import retry ++from .api import retry, retry_call + + + # Set default logging handler to avoid "No handler found" warnings. diff --git a/contrib/python/retry/py3/retry/__init__.py b/contrib/python/retry/py3/retry/__init__.py new file mode 100644 index 0000000000..1dd5705356 --- /dev/null +++ b/contrib/python/retry/py3/retry/__init__.py @@ -0,0 +1,18 @@ +__all__ = ['retry', 'retry_call'] + +import logging + +from .api import retry, retry_call + + +# Set default logging handler to avoid "No handler found" warnings. +try: # Python 2.7+ + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + + def emit(self, record): + pass + +log = logging.getLogger(__name__) +log.addHandler(NullHandler()) diff --git a/contrib/python/retry/py3/retry/api.py b/contrib/python/retry/py3/retry/api.py new file mode 100644 index 0000000000..245e4e4c08 --- /dev/null +++ b/contrib/python/retry/py3/retry/api.py @@ -0,0 +1,101 @@ +import logging +import random +import time + +from functools import partial + +from retry.compat import decorator + + +logging_logger = logging.getLogger(__name__) + + +def __retry_internal(f, exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1, jitter=0, + logger=logging_logger): + """ + Executes a function and retries it if it failed. + + :param f: the function to execute. + :param exceptions: an exception or a tuple of exceptions to catch. default: Exception. + :param tries: the maximum number of attempts. default: -1 (infinite). + :param delay: initial delay between attempts. default: 0. + :param max_delay: the maximum value of delay. default: None (no limit). + :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff). + :param jitter: extra seconds added to delay between attempts. default: 0. + fixed if a number, random if a range tuple (min, max) + :param logger: logger.warning(fmt, error, delay) will be called on failed attempts. + default: retry.logging_logger. if None, logging is disabled. + :returns: the result of the f function. + """ + _tries, _delay = tries, delay + while _tries: + try: + return f() + except exceptions as e: + _tries -= 1 + if not _tries: + raise + + if logger is not None: + logger.warning('%s, retrying in %s seconds...', e, _delay) + + time.sleep(_delay) + _delay *= backoff + + if isinstance(jitter, tuple): + _delay += random.uniform(*jitter) + else: + _delay += jitter + + if max_delay is not None: + _delay = min(_delay, max_delay) + + +def retry(exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1, jitter=0, logger=logging_logger): + """Returns a retry decorator. + + :param exceptions: an exception or a tuple of exceptions to catch. default: Exception. + :param tries: the maximum number of attempts. default: -1 (infinite). + :param delay: initial delay between attempts. default: 0. + :param max_delay: the maximum value of delay. default: None (no limit). + :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff). + :param jitter: extra seconds added to delay between attempts. default: 0. + fixed if a number, random if a range tuple (min, max) + :param logger: logger.warning(fmt, error, delay) will be called on failed attempts. + default: retry.logging_logger. if None, logging is disabled. + :returns: a retry decorator. + """ + + @decorator + def retry_decorator(f, *fargs, **fkwargs): + args = fargs if fargs else list() + kwargs = fkwargs if fkwargs else dict() + return __retry_internal(partial(f, *args, **kwargs), exceptions, tries, delay, max_delay, backoff, jitter, + logger) + + return retry_decorator + + +def retry_call(f, fargs=None, fkwargs=None, exceptions=Exception, tries=-1, delay=0, max_delay=None, backoff=1, + jitter=0, + logger=logging_logger): + """ + Calls a function and re-executes it if it failed. + + :param f: the function to execute. + :param fargs: the positional arguments of the function to execute. + :param fkwargs: the named arguments of the function to execute. + :param exceptions: an exception or a tuple of exceptions to catch. default: Exception. + :param tries: the maximum number of attempts. default: -1 (infinite). + :param delay: initial delay between attempts. default: 0. + :param max_delay: the maximum value of delay. default: None (no limit). + :param backoff: multiplier applied to delay between attempts. default: 1 (no backoff). + :param jitter: extra seconds added to delay between attempts. default: 0. + fixed if a number, random if a range tuple (min, max) + :param logger: logger.warning(fmt, error, delay) will be called on failed attempts. + default: retry.logging_logger. if None, logging is disabled. + :returns: the result of the f function. + """ + args = fargs if fargs else list() + kwargs = fkwargs if fkwargs else dict() + return __retry_internal(partial(f, *args, **kwargs), exceptions, tries, delay, max_delay, backoff, jitter, logger) diff --git a/contrib/python/retry/py3/retry/compat.py b/contrib/python/retry/py3/retry/compat.py new file mode 100644 index 0000000000..f39510d171 --- /dev/null +++ b/contrib/python/retry/py3/retry/compat.py @@ -0,0 +1,18 @@ +import functools + + +try: + from decorator import decorator +except ImportError: + def decorator(caller): + """ Turns caller into a decorator. + Unlike decorator module, function signature is not preserved. + + :param caller: caller(f, *args, **kwargs) + """ + def decor(f): + @functools.wraps(f) + def wrapper(*args, **kwargs): + return caller(f, *args, **kwargs) + return wrapper + return decor diff --git a/contrib/python/retry/py3/retry/tests/__init__.py b/contrib/python/retry/py3/retry/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/contrib/python/retry/py3/retry/tests/__init__.py diff --git a/contrib/python/retry/py3/retry/tests/test_retry.py b/contrib/python/retry/py3/retry/tests/test_retry.py new file mode 100644 index 0000000000..64f45cd89d --- /dev/null +++ b/contrib/python/retry/py3/retry/tests/test_retry.py @@ -0,0 +1,185 @@ +try: + from unittest.mock import create_autospec +except ImportError: + from mock import create_autospec + +try: + from unittest.mock import MagicMock +except ImportError: + from mock import MagicMock + +import time + +import pytest + +from retry.api import retry_call +from retry.api import retry + + +def test_retry(monkeypatch): + mock_sleep_time = [0] + + def mock_sleep(seconds): + mock_sleep_time[0] += seconds + + monkeypatch.setattr(time, 'sleep', mock_sleep) + + hit = [0] + + tries = 5 + delay = 1 + backoff = 2 + + @retry(tries=tries, delay=delay, backoff=backoff) + def f(): + hit[0] += 1 + 1 / 0 + + with pytest.raises(ZeroDivisionError): + f() + assert hit[0] == tries + assert mock_sleep_time[0] == sum( + delay * backoff ** i for i in range(tries - 1)) + + +def test_tries_inf(): + hit = [0] + target = 10 + + @retry(tries=float('inf')) + def f(): + hit[0] += 1 + if hit[0] == target: + return target + else: + raise ValueError + assert f() == target + + +def test_tries_minus1(): + hit = [0] + target = 10 + + @retry(tries=-1) + def f(): + hit[0] += 1 + if hit[0] == target: + return target + else: + raise ValueError + assert f() == target + + +def test_max_delay(monkeypatch): + mock_sleep_time = [0] + + def mock_sleep(seconds): + mock_sleep_time[0] += seconds + + monkeypatch.setattr(time, 'sleep', mock_sleep) + + hit = [0] + + tries = 5 + delay = 1 + backoff = 2 + max_delay = delay # Never increase delay + + @retry(tries=tries, delay=delay, max_delay=max_delay, backoff=backoff) + def f(): + hit[0] += 1 + 1 / 0 + + with pytest.raises(ZeroDivisionError): + f() + assert hit[0] == tries + assert mock_sleep_time[0] == delay * (tries - 1) + + +def test_fixed_jitter(monkeypatch): + mock_sleep_time = [0] + + def mock_sleep(seconds): + mock_sleep_time[0] += seconds + + monkeypatch.setattr(time, 'sleep', mock_sleep) + + hit = [0] + + tries = 10 + jitter = 1 + + @retry(tries=tries, jitter=jitter) + def f(): + hit[0] += 1 + 1 / 0 + + with pytest.raises(ZeroDivisionError): + f() + assert hit[0] == tries + assert mock_sleep_time[0] == sum(range(tries - 1)) + + +def test_retry_call(): + f_mock = MagicMock(side_effect=RuntimeError) + tries = 2 + try: + retry_call(f_mock, exceptions=RuntimeError, tries=tries) + except RuntimeError: + pass + + assert f_mock.call_count == tries + + +def test_retry_call_2(): + side_effect = [RuntimeError, RuntimeError, 3] + f_mock = MagicMock(side_effect=side_effect) + tries = 5 + result = None + try: + result = retry_call(f_mock, exceptions=RuntimeError, tries=tries) + except RuntimeError: + pass + + assert result == 3 + assert f_mock.call_count == len(side_effect) + + +def test_retry_call_with_args(): + + def f(value=0): + if value < 0: + return value + else: + raise RuntimeError + + return_value = -1 + result = None + f_mock = MagicMock(spec=f, return_value=return_value) + try: + result = retry_call(f_mock, fargs=[return_value]) + except RuntimeError: + pass + + assert result == return_value + assert f_mock.call_count == 1 + + +def test_retry_call_with_kwargs(): + + def f(value=0): + if value < 0: + return value + else: + raise RuntimeError + + kwargs = {'value': -1} + result = None + f_mock = MagicMock(spec=f, return_value=kwargs['value']) + try: + result = retry_call(f_mock, fkwargs=kwargs) + except RuntimeError: + pass + + assert result == kwargs['value'] + assert f_mock.call_count == 1 diff --git a/contrib/python/retry/py3/tests/ya.make b/contrib/python/retry/py3/tests/ya.make new file mode 100644 index 0000000000..b7d8e73ae8 --- /dev/null +++ b/contrib/python/retry/py3/tests/ya.make @@ -0,0 +1,23 @@ +PY3TEST() + +SUBSCRIBER(g:python-contrib) + +PEERDIR( + contrib/python/retry +) + +IF (PYTHON2) + PEERDIR( + contrib/python/mock + ) +ENDIF() + +SRCDIR(contrib/python/retry/py3/retry/tests) + +TEST_SRCS( + test_retry.py +) + +NO_LINT() + +END() diff --git a/contrib/python/retry/py3/ya.make b/contrib/python/retry/py3/ya.make new file mode 100644 index 0000000000..5c3b309b71 --- /dev/null +++ b/contrib/python/retry/py3/ya.make @@ -0,0 +1,35 @@ +# Generated by devtools/yamaker (pypi). + +PY3_LIBRARY() + +SUBSCRIBER(g:python-contrib) + +VERSION(0.9.2) + +LICENSE(Apache-2.0) + +PEERDIR( + contrib/python/decorator + contrib/python/py +) + +NO_LINT() + +PY_SRCS( + TOP_LEVEL + retry/__init__.py + retry/api.py + retry/compat.py +) + +RESOURCE_FILES( + PREFIX contrib/python/retry/py3/ + .dist-info/METADATA + .dist-info/top_level.txt +) + +END() + +RECURSE_FOR_TESTS( + tests +) diff --git a/contrib/python/retry/ya.make b/contrib/python/retry/ya.make new file mode 100644 index 0000000000..3766ebcd67 --- /dev/null +++ b/contrib/python/retry/ya.make @@ -0,0 +1,20 @@ +PY23_LIBRARY() + +LICENSE(Service-Py23-Proxy) + +SUBSCRIBER(g:python-contrib) + +IF (PYTHON2) + PEERDIR(contrib/python/retry/py2) +ELSE() + PEERDIR(contrib/python/retry/py3) +ENDIF() + +NO_LINT() + +END() + +RECURSE( + py2 + py3 +) |