Merge pull request #12 from bgreen-litl/callable-kwargs

Support runtime configuration with optionally callable kwargs

Author: bgreen-litl

README.rst

@@ -128,6 +128,37 @@ backoff behavior for different cases::
     def poll_for_message(queue):
         return queue.get()
 
+Runtime Configuration
+---------------------
+
+The decorator functions ``on_exception`` and ``on_predicate`` are
+generally evaluated at import time. This is fine when the keyword args
+are passed as constant values, but suppose we want to consult a
+dictionary with configuration options that only become available at
+runtime. The relevant values are not available at import time. Instead,
+decorator functions can be passed callables which are evaluated at
+runtime to obtain the value::
+
+    def lookup_max_tries():
+        # pretend we have a global reference to 'app' here
+        # and that it has a dictionary-like 'config' property
+        return app.config["BACKOFF_MAX_TRIES"]
+
+    @backoff.on_exception(backoff.expo,
+                          ValueError,
+                          max_tries=lookup_max_tries)
+
+More cleverly, you might define a function which returns a lookup
+function for a specified variable::
+
+    def config(app, name):
+        return functools.partial(app.config.get, name)
+
+    @backoff.on_exception(backoff.expo,
+                          ValueError,
+                          max_value=config(app, "BACKOFF_MAX_VALUE")
+                          max_tries=config(app, "BACKOFF_MAX_TRIES"))
+
 Event handlers
 --------------
 

backoff.py

@@ -148,7 +148,9 @@ def on_predicate(wait_gen,
             is exceeded.  The parameter is a dict containing details
             about the invocation.
         **wait_gen_kwargs: Any additional keyword args specified will be
-            passed to wait_gen when it is initialized.
+            passed to wait_gen when it is initialized.  Any callable
+            args will first be evaluated and their return values passed.
+            This is useful for runtime configuration.
     """
     success_hdlrs = _handlers(on_success)
     backoff_hdlrs = _handlers(on_backoff, _log_backoff)
@@ -158,14 +160,19 @@ def decorate(target):
 
         @functools.wraps(target)
         def retry(*args, **kwargs):
-            tries = 0
+            # change names because python 2.x doesn't have nonlocal
+            max_tries_ = _maybe_call(max_tries)
+
+            # there are no dictionary comprehensions in python 2.6
+            wait = wait_gen(**dict((k, _maybe_call(v))
+                                   for k, v in wait_gen_kwargs.items()))
 
-            wait = wait_gen(**wait_gen_kwargs)
+            tries = 0
             while True:
                 tries += 1
                 ret = target(*args, **kwargs)
                 if predicate(ret):
-                    if tries == max_tries:
+                    if tries == max_tries_:
                         for hdlr in giveup_hdlrs:
                             hdlr({'target': target,
                                   'args': args,
@@ -231,7 +238,8 @@ def on_exception(wait_gen,
         max_tries: The maximum number of attempts to make before giving
             up. Once exhausted, the exception will be allowed to escape.
             The default value of None means their is no limit to the
-            number of tries.
+            number of tries. If a callable is passed, it will be
+            evaluated at runtime and its return value used.
         jitter: A function of the value yielded by wait_gen returning
             the actual time to wait. This distributes wait times
             stochastically in order to avoid timing collisions across
@@ -252,8 +260,9 @@ def on_exception(wait_gen,
             is exceeded.  The parameter is a dict containing details
             about the invocation.
         **wait_gen_kwargs: Any additional keyword args specified will be
-            passed to wait_gen when it is initialized.
-
+            passed to wait_gen when it is initialized.  Any callable
+            args will first be evaluated and their return values passed.
+            This is useful for runtime configuration.
     """
     success_hdlrs = _handlers(on_success)
     backoff_hdlrs = _handlers(on_backoff, _log_backoff)
@@ -263,15 +272,20 @@ def decorate(target):
 
         @functools.wraps(target)
         def retry(*args, **kwargs):
+            # change names because python 2.x doesn't have nonlocal
+            max_tries_ = _maybe_call(max_tries)
+
+            # there are no dictionary comprehensions in python 2.6
+            wait = wait_gen(**dict((k, _maybe_call(v))
+                                   for k, v in wait_gen_kwargs.items()))
+
             tries = 0
-            wait = wait_gen(**wait_gen_kwargs)
             while True:
                 try:
                     tries += 1
                     ret = target(*args, **kwargs)
                 except exception as e:
-                    if giveup(e) or tries == max_tries:
-
+                    if giveup(e) or tries == max_tries_:
                         for hdlr in giveup_hdlrs:
                             hdlr({'target': target,
                                   'args': args,
@@ -326,6 +340,11 @@ def _handlers(hdlr, default=None):
     return defaults + [hdlr]
 
 
+# Evaluate arg that can be either a fixed value or a callable.
+def _maybe_call(f, *args, **kwargs):
+    return f(*args, **kwargs) if callable(f) else f
+
+
 # Formats a function invocation as a unicode string for logging.
 def _invoc_repr(details):
     f, args, kwargs = details['target'], details['args'], details['kwargs']

backoff_tests.py

@@ -516,3 +516,48 @@ def success(*args, **kwargs):
                        'target': success._target,
                        'tries': 3,
                        'value': True}
+
+
+def test_on_exception_callable_max_tries(monkeypatch):
+    monkeypatch.setattr('time.sleep', lambda x: None)
+
+    def lookup_max_tries():
+        return 3
+
+    log = []
+
+    @backoff.on_exception(backoff.constant,
+                          ValueError,
+                          max_tries=lookup_max_tries)
+    def exceptor():
+        log.append(True)
+        raise ValueError()
+
+    with pytest.raises(ValueError):
+        exceptor()
+
+    assert len(log) == 3
+
+
+def test_on_exception_callable_gen_kwargs():
+
+    def lookup_foo():
+        return "foo"
+
+    def wait_gen(foo=None, bar=None):
+        assert foo == "foo"
+        assert bar == "bar"
+
+        while True:
+            yield 0
+
+    @backoff.on_exception(wait_gen,
+                          ValueError,
+                          max_tries=2,
+                          foo=lookup_foo,
+                          bar="bar")
+    def exceptor():
+        raise ValueError("aah")
+
+    with pytest.raises(ValueError):
+        exceptor()