Support runtime configuration with optionally callable kwargs

This makes the max_tries kwarg on on_exception and on_predicate
optionally accept a callable to be evaluated at runtime rather
than a fixed value.

Similarly, wait_gen_kwargs also optionally accept callables to be
evaluated before passing into the wait_gen. This means that all
current kwargs on backoff.expo, backoff.fibo, backoff.constant now
accept callables, and also that all future and user-space wait
generators will as well.

This addresses #10

Author: bgreen-litl

README.rst

@@ -128,6 +128,37 @@ backoff behavior for different cases::
     def poll_for_message(queue):
         return queue.get()
 
+Runtime Configuration
+---------------------
+
+The decorator functions ``on_exception`` and ``on_predicate`` are
+generally evaluated at import time. This is fine when the keyword args
+are passed as constant values, but suppose we want to consult a
+dictionary with configuration options that only become available at
+runtime. The relevant values are not available at import time. Instead,
+decorator functions can be passed callables which are evaluated at
+runtime to obtain the value::
+
+    def lookup_max_tries():
+        # pretend we have a global reference to 'app' here
+        # and that it has a dictionary-like 'config' property
+        return app.config["BACKOFF_MAX_TRIES"]
+
+    @backoff.on_exception(backoff.expo,
+                          ValueError,
+                          max_tries=lookup_max_tries)
+
+More cleverly, you might define a function which returns a lookup
+function for a specified variable::
+
+    def config(app, name):
+        return functools.partial(app.config.get, name)
+
+    @backoff.on_exception(backoff.expo,
+                          ValueError,
+                          max_value=config(app, "BACKOFF_MAX_VALUE")
+                          max_tries=config(app, "BACKOFF_MAX_TRIES"))
+
 Event handlers
 --------------
 

backoff.py

@@ -148,7 +148,9 @@ def on_predicate(wait_gen,
             is exceeded.  The parameter is a dict containing details
             about the invocation.
         **wait_gen_kwargs: Any additional keyword args specified will be
-            passed to wait_gen when it is initialized.
+            passed to wait_gen when it is initialized.  Any callable
+            args will first be evaluated and their return values passed.
+            This is useful for runtime configuration.
     """
     success_hdlrs = _handlers(on_success)
     backoff_hdlrs = _handlers(on_backoff, _log_backoff)
@@ -158,14 +160,19 @@ def decorate(target):
 
         @functools.wraps(target)
         def retry(*args, **kwargs):
-            tries = 0
+            # change names because python 2.x doesn't have nonlocal
+            max_tries_ = _maybe_call(max_tries)
+
+            # there are no dictionary comprehensions in python 2.6
+            wait = wait_gen(**dict((k, _maybe_call(v))
+                                   for k, v in wait_gen_kwargs.items()))
 
-            wait = wait_gen(**wait_gen_kwargs)
+            tries = 0
             while True:
                 tries += 1
                 ret = target(*args, **kwargs)
                 if predicate(ret):
-                    if tries == max_tries:
+                    if tries == max_tries_:
                         for hdlr in giveup_hdlrs:
                             hdlr({'target': target,
                                   'args': args,
@@ -231,7 +238,8 @@ def on_exception(wait_gen,
         max_tries: The maximum number of attempts to make before giving
             up. Once exhausted, the exception will be allowed to escape.
             The default value of None means their is no limit to the
-            number of tries.
+            number of tries. If a callable is passed, it will be
+            evaluated at runtime and its return value used.
         jitter: A function of the value yielded by wait_gen returning
             the actual time to wait. This distributes wait times
             stochastically in order to avoid timing collisions across
@@ -252,8 +260,9 @@ def on_exception(wait_gen,
             is exceeded.  The parameter is a dict containing details
             about the invocation.
         **wait_gen_kwargs: Any additional keyword args specified will be
-            passed to wait_gen when it is initialized.
-
+            passed to wait_gen when it is initialized.  Any callable
+            args will first be evaluated and their return values passed.
+            This is useful for runtime configuration.
     """
     success_hdlrs = _handlers(on_success)
     backoff_hdlrs = _handlers(on_backoff, _log_backoff)
@@ -263,15 +272,20 @@ def decorate(target):
 
         @functools.wraps(target)
         def retry(*args, **kwargs):
+            # change names because python 2.x doesn't have nonlocal
+            max_tries_ = _maybe_call(max_tries)
+
+            # there are no dictionary comprehensions in python 2.6
+            wait = wait_gen(**dict((k, _maybe_call(v))
+                                   for k, v in wait_gen_kwargs.items()))
+
             tries = 0
-            wait = wait_gen(**wait_gen_kwargs)
             while True:
                 try:
                     tries += 1
                     ret = target(*args, **kwargs)
                 except exception as e:
-                    if giveup(e) or tries == max_tries:
-
+                    if giveup(e) or tries == max_tries_:
                         for hdlr in giveup_hdlrs:
                             hdlr({'target': target,
                                   'args': args,
@@ -326,6 +340,11 @@ def _handlers(hdlr, default=None):
     return defaults + [hdlr]
 
 
+# Evaluate arg that can be either a fixed value or a callable.
+def _maybe_call(f, *args, **kwargs):
+    return f(*args, **kwargs) if callable(f) else f
+
+
 # Formats a function invocation as a unicode string for logging.
 def _invoc_repr(details):
     f, args, kwargs = details['target'], details['args'], details['kwargs']

backoff_tests.py

@@ -516,3 +516,48 @@ def success(*args, **kwargs):
                        'target': success._target,
                        'tries': 3,
                        'value': True}
+
+
+def test_on_exception_callable_max_tries(monkeypatch):
+    monkeypatch.setattr('time.sleep', lambda x: None)
+
+    def lookup_max_tries():
+        return 3
+
+    log = []
+
+    @backoff.on_exception(backoff.constant,
+                          ValueError,
+                          max_tries=lookup_max_tries)
+    def exceptor():
+        log.append(True)
+        raise ValueError()
+
+    with pytest.raises(ValueError):
+        exceptor()
+
+    assert len(log) == 3
+
+
+def test_on_exception_callable_gen_kwargs():
+
+    def lookup_foo():
+        return "foo"
+
+    def wait_gen(foo=None, bar=None):
+        assert foo == "foo"
+        assert bar == "bar"
+
+        while True:
+            yield 0
+
+    @backoff.on_exception(wait_gen,
+                          ValueError,
+                          max_tries=2,
+                          foo=lookup_foo,
+                          bar="bar")
+    def exceptor():
+        raise ValueError("aah")
+
+    with pytest.raises(ValueError):
+        exceptor()