invlidate cluster info cache after any error in get/set/delete methods

Author: gusdan

README.rst

@@ -127,4 +127,4 @@ Testing
 
 Run the tests like this::
 
-    nosetest
+    nosetests

django_elasticache/__init__.py

@@ -1,2 +1,2 @@
-VERSION = (0, 0, 3)
+VERSION = (0, 0, 4)
 __version__ = '.'.join(map(str, VERSION))

django_elasticache/memcached.py

@@ -2,12 +2,26 @@
 Backend for django cache
 """
 import socket
+from functools import wraps
 from django.core.cache import InvalidCacheBackendError
 from django.core.cache.backends.memcached import PyLibMCCache
-from django.utils.functional import cached_property
 from .cluster_utils import get_cluster_info
 
 
+def invalidate_cache_after_error(f):
+    """
+    catch any exception and invalidate internal cache with list of nodes
+    """
+    @wraps(f)
+    def wrapper(self, *args, **kwds):
+        try:
+            return f(self, *args, **kwds)
+        except Exception:
+            self.clear_cluster_nodes_cache()
+            raise
+    return wrapper
+
+
 class ElastiCache(PyLibMCCache):
     """
     backend for Amazon ElastiCache (memcached) with auto discovery mode
@@ -40,18 +54,25 @@ def update_params(self, params):
                 'ketama': True
             }
 
-    @cached_property
+    def clear_cluster_nodes_cache(self):
+        """clear internal cache with list of nodes in cluster"""
+        if hasattr(self, '_cluster_nodes_cache'):
+            del self._cluster_nodes_cache
+
     def get_cluster_nodes(self):
         """
         return list with all nodes in cluster
         """
-        server, port = self._servers[0].split(':')
-        try:
-            return get_cluster_info(server, port)['nodes']
-        except (socket.gaierror, socket.timeout) as err:
-            raise Exception('Cannot connect to cluster {} ({})'.format(
-                self._servers[0], err
-            ))
+        if not hasattr(self, '_cluster_nodes_cache'):
+            server, port = self._servers[0].split(':')
+            try:
+                self._cluster_nodes_cache = (
+                    get_cluster_info(server, port)['nodes'])
+            except (socket.gaierror, socket.timeout) as err:
+                raise Exception('Cannot connect to cluster {} ({})'.format(
+                    self._servers[0], err
+                ))
+        return self._cluster_nodes_cache
 
     @property
     def _cache(self):
@@ -67,10 +88,30 @@ def _cache(self):
         if client:
             return client
 
-        client = self._lib.Client(self.get_cluster_nodes)
+        client = self._lib.Client(self.get_cluster_nodes())
         if self._options:
             client.behaviors = self._options
 
         container._client = client
 
         return client
+
+    @invalidate_cache_after_error
+    def get(self, *args, **kwargs):
+        return super(ElastiCache, self).get(*args, **kwargs)
+
+    @invalidate_cache_after_error
+    def get_many(self, *args, **kwargs):
+        return super(ElastiCache, self).get_many(*args, **kwargs)
+
+    @invalidate_cache_after_error
+    def set(self, *args, **kwargs):
+        return super(ElastiCache, self).set(*args, **kwargs)
+
+    @invalidate_cache_after_error
+    def set_many(self, *args, **kwargs):
+        return super(ElastiCache, self).set_many(*args, **kwargs)
+
+    @invalidate_cache_after_error
+    def delete(self, *args, **kwargs):
+        return super(ElastiCache, self).delete(*args, **kwargs)

tests/test_backend.py

@@ -53,14 +53,48 @@ def test_split_servers(get_cluster_info):
 
 @patch('django.conf.settings', global_settings)
 @patch('django_elasticache.memcached.get_cluster_info')
-def test_property_cache(get_cluster_info):
+def test_node_info_cache(get_cluster_info):
     from django_elasticache.memcached import ElastiCache
-    backend = ElastiCache('h:0', {})
     servers = ['h1:p', 'h2:p']
     get_cluster_info.return_value = {
         'nodes': servers
     }
+
+    backend = ElastiCache('h:0', {})
     backend._lib.Client = Mock()
     backend.set('key1', 'val')
+    backend.get('key1')
     backend.set('key2', 'val')
+    backend.get('key2')
     backend._lib.Client.assert_called_once_with(servers)
+    eq_(backend._cache.get.call_count, 2)
+    eq_(backend._cache.set.call_count, 2)
+
+    get_cluster_info.assert_called_once_with('h', '0')
+
+
+@patch('django.conf.settings', global_settings)
+@patch('django_elasticache.memcached.get_cluster_info')
+def test_invalidate_cache(get_cluster_info):
+    from django_elasticache.memcached import ElastiCache
+    servers = ['h1:p', 'h2:p']
+    get_cluster_info.return_value = {
+        'nodes': servers
+    }
+
+    backend = ElastiCache('h:0', {})
+    backend._lib.Client = Mock()
+    assert backend._cache
+    backend._cache.get = Mock()
+    backend._cache.get.side_effect = Exception()
+    try:
+        backend.get('key1', 'val')
+    except Exception:
+        pass
+    backend._local._client = None
+    try:
+        backend.get('key1', 'val')
+    except Exception:
+        pass
+    eq_(backend._cache.get.call_count, 2)
+    eq_(get_cluster_info.call_count, 2)