Merge pull request #17 from hectcastro/feature/hmc/graceful-degradation

Add support for when cluster endpoint is not available

Author: gusdan

README.rst

@@ -36,10 +36,13 @@ Your cache backend should look something like this::
         'default': {
             'BACKEND': 'django_elasticache.memcached.ElastiCache',
             'LOCATION': 'cache-c.draaaf.cfg.use1.cache.amazonaws.com:11211',
+            'OPTIONS' {
+                'IGNORE_CLUSTER_ERRORS': [True,False],
+            },
         }
     }
 
-By the first call to cache it connects to cluster (using LOCATION param),
+By the first call to cache it connects to cluster (using ``LOCATION`` param),
 gets list of all nodes and setup pylibmc client using full
 list of nodes. As result your cache will work with all nodes in cluster and
 automatically detect new nodes in cluster. List of nodes are stored in class-level
@@ -48,6 +51,10 @@ But if you're using gunicorn or mod_wsgi you usually have max_request settings w
 restart process after some count of processed requests, so auto discovery will work
 fine.
 
+The ``IGNORE_CLUSTER_ERRORS`` option is useful when ``LOCATION`` doesn't have support
+for ``config get cluster``. When set to ``True``, and ``config get cluster`` fails,
+it returns a list of a single node with the same endpoint supplied to ``LOCATION``.
+
 Django-elasticache changes default pylibmc params to increase performance.
 
 Another solutions

django_elasticache/cluster_utils.py

@@ -17,7 +17,7 @@ def __init__(self, cmd, response):
             'Unexpected response {} for command {}'.format(response, cmd))
 
 
-def get_cluster_info(host, port):
+def get_cluster_info(host, port, ignore_cluster_errors=False):
     """
     return dict with info about nodes in cluster and current version
     {
@@ -40,8 +40,21 @@ def get_cluster_info(host, port):
     else:
         cmd = b'get AmazonElastiCache:cluster\n'
     client.write(cmd)
-    res = client.read_until(b'\n\r\nEND\r\n')
+    regex_index, match_object, res = client.expect([
+        re.compile(b'\n\r\nEND\r\n'),
+        re.compile(b'ERROR\r\n')
+    ])
     client.close()
+
+    if res == b'ERROR\r\n' and ignore_cluster_errors:
+        return {
+            'version': version,
+            'nodes': [
+                '{}:{}'.format(smart_text(host),
+                               smart_text(port))
+            ]
+        }
+
     ls = list(filter(None, re.compile(br'\r?\n').split(res)))
     if len(ls) != 4:
         raise WrongProtocolData(cmd, res)

django_elasticache/memcached.py

@@ -38,6 +38,9 @@ def __init__(self, server, params):
             raise InvalidCacheBackendError(
                 'Server configuration should be in format IP:port')
 
+        self._ignore_cluster_errors = self._options.get(
+            'IGNORE_CLUSTER_ERRORS', False)
+
     def update_params(self, params):
         """
         update connection params to maximize performance
@@ -67,7 +70,8 @@ def get_cluster_nodes(self):
             server, port = self._servers[0].split(':')
             try:
                 self._cluster_nodes_cache = (
-                    get_cluster_info(server, port)['nodes'])
+                    get_cluster_info(server, port,
+                                     self._ignore_cluster_errors)['nodes'])
             except (socket.gaierror, socket.timeout) as err:
                 raise Exception('Cannot connect to cluster {} ({})'.format(
                     self._servers[0], err

tests/test_backend.py

@@ -47,7 +47,7 @@ def test_split_servers(get_cluster_info):
     }
     backend._lib.Client = Mock()
     assert backend._cache
-    get_cluster_info.assert_called_once_with('h', '0')
+    get_cluster_info.assert_called_once_with('h', '0', False)
     backend._lib.Client.assert_called_once_with(servers)
 
 
@@ -70,7 +70,7 @@ def test_node_info_cache(get_cluster_info):
     eq_(backend._cache.get.call_count, 2)
     eq_(backend._cache.set.call_count, 2)
 
-    get_cluster_info.assert_called_once_with('h', '0')
+    get_cluster_info.assert_called_once_with('h', '0', False)
 
 
 @patch('django.conf.settings', global_settings)

tests/test_protocol.py

@@ -8,26 +8,44 @@
     from unittest.mock import patch, call, MagicMock
 
 
-TEST_PROTOCOL_1 = [
+TEST_PROTOCOL_1_READ_UNTIL = [
     b'VERSION 1.4.14',
-    b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n',
 ]
 
-TEST_PROTOCOL_2 = [
+TEST_PROTOCOL_1_EXPECT = [
+    (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n'),  # NOQA
+]
+
+TEST_PROTOCOL_2_READ_UNTIL = [
     b'VERSION 1.4.13',
-    b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n',
 ]
 
-TEST_PROTOCOL_3 = [
+TEST_PROTOCOL_2_EXPECT = [
+    (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n'),  # NOQA
+]
+
+TEST_PROTOCOL_3_READ_UNTIL = [
     b'VERSION 1.4.14 (Ubuntu)',
-    b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n',
+]
+
+TEST_PROTOCOL_3_EXPECT = [
+    (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n'),  # NOQA
+]
+
+TEST_PROTOCOL_4_READ_UNTIL = [
+    b'VERSION 1.4.34',
+]
+
+TEST_PROTOCOL_4_EXPECT = [
+    (0, None, b'ERROR\r\n'),
 ]
 
 
 @patch('django_elasticache.cluster_utils.Telnet')
 def test_happy_path(Telnet):
     client = Telnet.return_value
-    client.read_until.side_effect = TEST_PROTOCOL_1
+    client.read_until.side_effect = TEST_PROTOCOL_1_READ_UNTIL
+    client.expect.side_effect = TEST_PROTOCOL_1_EXPECT
     info = get_cluster_info('', 0)
     eq_(info['version'], 1)
     eq_(info['nodes'], ['ip:port', 'host:port'])
@@ -42,7 +60,8 @@ def test_bad_protocol():
 @patch('django_elasticache.cluster_utils.Telnet')
 def test_last_versions(Telnet):
     client = Telnet.return_value
-    client.read_until.side_effect = TEST_PROTOCOL_1
+    client.read_until.side_effect = TEST_PROTOCOL_1_READ_UNTIL
+    client.expect.side_effect = TEST_PROTOCOL_1_EXPECT
     get_cluster_info('', 0)
     client.write.assert_has_calls([
         call(b'version\n'),
@@ -53,7 +72,8 @@ def test_last_versions(Telnet):
 @patch('django_elasticache.cluster_utils.Telnet')
 def test_prev_versions(Telnet):
     client = Telnet.return_value
-    client.read_until.side_effect = TEST_PROTOCOL_2
+    client.read_until.side_effect = TEST_PROTOCOL_2_READ_UNTIL
+    client.expect.side_effect = TEST_PROTOCOL_2_EXPECT
     get_cluster_info('', 0)
     client.write.assert_has_calls([
         call(b'version\n'),
@@ -64,7 +84,8 @@ def test_prev_versions(Telnet):
 @patch('django_elasticache.cluster_utils.Telnet')
 def test_ubuntu_protocol(Telnet):
     client = Telnet.return_value
-    client.read_until.side_effect = TEST_PROTOCOL_3
+    client.read_until.side_effect = TEST_PROTOCOL_3_READ_UNTIL
+    client.expect.side_effect = TEST_PROTOCOL_3_EXPECT
 
     try:
         get_cluster_info('', 0)
@@ -75,3 +96,30 @@ def test_ubuntu_protocol(Telnet):
         call(b'version\n'),
         call(b'config get cluster\n'),
     ])
+
+
+@patch('django_elasticache.cluster_utils.Telnet')
+def test_no_configuration_protocol_support_with_errors_ignored(Telnet):
+    client = Telnet.return_value
+    client.read_until.side_effect = TEST_PROTOCOL_4_READ_UNTIL
+    client.expect.side_effect = TEST_PROTOCOL_4_EXPECT
+    info = get_cluster_info('test', 0, ignore_cluster_errors=True)
+    client.write.assert_has_calls([
+        call(b'version\n'),
+        call(b'config get cluster\n'),
+    ])
+    eq_(info['version'], '1.4.34')
+    eq_(info['nodes'], ['test:0'])
+
+
+@raises(WrongProtocolData)
+@patch('django_elasticache.cluster_utils.Telnet')
+def test_no_configuration_protocol_support_with_errors(Telnet):
+    client = Telnet.return_value
+    client.read_until.side_effect = TEST_PROTOCOL_4_READ_UNTIL
+    client.expect.side_effect = TEST_PROTOCOL_4_EXPECT
+    get_cluster_info('test', 0, ignore_cluster_errors=False)
+    client.write.assert_has_calls([
+        call(b'version\n'),
+        call(b'config get cluster\n'),
+    ])