diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..d3a512c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +--- +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: daily + time: '09:00' + timezone: Asia/Tokyo + open-pull-requests-limit: 10 diff --git a/.gitignore b/.gitignore index b288303..9adce58 100644 --- a/.gitignore +++ b/.gitignore @@ -1,37 +1,101 @@ +### https://raw.github.com/github/gitignore/f57304e9762876ae4c9b02867ed0cb887316387e/python.gitignore + +# Byte-compiled / optimized / DLL files +__pycache__/ *.py[cod] +*$py.class # C extensions *.so -# Packages -*.egg -*.egg-info -dist -build -eggs -parts -bin -var -sdist -develop-eggs +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ .installed.cfg -lib -lib64 -__pycache__ +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec # Installer logs pip-log.txt +pip-delete-this-directory.txt # Unit test / coverage reports +htmlcov/ +.tox/ .coverage -.tox +.coverage.* +.cache nosetests.xml +coverage.xml +*,cover +.hypothesis/ # Translations *.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + -# Mr Developer -.mr.developer.cfg -.project -.pydevproject -.idea diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..e1bae10 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,43 @@ +sudo: false +language: python +python: + - 3.6 + - 3.7 + - 3.8 +cache: pip +env: + - DJANGO=2.2 + - DJANGO=3.0 + - DJANGO=3.1 + - DJANGO=dev + +matrix: + include: + - python: 3.8 + env: TOXENV=flake8 + - python: 3.8 + env: TOXENV=isort + - python: 3.8 + env: TOXENV=readme + - python: 3.8 + env: TOXENV=check-manifest + allow_failures: + - env: DJANGO=dev +install: +- travis_retry python3 -m pip install tox-travis +script: +- tox +deploy: + provider: pypi + username: __token__ + password: + secure: "HBbDdcrrzRrSXbQGz3cz24/xNIW77tVlHQ+D6DS5raOSV7en5CPaiWZ50YkMQa08nfBOR1RBT8JivrC05qz1+TqMwOz0+rinFeGPmu9gLY8f+1kx0UZnmXG7GHsm6dBGOP6nIHA40+rtK/YLuhskdd2hEBIOAuAxXuCSd1UQ/3bj/nuspDwcot9OdJiyn+Xl+cjI39Gt8WlF7ZBjgFRw1hNSIJ2f0or38zf+SSDY6QgKiJZCn+4sooy0assCms84LSKumSl7Ya50IWwBGJgJaNPxikqgTf3rDP9bT25OLtJw78zJnO4QC2PJ9Y7j2tzeYkelHJM794uAtKNaW1SanlYdBXQoE7m1CE8DszZxCtN/siuz+30whFJ885pPaSN3m5uJct/v9GxBEAHtoCrwwdfrTvzVCU43PGwPZvO768k465rRGUnBInLybO5hTq6myajecZB2WhVXMh7/KMcXlVtnD5PvIhxLkLPVIGBK+1wFfvH/4ijZl8Dhz5PXP53AcX/Ohdli7+kd9DQEvieE2bhDpBaJq2r7B4etW/bjz76GQ704uzuF5JXXEibaT1fox8nqJzghuWfcXJvoTa20iR1quuHg0Z4qBIYh76CUXafi4iIZ0fCcIDVPWM7KSp1rvt4BgCdi1csQdTasw0l0WGoKdOSu+LlNSrqzPLoXQlE=" + distributions: sdist bdist_wheel + skip_upload_docs: true + on: + repo: harikitech/django-elastipymemcache + tags: true + python: 3.8 + condition: "$DJANGO = 3.0" +notifications: + email: false diff --git a/LICENSE b/LICENSE index 9b5f8a5..e4835f6 100644 --- a/LICENSE +++ b/LICENSE @@ -18,3 +18,4 @@ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/MANIFEST.in b/MANIFEST.in index 4c4c31e..089a35f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,13 @@ +include LICENSE include README.rst -include MANIFEST.in -include setup.py \ No newline at end of file + +exclude requirements.txt +exclude tox.ini + +global-exclude __pycache__ +global-exclude *.py[co] + +prune .github +prune tests + +graft django_elastipymemcache diff --git a/README.rst b/README.rst index 644a037..587d806 100644 --- a/README.rst +++ b/README.rst @@ -1,31 +1,35 @@ -Amazon ElastiCache backend for Django -===================================== +======================= +django-elastipymemcache +======================= + +.. index: README +.. image:: https://travis-ci.org/harikitech/django-elastipymemcache.svg?branch=master + :target: https://travis-ci.org/harikitech/django-elastipymemcache +.. image:: https://codecov.io/gh/harikitech/django-elastipymemcache/branch/master/graph/badge.svg + :target: https://codecov.io/gh/harikitech/django-elastipymemcache + +Purpose +------- Simple Django cache backend for Amazon ElastiCache (memcached based). It uses -`pylibmc `_ and sets up a connection to each +`pymemcache `_ and sets up a connection to each node in the cluster using `auto discovery `_. - +Originally forked from `django-elasticache `_. Requirements ------------ -* pylibmc -* Django 1.5+. - -It was written and tested on Python 2.7 and 3.4. +* pymemcache +* Django>=2.2 +* django-pymemcache>=1.0 Installation ------------ -Get it from `pypi `_:: - - pip install django-elasticache - -or `github `_:: - - pip install -e git://github.com/gusdan/django-elasticache.git#egg=django-elasticache +Get it from `pypi `_:: + pip install django-elastipymemcache Usage ----- @@ -34,95 +38,15 @@ Your cache backend should look something like this:: CACHES = { 'default': { - 'BACKEND': 'django_elasticache.memcached.ElastiCache', - 'LOCATION': 'cache-c.draaaf.cfg.use1.cache.amazonaws.com:11211', - 'OPTIONS' { - 'IGNORE_CLUSTER_ERRORS': [True,False], - }, - } - } - -By the first call to cache it connects to cluster (using ``LOCATION`` param), -gets list of all nodes and setup pylibmc client using full -list of nodes. As result your cache will work with all nodes in cluster and -automatically detect new nodes in cluster. List of nodes are stored in class-level -cached, so any changes in cluster take affect only after restart of working process. -But if you're using gunicorn or mod_wsgi you usually have max_request settings which -restart process after some count of processed requests, so auto discovery will work -fine. - -The ``IGNORE_CLUSTER_ERRORS`` option is useful when ``LOCATION`` doesn't have support -for ``config get cluster``. When set to ``True``, and ``config get cluster`` fails, -it returns a list of a single node with the same endpoint supplied to ``LOCATION``. - -Django-elasticache changes default pylibmc params to increase performance. - -Another solutions ------------------ - -ElastiCache provides memcached interface so there are three solution of using it: - -1. Memcached configured with location = Configuration Endpoint -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In this case your application -will randomly connect to nodes in cluster and cache will be used with not optimal -way. At some moment you will be connected to first node and set item. Minute later -you will be connected to another node and will not able to get this item. - - :: - - CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', - 'LOCATION': 'cache.gasdbp.cfg.use1.cache.amazonaws.com:11211', + 'BACKEND': 'django_elastipymemcache.backend.ElastiPymemcache', + 'LOCATION': '[configuration endpoint]:11211', + 'OPTIONS': { + 'ignore_exc': True, # pymemcache Client params + 'ignore_cluster_errors': True, # ignore get cluster info error + } } } - -2. Memcached configured with all nodes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It will work fine, memcache client will -separate items between all nodes and will balance loading on client side. You will -have problems only after adding new nodes or delete old nodes. In this case you should -add new nodes manually and don't forget update your app after all changes on AWS. - - :: - - CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', - 'LOCATION': [ - 'cache.gqasdbp.0001.use1.cache.amazonaws.com:11211', - 'cache.gqasdbp.0002.use1.cache.amazonaws.com:11211', - ] - } - } - - -3. Use django-elasticache -~~~~~~~~~~~~~~~~~~~~~~~~~ - -It will connect to cluster and retrieve ip addresses -of all nodes and configure memcached to use all nodes. - - :: - - CACHES = { - 'default': { - 'BACKEND': 'django_elasticache.memcached.ElastiCache', - 'LOCATION': 'cache-c.draaaf.cfg.use1.cache.amazonaws.com:11211', - } - } - - -Difference between setup with nodes list (django-elasticache) and -connection to only one configuration Endpoint (using dns routing) you can see on -this graph: - -.. image:: https://raw.github.com/gusdan/django-elasticache/master/docs/images/get%20operation%20in%20cluster.png - Testing ------- diff --git a/django_elasticache/cluster_utils.py b/django_elasticache/cluster_utils.py deleted file mode 100644 index 60594ca..0000000 --- a/django_elasticache/cluster_utils.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -utils for discovery cluster -""" -from distutils.version import StrictVersion -from django.utils.encoding import smart_text -import re -from telnetlib import Telnet - - -class WrongProtocolData(ValueError): - """ - Exception for raising when we get something unexpected - in telnet protocol - """ - def __init__(self, cmd, response): - super(WrongProtocolData, self).__init__( - 'Unexpected response {0} for command {1}'.format(response, cmd)) - - -def get_cluster_info(host, port, ignore_cluster_errors=False): - """ - return dict with info about nodes in cluster and current version - { - 'nodes': [ - 'IP:port', - 'IP:port', - ], - 'version': '1.4.4' - } - """ - client = Telnet(host, int(port)) - client.write(b'version\n') - res = client.read_until(b'\r\n').strip() - version_list = res.split(b' ') - if len(version_list) not in [2, 3] or version_list[0] != b'VERSION': - raise WrongProtocolData('version', res) - version = version_list[1] - if StrictVersion(smart_text(version)) >= StrictVersion('1.4.14'): - cmd = b'config get cluster\n' - else: - cmd = b'get AmazonElastiCache:cluster\n' - client.write(cmd) - regex_index, match_object, res = client.expect([ - re.compile(b'\n\r\nEND\r\n'), - re.compile(b'ERROR\r\n') - ]) - client.close() - - if res == b'ERROR\r\n' and ignore_cluster_errors: - return { - 'version': version, - 'nodes': [ - '{0}:{1}'.format(smart_text(host), - smart_text(port)) - ] - } - - ls = list(filter(None, re.compile(br'\r?\n').split(res))) - if len(ls) != 4: - raise WrongProtocolData(cmd, res) - - try: - version = int(ls[1]) - except ValueError: - raise WrongProtocolData(cmd, res) - nodes = [] - try: - for node in ls[2].split(b' '): - host, ip, port = node.split(b'|') - nodes.append('{0}:{1}'.format(smart_text(ip or host), - smart_text(port))) - except ValueError: - raise WrongProtocolData(cmd, res) - return { - 'version': version, - 'nodes': nodes - } diff --git a/django_elasticache/memcached.py b/django_elasticache/memcached.py deleted file mode 100644 index 0b0caf3..0000000 --- a/django_elasticache/memcached.py +++ /dev/null @@ -1,138 +0,0 @@ -""" -Backend for django cache -""" -import socket -import logging - -from functools import wraps -from django.core.cache import InvalidCacheBackendError -from django.core.cache.backends.memcached import PyLibMCCache -from .cluster_utils import get_cluster_info - - -log = logging.getLogger('django.elasticache') - - -def invalidate_cache_after_error(f): - """ - catch any exception and invalidate internal cache with list of nodes - """ - @wraps(f) - def wrapper(self, *args, **kwds): - try: - return f(self, *args, **kwds) - except Exception as e: - log.warning('MemcachedError: %s', e, exc_info=True) - self.clear_cluster_nodes_cache() - return None # Treat as a cache miss - return wrapper - - -class ElastiCache(PyLibMCCache): - """ - backend for Amazon ElastiCache (memcached) with auto discovery mode - it used pylibmc in binary mode - """ - def __init__(self, server, params): - self.update_params(params) - super(ElastiCache, self).__init__(server, params) - if len(self._servers) > 1: - raise InvalidCacheBackendError( - 'ElastiCache should be configured with only one server ' - '(Configuration Endpoint)') - if len(self._servers[0].split(':')) != 2: - raise InvalidCacheBackendError( - 'Server configuration should be in format IP:port') - - self._ignore_cluster_errors = self._options.get( - 'IGNORE_CLUSTER_ERRORS', False) - - def update_params(self, params): - """ - update connection params to maximize performance - """ - if not params.get('BINARY', True): - raise Warning('To increase performance please use ElastiCache' - ' in binary mode') - else: - params['BINARY'] = True # patch params, set binary mode - if 'OPTIONS' not in params: - # set special 'behaviors' pylibmc attributes - params['OPTIONS'] = { - 'tcp_nodelay': True, - 'ketama': True - } - - def clear_cluster_nodes_cache(self): - """clear internal cache with list of nodes in cluster""" - if hasattr(self, '_cluster_nodes_cache'): - del self._cluster_nodes_cache - - def get_cluster_nodes(self): - """ - return list with all nodes in cluster - """ - if not hasattr(self, '_cluster_nodes_cache'): - server, port = self._servers[0].split(':') - try: - self._cluster_nodes_cache = ( - get_cluster_info(server, port, - self._ignore_cluster_errors)['nodes']) - except (socket.gaierror, socket.timeout) as err: - raise Exception('Cannot connect to cluster {0} ({1})'.format( - self._servers[0], err - )) - return self._cluster_nodes_cache - - @property - def _cache(self): - # PylibMC uses cache options as the 'behaviors' attribute. - # It also needs to use threadlocals, because some versions of - # PylibMC don't play well with the GIL. - - # instance to store cached version of client - # in Django 1.7 use self - # in Django < 1.7 use thread local - container = getattr(self, '_local', self) - client = getattr(container, '_client', None) - if client: - return client - - client = self._lib.Client(self.get_cluster_nodes()) - if self._options: - # In Django 1.11, all behaviors are shifted into a behaviors dict - # Attempt to get from there, and fall back to old behavior if the behaviors - # key does not exist - client.behaviors = self._options.get('behaviors', self._options) - - container._client = client - - return client - - @invalidate_cache_after_error - def add(self, *args, **kwargs): - return super(ElastiCache, self).add(*args, **kwargs) - - @invalidate_cache_after_error - def get(self, *args, **kwargs): - return super(ElastiCache, self).get(*args, **kwargs) - - @invalidate_cache_after_error - def get_many(self, *args, **kwargs): - return super(ElastiCache, self).get_many(*args, **kwargs) - - @invalidate_cache_after_error - def set(self, *args, **kwargs): - return super(ElastiCache, self).set(*args, **kwargs) - - @invalidate_cache_after_error - def set_many(self, *args, **kwargs): - return super(ElastiCache, self).set_many(*args, **kwargs) - - @invalidate_cache_after_error - def delete(self, *args, **kwargs): - return super(ElastiCache, self).delete(*args, **kwargs) - - @invalidate_cache_after_error - def delete_many(self, *args, **kwargs): - return super(ElastiCache, self).delete_many(*args, **kwargs) diff --git a/django_elasticache/__init__.py b/django_elastipymemcache/__init__.py similarity index 67% rename from django_elasticache/__init__.py rename to django_elastipymemcache/__init__.py index 65f9591..f40c6d0 100644 --- a/django_elasticache/__init__.py +++ b/django_elastipymemcache/__init__.py @@ -1,2 +1,2 @@ -VERSION = (1, 0, 4) +VERSION = (2, 0, 4) __version__ = '.'.join(map(str, VERSION)) diff --git a/django_elastipymemcache/backend.py b/django_elastipymemcache/backend.py new file mode 100644 index 0000000..294f647 --- /dev/null +++ b/django_elastipymemcache/backend.py @@ -0,0 +1,140 @@ +""" +Backend for django cache +""" +import logging +import socket +from functools import wraps + +from django.core.cache import InvalidCacheBackendError +from django.core.cache.backends.memcached import BaseMemcachedCache +from djpymemcache import client as djpymemcache_client + +from .client import ConfigurationEndpointClient + +logger = logging.getLogger(__name__) + + +def invalidate_cache_after_error(f): + """ + Catch any exception and invalidate internal cache with list of nodes + """ + @wraps(f) + def wrapper(self, *args, **kwds): + try: + return f(self, *args, **kwds) + except Exception as e: + logger.warning('MemcachedError: %s', e, exc_info=True) + self.clear_cluster_nodes_cache() + return None # Treat as a cache miss + return wrapper + + +class ElastiPymemcache(BaseMemcachedCache): + """ + Backend for Amazon ElastiCache (memcached) with auto discovery mode + it used pymemcache + """ + def __init__(self, server, params): + params['OPTIONS'] = params.get('OPTIONS', {}) + params['OPTIONS'].setdefault('ignore_exc', True) + + self._cluster_timeout = params['OPTIONS'].pop( + 'cluster_timeout', + socket._GLOBAL_DEFAULT_TIMEOUT, + ) + self._ignore_cluster_errors = params['OPTIONS'].pop( + 'ignore_cluster_errors', + False, + ) + + super().__init__( + server, + params, + library=djpymemcache_client, + value_not_found_exception=ValueError, + ) + + if len(self._servers) > 1: + raise InvalidCacheBackendError( + 'ElastiCache should be configured with only one server ' + '(Configuration Endpoint)', + ) + try: + host, port = self._servers[0].split(':') + port = int(port) + except ValueError: + raise InvalidCacheBackendError( + 'Server configuration should be in format IP:Port', + ) + + self.configuration_endpoint_client = ConfigurationEndpointClient( + (host, port), + ignore_cluster_errors=self._ignore_cluster_errors, + **self._options, + ) + + def clear_cluster_nodes_cache(self): + """Clear internal cache with list of nodes in cluster""" + if hasattr(self, '_client'): + del self._client + + def get_cluster_nodes(self): + try: + return self.configuration_endpoint_client \ + .get_cluster_info()['nodes'] + except ( + OSError, + socket.gaierror, + socket.timeout, + ) as e: + logger.warn( + 'Cannot connect to cluster %s, err: %s', + self.configuration_endpoint_client.server, + e, + ) + return [] + + @property + def _cache(self): + if getattr(self, '_client', None) is None: + self._client = self._lib.Client( + self.get_cluster_nodes(), + **self._options, + ) + return self._client + + @invalidate_cache_after_error + def add(self, *args, **kwargs): + return super().add(*args, **kwargs) + + @invalidate_cache_after_error + def get(self, *args, **kwargs): + return super().get(*args, **kwargs) + + @invalidate_cache_after_error + def set(self, *args, **kwargs): + return super().set(*args, **kwargs) + + @invalidate_cache_after_error + def delete(self, *args, **kwargs): + return super().delete(*args, **kwargs) + + @invalidate_cache_after_error + def get_many(self, *args, **kwargs): + return super().get_many(*args, **kwargs) + + @invalidate_cache_after_error + def set_many(self, *args, **kwargs): + return super().set_many(*args, **kwargs) + + @invalidate_cache_after_error + def delete_many(self, *args, **kwargs): + return super().delete_many(*args, **kwargs) + + @invalidate_cache_after_error + def incr(self, *args, **kwargs): + return super().incr(*args, **kwargs) + + @invalidate_cache_after_error + def decr(self, *args, **kwargs): + return super().decr(*args, **kwargs) diff --git a/django_elastipymemcache/client.py b/django_elastipymemcache/client.py new file mode 100644 index 0000000..f9243b7 --- /dev/null +++ b/django_elastipymemcache/client.py @@ -0,0 +1,79 @@ +import logging +from distutils.version import StrictVersion + +from django.utils.encoding import smart_text +from pymemcache.client.base import Client, _readline +from pymemcache.exceptions import MemcacheUnknownError + +logger = logging.getLogger(__name__) + + +class ConfigurationEndpointClient(Client): + # https://docs.aws.amazon.com/AmazonElastiCache/latest/mem-ug/AutoDiscovery.AddingToYourClientLibrary.html + + def __init__(self, *args, ignore_cluster_errors=False, **kwargs): + client = super().__init__(*args, **kwargs) + self.ignore_cluster_errors = ignore_cluster_errors + return client + + def _get_cluster_info_cmd(self): + if StrictVersion(smart_text(self.version())) < StrictVersion('1.4.14'): + return b'get AmazonElastiCache:cluster\r\n' + return b'config get cluster\r\n' + + def _extract_cluster_info(self, line): + raw_version, raw_nodes, _ = line.split(b'\n') + nodes = [] + for raw_node in raw_nodes.split(b' '): + host, ip, port = raw_node.split(b'|') + nodes.append('{host}:{port}'.format( + host=smart_text(ip or host), + port=int(port) + )) + return { + 'version': int(raw_version), + 'nodes': nodes, + } + + def _fetch_cluster_info_cmd(self, cmd, name): + if self.sock is None: + self._connect() + self.sock.sendall(cmd) + + buf = b'' + result = {} + number_of_line = 0 + + while True: + buf, line = _readline(self.sock, buf) + self._raise_errors(line, name) + if line == b'END': + if number_of_line != 2: + raise MemcacheUnknownError('Wrong response') + return result + if number_of_line == 1: + try: + result = self._extract_cluster_info(line) + except ValueError: + raise MemcacheUnknownError('Wrong format: {line}'.format( + line=line, + )) + number_of_line += 1 + + def get_cluster_info(self): + cmd = self._get_cluster_info_cmd() + try: + return self._fetch_cluster_info_cmd(cmd, 'config cluster') + except Exception as e: + if self.ignore_cluster_errors: + logger.warn('Failed to get cluster: %s', e) + return { + 'version': None, + 'nodes': [ + '{host}:{port:d}'.format( + host=self.server[0], + port=int(self.server[1]), + ), + ] + } + raise diff --git a/docs/images/get operation in cluster.png b/docs/images/get operation in cluster.png deleted file mode 100644 index a85f01d..0000000 Binary files a/docs/images/get operation in cluster.png and /dev/null differ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..666e122 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +check-manifest==0.42 +coverage==5.2.1 +flake8==3.8.3 +isort==5.5.0 +mock==4.0.2 +nose==1.3.7 +pymemcache==3.3.0 +readme-renderer==26.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..ca864d7 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,17 @@ +[wheel] +universal = 1 + +[isort] +include_trailing_comma=True +line_length=80 +multi_line_output=3 +not_skip=__init__.py +known_first_party=django_elastipymemcache + +[check-manifest] +ignore = + *.swp + +[coverage:run] +branch = True +omit = tests/* diff --git a/setup.py b/setup.py index f65143c..a914629 100644 --- a/setup.py +++ b/setup.py @@ -1,30 +1,42 @@ -from setuptools import setup +#!/usr/bin/env python +# -*- encoding: utf-8 -*- -import django_elasticache +import io +from setuptools import find_packages, setup + +import django_elastipymemcache setup( - name='django-elasticache', - version=django_elasticache.__version__, + name='django-elastipymemcache', + version=django_elastipymemcache.__version__, description='Django cache backend for Amazon ElastiCache (memcached)', - long_description=open('README.rst').read(), - author='Danil Gusev', - platforms='any', - author_email='danil.gusev@gmail.com', - url='http://github.com/gusdan/django-elasticache', + keywords='elasticache amazon cache pymemcache memcached aws', + author='HarikiTech', + author_email='harikitech+noreply@googlegroups.com', + url='http://github.com/harikitech/django-elastipymemcache', license='MIT', - keywords='elasticache amazon cache pylibmc memcached aws', - packages=['django_elasticache'], - install_requires=['pylibmc', 'Django>=1.3'], + long_description=io.open('README.rst').read(), + platforms='any', + zip_safe=False, classifiers=[ 'Development Status :: 4 - Beta', 'Environment :: Web Environment', - 'Environment :: Web Environment :: Mozilla', - 'Framework :: Django', + 'Framework :: Django :: 2.2', + 'Framework :: Django :: 3.0', + 'Framework :: Django :: 3.1', 'Intended Audience :: Developers', - 'License :: OSI Approved :: BSD License', + 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', - 'Programming Language :: Python', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Topic :: Software Development :: Libraries :: Python Modules', ], + packages=find_packages(exclude=('tests',)), + include_package_data=True, + install_requires=[ + 'django-pymemcache>=1.0', + 'Django>=2.2', + ], ) diff --git a/tests/test_backend.py b/tests/test_backend.py index 9b6f425..628f263 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -1,88 +1,82 @@ -from django.conf import global_settings, settings -from nose.tools import eq_, raises -import sys -if sys.version < '3': - from mock import patch, Mock -else: - from unittest.mock import patch, Mock - - -# Initialize django 1.7 -settings.configure() -global_settings.configured = True +from unittest.mock import Mock, patch +import django +from django.core.cache import InvalidCacheBackendError +from nose.tools import eq_, raises -@patch('django.conf.settings', global_settings) -def test_patch_params(): - from django_elasticache.memcached import ElastiCache - params = {} - ElastiCache('qew:12', params) - eq_(params['BINARY'], True) - eq_(params['OPTIONS']['tcp_nodelay'], True) - eq_(params['OPTIONS']['ketama'], True) +from django_elastipymemcache.client import ConfigurationEndpointClient -@raises(Exception) -@patch('django.conf.settings', global_settings) -def test_wrong_params(): - from django_elasticache.memcached import ElastiCache - ElastiCache('qew', {}) +@raises(InvalidCacheBackendError) +def test_multiple_servers(): + from django_elastipymemcache.backend import ElastiPymemcache + ElastiPymemcache('h1:0,h2:0', {}) -@raises(Warning) -@patch('django.conf.settings', global_settings) -def test_wrong_params_warning(): - from django_elasticache.memcached import ElastiCache - ElastiCache('qew', {'BINARY': False}) +@raises(InvalidCacheBackendError) +def test_wrong_server_format(): + from django_elastipymemcache.backend import ElastiPymemcache + ElastiPymemcache('h', {}) -@patch('django.conf.settings', global_settings) -@patch('django_elasticache.memcached.get_cluster_info') +@patch.object(ConfigurationEndpointClient, 'get_cluster_info') def test_split_servers(get_cluster_info): - from django_elasticache.memcached import ElastiCache - backend = ElastiCache('h:0', {}) - servers = ['h1:p', 'h2:p'] + from django_elastipymemcache.backend import ElastiPymemcache + backend = ElastiPymemcache('h:0', {}) + servers = [('h1', 0), ('h2', 0)] get_cluster_info.return_value = { 'nodes': servers } backend._lib.Client = Mock() assert backend._cache - get_cluster_info.assert_called_once_with('h', '0', False) - backend._lib.Client.assert_called_once_with(servers) + get_cluster_info.assert_called() + backend._lib.Client.assert_called_once_with( + servers, + ignore_exc=True, + ) -@patch('django.conf.settings', global_settings) -@patch('django_elasticache.memcached.get_cluster_info') +@patch.object(ConfigurationEndpointClient, 'get_cluster_info') def test_node_info_cache(get_cluster_info): - from django_elasticache.memcached import ElastiCache - servers = ['h1:p', 'h2:p'] + from django_elastipymemcache.backend import ElastiPymemcache + servers = ['h1:0', 'h2:0'] get_cluster_info.return_value = { 'nodes': servers } - backend = ElastiCache('h:0', {}) + backend = ElastiPymemcache('h:0', {}) backend._lib.Client = Mock() backend.set('key1', 'val') backend.get('key1') backend.set('key2', 'val') backend.get('key2') - backend._lib.Client.assert_called_once_with(servers) + backend._lib.Client.assert_called_once_with( + servers, + ignore_exc=True, + ) eq_(backend._cache.get.call_count, 2) eq_(backend._cache.set.call_count, 2) - get_cluster_info.assert_called_once_with('h', '0', False) + get_cluster_info.assert_called_once() -@patch('django.conf.settings', global_settings) -@patch('django_elasticache.memcached.get_cluster_info') +@patch.object(ConfigurationEndpointClient, 'get_cluster_info') +def test_failed_to_connect_servers(get_cluster_info): + from django_elastipymemcache.backend import ElastiPymemcache + backend = ElastiPymemcache('h:0', {}) + get_cluster_info.side_effect = OSError() + eq_(backend.get_cluster_nodes(), []) + + +@patch.object(ConfigurationEndpointClient, 'get_cluster_info') def test_invalidate_cache(get_cluster_info): - from django_elasticache.memcached import ElastiCache - servers = ['h1:p', 'h2:p'] + from django_elastipymemcache.backend import ElastiPymemcache + servers = ['h1:0', 'h2:0'] get_cluster_info.return_value = { 'nodes': servers } - backend = ElastiCache('h:0', {}) + backend = ElastiPymemcache('h:0', {}) backend._lib.Client = Mock() assert backend._cache backend._cache.get = Mock() @@ -99,4 +93,149 @@ def test_invalidate_cache(get_cluster_info): except Exception: pass eq_(backend._cache.get.call_count, 2) - eq_(get_cluster_info.call_count, 2) + eq_(get_cluster_info.call_count, 3) + + +@patch.object(ConfigurationEndpointClient, 'get_cluster_info') +def test_client_add(get_cluster_info): + from django_elastipymemcache.backend import ElastiPymemcache + + servers = ['h1:0', 'h2:0'] + get_cluster_info.return_value = { + 'nodes': servers + } + + backend = ElastiPymemcache('h:0', {}) + ret = backend.add('key1', 'value1') + eq_(ret, False) + + +@patch.object(ConfigurationEndpointClient, 'get_cluster_info') +def test_client_delete(get_cluster_info): + from django_elastipymemcache.backend import ElastiPymemcache + + servers = ['h1:0', 'h2:0'] + get_cluster_info.return_value = { + 'nodes': servers + } + + backend = ElastiPymemcache('h:0', {}) + ret = backend.delete('key1') + if django.get_version() >= '3.1': + eq_(ret, False) + else: + eq_(ret, None) + + +@patch.object(ConfigurationEndpointClient, 'get_cluster_info') +def test_client_get_many(get_cluster_info): + from django_elastipymemcache.backend import ElastiPymemcache + + servers = ['h1:0', 'h2:0'] + get_cluster_info.return_value = { + 'nodes': servers + } + + backend = ElastiPymemcache('h:0', {}) + ret = backend.get_many(['key1']) + eq_(ret, {}) + + # When server does not found... + with patch('pymemcache.client.hash.HashClient._get_client') as p: + p.return_value = None + ret = backend.get_many(['key2']) + eq_(ret, {}) + + with patch('pymemcache.client.hash.HashClient._safely_run_func') as p2: + p2.return_value = { + ':1:key3': 1509111630.048594 + } + + ret = backend.get_many(['key3']) + eq_(ret, {'key3': 1509111630.048594}) + + # If False value is included, ignore it. + with patch('pymemcache.client.hash.HashClient.get_many') as p: + p.return_value = { + ':1:key1': 1509111630.048594, + ':1:key2': False, + ':1:key3': 1509111630.058594, + } + ret = backend.get_many(['key1', 'key2', 'key3']) + eq_( + ret, + { + 'key1': 1509111630.048594, + 'key3': 1509111630.058594 + }, + ) + + with patch('pymemcache.client.hash.HashClient.get_many') as p: + p.return_value = { + ':1:key1': None, + ':1:key2': 1509111630.048594, + ':1:key3': False, + } + ret = backend.get_many(['key1', 'key2', 'key3']) + eq_( + ret, + { + 'key2': 1509111630.048594, + }, + ) + + +@patch.object(ConfigurationEndpointClient, 'get_cluster_info') +def test_client_set_many(get_cluster_info): + from django_elastipymemcache.backend import ElastiPymemcache + + servers = ['h1:0', 'h2:0'] + get_cluster_info.return_value = { + 'nodes': servers + } + + backend = ElastiPymemcache('h:0', {}) + ret = backend.set_many({'key1': 'value1', 'key2': 'value2'}) + eq_(ret, ['key1', 'key2']) + + +@patch.object(ConfigurationEndpointClient, 'get_cluster_info') +def test_client_delete_many(get_cluster_info): + from django_elastipymemcache.backend import ElastiPymemcache + + servers = ['h1:0', 'h2:0'] + get_cluster_info.return_value = { + 'nodes': servers + } + + backend = ElastiPymemcache('h:0', {}) + ret = backend.delete_many(['key1', 'key2']) + eq_(ret, None) + + +@patch.object(ConfigurationEndpointClient, 'get_cluster_info') +def test_client_incr(get_cluster_info): + from django_elastipymemcache.backend import ElastiPymemcache + + servers = ['h1:0', 'h2:0'] + get_cluster_info.return_value = { + 'nodes': servers + } + + backend = ElastiPymemcache('h:0', {}) + ret = backend.incr('key1', 1) + eq_(ret, False) + + +@patch.object(ConfigurationEndpointClient, 'get_cluster_info') +def test_client_decr(get_cluster_info): + from django_elastipymemcache.backend import ElastiPymemcache + + servers = ['h1:0', 'h2:0'] + get_cluster_info.return_value = { + 'nodes': servers + } + + backend = ElastiPymemcache('h:0', {}) + ret = backend.decr('key1', 1) + eq_(ret, False) diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 0000000..724e0ad --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,110 @@ +import collections +from unittest.mock import call, patch + +from nose.tools import eq_, raises +from pymemcache.exceptions import ( + MemcacheUnknownCommandError, + MemcacheUnknownError, +) + +from django_elastipymemcache.client import ConfigurationEndpointClient + +EXAMPLE_RESPONSE = [ + b'CONFIG cluster 0 147\r\n', + b'12\n' + b'myCluster.pc4ldq.0001.use1.cache.amazonaws.com|10.82.235.120|11211 ' + b'myCluster.pc4ldq.0002.use1.cache.amazonaws.com|10.80.249.27|11211\n\r\n', + b'END\r\n', +] + + +@patch('socket.socket') +def test_get_cluster_info(socket): + recv_bufs = collections.deque([ + b'VERSION 1.4.14\r\n', + ] + EXAMPLE_RESPONSE) + + client = socket.return_value + client.recv.side_effect = lambda *args, **kwargs: recv_bufs.popleft() + cluster_info = ConfigurationEndpointClient(('h', 0)).get_cluster_info() + eq_(cluster_info['nodes'], ['10.82.235.120:11211', '10.80.249.27:11211']) + client.sendall.assert_has_calls([ + call(b'version\r\n'), + call(b'config get cluster\r\n'), + ]) + + +@patch('socket.socket') +def test_get_cluster_info_before_1_4_13(socket): + recv_bufs = collections.deque([ + b'VERSION 1.4.13\r\n', + ] + EXAMPLE_RESPONSE) + + client = socket.return_value + client.recv.side_effect = lambda *args, **kwargs: recv_bufs.popleft() + cluster_info = ConfigurationEndpointClient(('h', 0)).get_cluster_info() + eq_(cluster_info['nodes'], ['10.82.235.120:11211', '10.80.249.27:11211']) + client.sendall.assert_has_calls([ + call(b'version\r\n'), + call(b'get AmazonElastiCache:cluster\r\n'), + ]) + + +@raises(MemcacheUnknownCommandError) +@patch('socket.socket') +def test_no_configuration_protocol_support_with_errors(socket): + recv_bufs = collections.deque([ + b'VERSION 1.4.13\r\n', + b'ERROR\r\n', + ]) + + client = socket.return_value + client.recv.side_effect = lambda *args, **kwargs: recv_bufs.popleft() + ConfigurationEndpointClient(('h', 0)).get_cluster_info() + + +@raises(MemcacheUnknownError) +@patch('socket.socket') +def test_cannot_parse_version(socket): + recv_bufs = collections.deque([ + b'VERSION 1.4.34\r\n', + b'CONFIG cluster 0 147\r\n', + b'fail\nhost|ip|11211 host|ip|11211\n\r\n', + b'END\r\n', + ]) + + client = socket.return_value + client.recv.side_effect = lambda *args, **kwargs: recv_bufs.popleft() + ConfigurationEndpointClient(('h', 0)).get_cluster_info() + + +@raises(MemcacheUnknownError) +@patch('socket.socket') +def test_cannot_parse_nodes(socket): + recv_bufs = collections.deque([ + b'VERSION 1.4.34\r\n', + b'CONFIG cluster 0 147\r\n', + b'1\nfail\n\r\n', + b'END\r\n', + ]) + + client = socket.return_value + client.recv.side_effect = lambda *args, **kwargs: recv_bufs.popleft() + ConfigurationEndpointClient(('h', 0)).get_cluster_info() + + +@patch('socket.socket') +def test_ignore_erros(socket): + recv_bufs = collections.deque([ + b'VERSION 1.4.34\r\n', + b'fail\nfail\n\r\n', + b'END\r\n', + ]) + + client = socket.return_value + client.recv.side_effect = lambda *args, **kwargs: recv_bufs.popleft() + cluster_info = ConfigurationEndpointClient( + ('h', 0), + ignore_cluster_errors=True, + ).get_cluster_info() + eq_(cluster_info['nodes'], ['h:0']) diff --git a/tests/test_protocol.py b/tests/test_protocol.py deleted file mode 100644 index 7232a1a..0000000 --- a/tests/test_protocol.py +++ /dev/null @@ -1,125 +0,0 @@ -from django_elasticache.cluster_utils import ( - get_cluster_info, WrongProtocolData) -from nose.tools import eq_, raises -import sys -if sys.version < '3': - from mock import patch, call, MagicMock -else: - from unittest.mock import patch, call, MagicMock - - -TEST_PROTOCOL_1_READ_UNTIL = [ - b'VERSION 1.4.14', -] - -TEST_PROTOCOL_1_EXPECT = [ - (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n'), # NOQA -] - -TEST_PROTOCOL_2_READ_UNTIL = [ - b'VERSION 1.4.13', -] - -TEST_PROTOCOL_2_EXPECT = [ - (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n'), # NOQA -] - -TEST_PROTOCOL_3_READ_UNTIL = [ - b'VERSION 1.4.14 (Ubuntu)', -] - -TEST_PROTOCOL_3_EXPECT = [ - (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|port host||port\n\r\nEND\r\n'), # NOQA -] - -TEST_PROTOCOL_4_READ_UNTIL = [ - b'VERSION 1.4.34', -] - -TEST_PROTOCOL_4_EXPECT = [ - (0, None, b'ERROR\r\n'), -] - - -@patch('django_elasticache.cluster_utils.Telnet') -def test_happy_path(Telnet): - client = Telnet.return_value - client.read_until.side_effect = TEST_PROTOCOL_1_READ_UNTIL - client.expect.side_effect = TEST_PROTOCOL_1_EXPECT - info = get_cluster_info('', 0) - eq_(info['version'], 1) - eq_(info['nodes'], ['ip:port', 'host:port']) - - -@raises(WrongProtocolData) -@patch('django_elasticache.cluster_utils.Telnet', MagicMock()) -def test_bad_protocol(): - get_cluster_info('', 0) - - -@patch('django_elasticache.cluster_utils.Telnet') -def test_last_versions(Telnet): - client = Telnet.return_value - client.read_until.side_effect = TEST_PROTOCOL_1_READ_UNTIL - client.expect.side_effect = TEST_PROTOCOL_1_EXPECT - get_cluster_info('', 0) - client.write.assert_has_calls([ - call(b'version\n'), - call(b'config get cluster\n'), - ]) - - -@patch('django_elasticache.cluster_utils.Telnet') -def test_prev_versions(Telnet): - client = Telnet.return_value - client.read_until.side_effect = TEST_PROTOCOL_2_READ_UNTIL - client.expect.side_effect = TEST_PROTOCOL_2_EXPECT - get_cluster_info('', 0) - client.write.assert_has_calls([ - call(b'version\n'), - call(b'get AmazonElastiCache:cluster\n'), - ]) - - -@patch('django_elasticache.cluster_utils.Telnet') -def test_ubuntu_protocol(Telnet): - client = Telnet.return_value - client.read_until.side_effect = TEST_PROTOCOL_3_READ_UNTIL - client.expect.side_effect = TEST_PROTOCOL_3_EXPECT - - try: - get_cluster_info('', 0) - except WrongProtocolData: - raise AssertionError('Raised WrongProtocolData with Ubuntu version.') - - client.write.assert_has_calls([ - call(b'version\n'), - call(b'config get cluster\n'), - ]) - - -@patch('django_elasticache.cluster_utils.Telnet') -def test_no_configuration_protocol_support_with_errors_ignored(Telnet): - client = Telnet.return_value - client.read_until.side_effect = TEST_PROTOCOL_4_READ_UNTIL - client.expect.side_effect = TEST_PROTOCOL_4_EXPECT - info = get_cluster_info('test', 0, ignore_cluster_errors=True) - client.write.assert_has_calls([ - call(b'version\n'), - call(b'config get cluster\n'), - ]) - eq_(info['version'], '1.4.34') - eq_(info['nodes'], ['test:0']) - - -@raises(WrongProtocolData) -@patch('django_elasticache.cluster_utils.Telnet') -def test_no_configuration_protocol_support_with_errors(Telnet): - client = Telnet.return_value - client.read_until.side_effect = TEST_PROTOCOL_4_READ_UNTIL - client.expect.side_effect = TEST_PROTOCOL_4_EXPECT - get_cluster_info('test', 0, ignore_cluster_errors=False) - client.write.assert_has_calls([ - call(b'version\n'), - call(b'config get cluster\n'), - ]) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..5929d8f --- /dev/null +++ b/tox.ini @@ -0,0 +1,55 @@ +[tox] +envlist = + py{36,37,38}-dj22, + py{36,37,38}-dj30, + py{36,37,38}-dj31, + py{36,37,38}-djdev, + flake8, + isort, + readme + check-manifest + +[travis:env] +DJANGO = + 2.2: dj22 + 3.0: dj30 + 3.1: dj31 + dev: djdev + +[testenv] +passenv = TOXENV CI TRAVIS TRAVIS_* CODECOV_* +deps = + dj22: Django<2.3 + dj30: Django<3.1 + dj31: Django<3.2 + django-pymemcache<2.0 + djdev: https://github.com/django/django/archive/master.tar.gz + -r{toxinidir}/requirements.txt + py38-dj31: codecov +setenv = + PYTHONPATH = {toxinidir} +commands = + coverage run --source=django_elastipymemcache -m nose --verbose + py38-dj31: coverage report + py38-dj31: coverage xml + py38-dj31: codecov + +[testenv:flake8] +basepython = python3.8 +commands = flake8 +deps = flake8 + +[testenv:isort] +basepython = python3.8 +commands = isort --verbose --check-only --diff django_elastipymemcache tests setup.py +deps = isort + +[testenv:readme] +basepython = python3.8 +commands = python setup.py check -r -s +deps = readme_renderer + +[testenv:check-manifest] +basepython = python3.8 +commands = check-manifest {toxinidir} +deps = check-manifest