diff --git a/.gitignore b/.gitignore index cf05cfd5..f3d430d7 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ MANIFEST .venv redis/ */_build/ +build/* diff --git a/.travis.yml b/.travis.yml index 281d95e5..cc13970d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: python python: - 2.7 - - 3.4 - 3.5 - 3.6 env: @@ -14,8 +13,6 @@ matrix: env: DJANGO_VERSION='>=2.0,<2.1' - python: 2.7 env: DJANGO_VERSION='>=2.1,<2.2' - - python: 3.4 - env: DJANGO_VERSION='>=2.1,<2.2' # command to run tests install: ./install_redis.sh script: make test DJANGO_VERSION=$DJANGO_VERSION diff --git a/README.rst b/README.rst index 2b795746..607a40ac 100644 --- a/README.rst +++ b/README.rst @@ -21,6 +21,16 @@ Docs can be found at http://django-redis-cache.readthedocs.org/en/latest/. Changelog ========= +2.0.0 +----- + +* Adds support for redis-py >= 3.0. +* Drops support for Redis 2.6. +* Drops support for Python 3.4. +* Removes custom ``expire`` method in lieu of Django's ``touch``. +* Removes ``CacheKey`` in favor of string literals. + + 1.8.0 ----- diff --git a/docs/api.rst b/docs/api.rst index 0f733059..9ea77209 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -93,6 +93,13 @@ Standard Django Cache API :param version: Version of key :type version: Integer or None +.. function:: touch(self, key, timeout): + + Updates the timeout on a key. + + :param key: Location of the value + :rtype: bool + Cache Methods Provided by django-redis-cache @@ -124,14 +131,22 @@ Cache Methods Provided by django-redis-cache :param version: Version of the keys -.. function:: get_or_set(self, key, func[, timeout=None]): +.. function:: get_or_set(self, key, func[, timeout=None, lock_timeout=None, stale_cache_timeout=None]): + + Get a value from the cache or call ``func`` to set it and return it. - Retrieves a key value from the cache and sets the value if it does not exist. + This implementation is slightly more advanced that Django's. It provides thundering herd + protection, which prevents multiple threads/processes from calling the value-generating + function at the same time. :param key: Location of the value :param func: Callable used to set the value if key does not exist. - :param timeout: Number of seconds to hold value in cache. + :param timeout: Time in seconds that value at key is considered fresh. :type timeout: Number of seconds or None + :param lock_timeout: Time in seconds that the lock will stay active and prevent other threads from acquiring the lock. + :type lock_timeout: Number of seconds or None + :param stale_cache_timeout: Time in seconds that the stale cache will remain after the key has expired. If ``None`` is specified, the stale value will remain indefinitely. + :type stale_cache_timeout: Number of seconds or None .. function:: reinsert_keys(self): @@ -147,12 +162,9 @@ Cache Methods Provided by django-redis-cache :param key: Location of the value :rtype: bool +.. function:: lock(self, key, timeout=None, sleep=0.1, blocking_timeout=None, thread_local=True) -.. function:: expire(self, key, timeout): - - Set the expire time on a key - - :param key: Location of the value - :rtype: bool + See docs for `redis-py`_. +.. _redis-py: https://redis-py.readthedocs.io/en/latest/_modules/redis/client.html#Redis.lock diff --git a/docs/intro_quick_start.rst b/docs/intro_quick_start.rst index f1b5384e..dd04b1cf 100644 --- a/docs/intro_quick_start.rst +++ b/docs/intro_quick_start.rst @@ -4,7 +4,7 @@ Intro and Quick Start Intro ===== -`django-redis-cache`_ is a cache backend for the `Django`_ webframework. It +`django-redis-cache`_ is a cache backend for the `Django`_ web framework. It uses the `redis`_ server, which is a in-memory key-value data structure server. Similar to the great `Memcached`_ in performance, it has several features that makes it more appealing. @@ -24,7 +24,7 @@ makes it more appealing. * Many more. Many of these features are irrelevant to caching, but can be used by other -areas of a web stack and therefore offer a compelling case to simplify your +areas of a web stack and therefore offers a compelling case to simplify your infrastructure. @@ -35,9 +35,9 @@ Quick Start **Recommended:** -* `redis`_ >= 2.4 +* `redis`_ >= 2.8 -* `redis-py`_ >= 2.10.3 +* `redis-py`_ >= 3.0.0 * `python`_ >= 2.7 @@ -59,7 +59,7 @@ of redis. Start the server by running ``./src/redis-server`` } **Warning: By default, django-redis-cache set keys in the database 1 of Redis. By default, a session with redis-cli start on database 0. Switch to database 1 with** ``SELECT 1``. - + .. _Django: https://www.djangoproject.com/ .. _django-redis-cache: http://github.com/sebleier/django-redis-cache .. _redis-py: http://github.com/andymccurdy/redis-py/ diff --git a/redis_cache/backends/base.py b/redis_cache/backends/base.py index 5ea3b53f..d473a635 100644 --- a/redis_cache/backends/base.py +++ b/redis_cache/backends/base.py @@ -1,3 +1,5 @@ +from functools import wraps + from django.core.cache.backends.base import ( BaseCache, DEFAULT_TIMEOUT, InvalidCacheBackendError, ) @@ -11,14 +13,9 @@ ) from redis.connection import DefaultParser - +from redis_cache.constants import KEY_EXPIRED, KEY_NON_VOLATILE from redis_cache.connection import pool -from redis_cache.utils import ( - CacheKey, get_servers, parse_connection_kwargs, import_class -) - - -from functools import wraps +from redis_cache.utils import get_servers, parse_connection_kwargs, import_class def get_client(write=False): @@ -28,8 +25,8 @@ def wrapper(method): @wraps(method) def wrapped(self, key, *args, **kwargs): version = kwargs.pop('version', None) - client = self.get_client(key, write=write) key = self.make_key(key, version=version) + client = self.get_client(key, write=write) return method(self, client, key, *args, **kwargs) return wrapped @@ -76,6 +73,12 @@ def __init__(self, server, params): **self.compressor_class_kwargs ) + redis_py_version = tuple(int(part) for part in redis.__version__.split('.')) + if redis_py_version < (3, 0, 0): + self.Redis = redis.StrictRedis + else: + self.Redis = redis.Redis + def __getstate__(self): return {'params': self.params, 'server': self.server} @@ -180,7 +183,7 @@ def create_client(self, server): socket_timeout=self.socket_timeout, socket_connect_timeout=self.socket_connect_timeout, ) - client = redis.Redis(**kwargs) + client = self.Redis(**kwargs) kwargs.update( parser_class=self.parser_class, connection_pool_class=self.connection_pool_class, @@ -216,12 +219,6 @@ def prep_value(self, value): value = self.serialize(value) return self.compress(value) - def make_key(self, key, version=None): - if not isinstance(key, CacheKey): - versioned_key = super(BaseRedisCache, self).make_key(key, version) - return CacheKey(key, versioned_key) - return key - def make_keys(self, keys, version=None): return [self.make_key(key, version=version) for key in keys] @@ -247,41 +244,34 @@ def add(self, client, key, value, timeout=DEFAULT_TIMEOUT): timeout = self.get_timeout(timeout) return self._set(client, key, self.prep_value(value), timeout, _add_only=True) + def _get(self, client, key, default=None): + value = client.get(key) + if value is None: + return default + value = self.get_value(value) + return value + @get_client() def get(self, client, key, default=None): """Retrieve a value from the cache. Returns deserialized value if key is found, the default if not. """ - value = client.get(key) - if value is None: - return default - value = self.get_value(value) - return value + return self._get(client, key, default) def _set(self, client, key, value, timeout, _add_only=False): - if timeout is None or timeout == 0: - if _add_only: - return client.setnx(key, value) - return client.set(key, value) - elif timeout > 0: - if _add_only: - added = client.setnx(key, value) - if added: - client.expire(key, timeout) - return added - return client.setex(key, value, timeout) - else: + if timeout is not None and timeout < 0: return False + elif timeout == 0: + return client.expire(key, 0) + return client.set(key, value, nx=_add_only, ex=timeout) @get_client(write=True) def set(self, client, key, value, timeout=DEFAULT_TIMEOUT): """Persist a value to the cache, and set an optional expiration time. """ timeout = self.get_timeout(timeout) - result = self._set(client, key, self.prep_value(value), timeout, _add_only=False) - return result @get_client(write=True) @@ -328,12 +318,6 @@ def get_many(self, keys, version=None): """Retrieve many keys.""" raise NotImplementedError - def _set_many(self, client, data): - # Only call mset if there actually is some data to save - if not data: - return True - return client.mset(data) - def set_many(self, data, timeout=DEFAULT_TIMEOUT, version=None): """Set a bunch of values in the cache at once from a dict of key/value pairs. This is much more efficient than calling set() multiple times. @@ -351,26 +335,27 @@ def incr(self, client, key, delta=1): exists = client.exists(key) if not exists: raise ValueError("Key '%s' not found" % key) - try: - value = client.incr(key, delta) - except redis.ResponseError: - key = key._original_key - value = self.get(key) + delta - self.set(key, value, timeout=None) + + value = client.incr(key, delta) + return value - def _incr_version(self, client, old, new, delta, version): + def _incr_version(self, client, old, new, original, delta, version): try: client.rename(old, new) except redis.ResponseError: - raise ValueError("Key '%s' not found" % old._original_key) + raise ValueError("Key '%s' not found" % original) return version + delta def incr_version(self, key, delta=1, version=None): """Adds delta to the cache version for the supplied key. Returns the new version. """ - raise NotImplementedError + + @get_client(write=True) + def touch(self, client, key, timeout=DEFAULT_TIMEOUT): + """Reset the timeout of a key to `timeout` seconds.""" + return client.expire(key, timeout) ##################### # Extra api methods # @@ -388,9 +373,13 @@ def ttl(self, client, key): Otherwise, the value is the number of seconds remaining. If the key does not exist, 0 is returned. """ - if client.exists(key): - return client.ttl(key) - return 0 + ttl = client.ttl(key) + if ttl == KEY_NON_VOLATILE: + return None + elif ttl == KEY_EXPIRED: + return 0 + else: + return ttl def _delete_pattern(self, client, pattern): keys = list(client.scan_iter(match=pattern)) @@ -400,33 +389,78 @@ def _delete_pattern(self, client, pattern): def delete_pattern(self, pattern, version=None): raise NotImplementedError + def lock( + self, + key, + timeout=None, + sleep=0.1, + blocking_timeout=None, + thread_local=True): + client = self.get_client(key, write=True) + return client.lock( + key, + timeout=timeout, + sleep=sleep, + blocking_timeout=blocking_timeout, + thread_local=thread_local + ) + @get_client(write=True) - def get_or_set(self, client, key, func, timeout=DEFAULT_TIMEOUT): + def get_or_set( + self, + client, + key, + func, + timeout=DEFAULT_TIMEOUT, + lock_timeout=None, + stale_cache_timeout=None): + """Get a value from the cache or call ``func`` to set it and return it. + + This implementation is slightly more advanced that Django's. It provides thundering herd + protection, which prevents multiple threads/processes from calling the value-generating + function too much. + + There are three timeouts you can specify: + + ``timeout``: Time in seconds that value at ``key`` is considered fresh. + ``lock_timeout``: Time in seconds that the lock will stay active and prevent other threads or + processes from acquiring the lock. + ``stale_cache_timeout``: Time in seconds that the stale cache will remain after the key has + expired. If ``None`` is specified, the stale value will remain indefinitely. + + """ if not callable(func): raise Exception("Must pass in a callable") - value = self.get(key._original_key) - - if value is None: - - dogpile_lock_key = "_lock" + key._versioned_key - dogpile_lock = client.get(dogpile_lock_key) + lock_key = "__lock__" + key + fresh_key = "__fresh__" + key - if dogpile_lock is None: - # Set the dogpile lock. - self._set(client, dogpile_lock_key, 0, None) + is_fresh = self._get(client, fresh_key) + value = self._get(client, key) - # calculate value of `func`. - try: - value = func() - finally: - # Regardless of error, release the dogpile lock. - client.expire(dogpile_lock_key, -1) - - timeout = self.get_timeout(timeout) + if is_fresh: + return value - # Set value of `func` and set timeout - self._set(client, key, self.prep_value(value), timeout) + timeout = self.get_timeout(timeout) + lock = self.lock(lock_key, timeout=lock_timeout) + + acquired = lock.acquire(blocking=False) + + if acquired: + try: + value = func() + except Exception: + raise + else: + key_timeout = ( + None if stale_cache_timeout is None else timeout + stale_cache_timeout + ) + pipeline = client.pipeline() + pipeline.set(key, self.prep_value(value), key_timeout) + pipeline.set(fresh_key, 1, timeout) + pipeline.execute() + finally: + lock.release() return value @@ -453,12 +487,3 @@ def persist(self, client, key): Returns True if successful and False if not. """ return client.persist(key) - - @get_client(write=True) - def expire(self, client, key, timeout): - """ - Set the expire time on a key - - returns True if successful and False if not. - """ - return client.expire(key, timeout) diff --git a/redis_cache/backends/multiple.py b/redis_cache/backends/multiple.py index 820dea98..d3d3f6c2 100644 --- a/redis_cache/backends/multiple.py +++ b/redis_cache/backends/multiple.py @@ -29,9 +29,8 @@ def shard(self, keys, write=False, version=None): """ clients = defaultdict(list) for key in keys: - clients[self.get_client(key, write)].append( - self.make_key(key, version) - ) + versioned_key = self.make_key(key, version=version) + clients[self.get_client(versioned_key, write)].append(versioned_key) return clients #################### @@ -63,41 +62,28 @@ def get_many(self, keys, version=None): data = {} clients = self.shard(keys, version=version) for client, versioned_keys in clients.items(): - original_keys = [key._original_key for key in versioned_keys] + versioned_keys = [self.make_key(key, version=version) for key in keys] data.update( - self._get_many( - client, - original_keys, - versioned_keys=versioned_keys - ) + self._get_many(client, keys, versioned_keys=versioned_keys) ) return data def set_many(self, data, timeout=DEFAULT_TIMEOUT, version=None): """ - Set a bunch of values in the cache at once from a dict of key/value - pairs. This is much more efficient than calling set() multiple times. + Set multiple values in the cache at once from a dict of key/value pairs. If timeout is given, that timeout will be used for the key; otherwise the default cache timeout will be used. """ timeout = self.get_timeout(timeout) + versioned_key_to_key = {self.make_key(key, version=version): key for key in data.keys()} + clients = self.shard(versioned_key_to_key.values(), write=True, version=version) - clients = self.shard(data.keys(), write=True, version=version) - - if timeout is None: - for client, keys in clients.items(): - subset = {} - for key in keys: - subset[key] = self.prep_value(data[key._original_key]) - self._set_many(client, subset) - return - - for client, keys in clients.items(): + for client, versioned_keys in clients.items(): pipeline = client.pipeline() - for key in keys: - value = self.prep_value(data[key._original_key]) - self._set(pipeline, key, value, timeout) + for versioned_key in versioned_keys: + value = self.prep_value(data[versioned_key_to_key[versioned_key]]) + self._set(pipeline, versioned_key, value, timeout) pipeline.execute() def incr_version(self, key, delta=1, version=None): @@ -113,7 +99,7 @@ def incr_version(self, key, delta=1, version=None): old = self.make_key(key, version=version) new = self.make_key(key, version=version + delta) - return self._incr_version(client, old, new, delta, version) + return self._incr_version(client, old, new, key, delta, version) ##################### # Extra api methods # diff --git a/redis_cache/backends/single.py b/redis_cache/backends/single.py index f13aec74..4a77ecf6 100644 --- a/redis_cache/backends/single.py +++ b/redis_cache/backends/single.py @@ -56,25 +56,18 @@ def get_many(self, keys, version=None): def set_many(self, data, timeout=DEFAULT_TIMEOUT, version=None): """ - Set a bunch of values in the cache at once from a dict of key/value - pairs. This is much more efficient than calling set() multiple times. + Set multiple values in the cache at once from a dict of key/value pairs. If timeout is given, that timeout will be used for the key; otherwise the default cache timeout will be used. """ timeout = self.get_timeout(timeout) - versioned_keys = self.make_keys(data.keys(), version=version) - if timeout is None: - new_data = {} - for key in versioned_keys: - new_data[key] = self.prep_value(data[key._original_key]) - return self._set_many(self.master_client, new_data) - pipeline = self.master_client.pipeline() - for key in versioned_keys: - value = self.prep_value(data[key._original_key]) - self._set(pipeline, key, value, timeout) + for key, value in data.items(): + value = self.prep_value(value) + versioned_key = self.make_key(key, version=version) + self._set(pipeline, versioned_key, value, timeout) pipeline.execute() def incr_version(self, key, delta=1, version=None): @@ -89,7 +82,7 @@ def incr_version(self, key, delta=1, version=None): old = self.make_key(key, version) new = self.make_key(key, version=version + delta) - return self._incr_version(self.master_client, old, new, delta, version) + return self._incr_version(self.master_client, old, new, key, delta, version) ##################### # Extra api methods # diff --git a/redis_cache/constants.py b/redis_cache/constants.py new file mode 100644 index 00000000..0c4e57b9 --- /dev/null +++ b/redis_cache/constants.py @@ -0,0 +1,2 @@ +KEY_EXPIRED = -2 +KEY_NON_VOLATILE = -1 diff --git a/redis_cache/utils.py b/redis_cache/utils.py index a7ecd720..bc7dc859 100644 --- a/redis_cache/utils.py +++ b/redis_cache/utils.py @@ -6,30 +6,10 @@ from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.six.moves.urllib.parse import parse_qs, urlparse +from redis._compat import unicode from redis.connection import SSLConnection -@python_2_unicode_compatible -class CacheKey(object): - """ - A stub string class that we can use to check if a key was created already. - """ - def __init__(self, key, versioned_key): - self._original_key = key - self._versioned_key = versioned_key - - def __eq__(self, other): - return self._versioned_key == other - - def __str__(self): - return force_text(self._versioned_key) - - def __hash__(self): - return hash(self._versioned_key) - - __repr__ = __str__ - - def get_servers(location): """Returns a list of servers given the server argument passed in from Django. diff --git a/requirements-dev.txt b/requirements-dev.txt index a910d2be..e13ff66c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,4 +2,4 @@ hiredis==0.2.0 django-nose==1.4.4 nose==1.3.6 msgpack-python==0.4.6 -pyyaml==3.11 +pyyaml>=4.2b1 diff --git a/requirements.txt b/requirements.txt index 91015469..fb8f42f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -redis==2.10.6 +redis<4.0 diff --git a/setup.py b/setup.py index 7373ddf6..ab9b64d3 100644 --- a/setup.py +++ b/setup.py @@ -5,11 +5,11 @@ url="http://github.com/sebleier/django-redis-cache/", author="Sean Bleier", author_email="sebleier@gmail.com", - version="1.8.1", + version="2.0.0", license="BSD", packages=["redis_cache", "redis_cache.backends"], description="Redis Cache Backend for Django", - install_requires=['redis==2.10.6'], + install_requires=['redis<4.0'], classifiers=[ "Programming Language :: Python", "Programming Language :: Python :: 2.7", diff --git a/tests/testapp/tests/base_tests.py b/tests/testapp/tests/base_tests.py index 4af2f3ed..719ec779 100644 --- a/tests/testapp/tests/base_tests.py +++ b/tests/testapp/tests/base_tests.py @@ -4,6 +4,7 @@ from hashlib import sha1 import os import subprocess +import threading import time @@ -21,6 +22,7 @@ from tests.testapp.models import Poll, expensive_calculation from redis_cache.cache import RedisCache, pool +from redis_cache.constants import KEY_EXPIRED, KEY_NON_VOLATILE from redis_cache.utils import get_servers, parse_connection_kwargs @@ -299,13 +301,12 @@ def test_set_expiration_timeout_None(self): def test_set_expiration_timeout_zero(self): key, value = self.cache.make_key('key'), 'value' self.cache.set(key, value, timeout=0) - self.assertIsNone(self.cache.get_client(key).ttl(key)) - self.assertIn(key, self.cache) + self.assertEqual(self.cache.get_client(key).ttl(key), KEY_EXPIRED) + self.assertNotIn(key, self.cache) def test_set_expiration_timeout_negative(self): key, value = self.cache.make_key('key'), 'value' self.cache.set(key, value, timeout=-1) - self.assertIsNone(self.cache.get_client(key).ttl(key)) self.assertNotIn(key, self.cache) def test_unicode(self): @@ -481,9 +482,9 @@ def test_ttl_of_reinsert_keys(self): self.cache.set('b', 'b', 5) self.cache.reinsert_keys() self.assertEqual(self.cache.get('a'), 'a') - self.assertGreater(self.cache.get_client('a').ttl(self.cache.make_key('a')), 1) + self.assertGreater(self.cache.ttl('a'), 1) self.assertEqual(self.cache.get('b'), 'b') - self.assertGreater(self.cache.get_client('b').ttl(self.cache.make_key('b')), 1) + self.assertGreater(self.cache.ttl('a'), 1) def test_get_or_set(self): @@ -510,6 +511,66 @@ def expensive_function(): self.assertEqual(expensive_function.num_calls, 2) self.assertEqual(value, 42) + def test_get_or_set_serving_from_stale_value(self): + + def expensive_function(x): + time.sleep(.5) + expensive_function.num_calls += 1 + return x + + expensive_function.num_calls = 0 + self.assertEqual(expensive_function.num_calls, 0) + results = {} + + def thread_worker(thread_id, return_value, timeout, lock_timeout, stale_cache_timeout): + value = self.cache.get_or_set( + 'key', + lambda: expensive_function(return_value), + timeout, + lock_timeout, + stale_cache_timeout + ) + results[thread_id] = value + + thread_0 = threading.Thread(target=thread_worker, args=(0, 'a', 1, None, 1)) + thread_1 = threading.Thread(target=thread_worker, args=(1, 'b', 1, None, 1)) + thread_2 = threading.Thread(target=thread_worker, args=(2, 'c', 1, None, 1)) + thread_3 = threading.Thread(target=thread_worker, args=(3, 'd', 1, None, 1)) + thread_4 = threading.Thread(target=thread_worker, args=(4, 'e', 1, None, 1)) + + # First thread should complete and return its value + thread_0.start() # t = 0, valid from t = .5 - 1.5, stale from t = 1.5 - 2.5 + + # Second thread will start while the first thread is still working and return None. + time.sleep(.25) # t = .25 + thread_1.start() + # Third thread will start after the first value is computed, but before it expires. + # its value. + time.sleep(.5) # t = .75 + thread_2.start() + # Fourth thread will start after the first value has expired and will re-compute its value. + # valid from t = 2.25 - 3.25, stale from t = 3.75 - 4.75. + time.sleep(1) # t = 1.75 + thread_3.start() + # Fifth thread will start after the fourth thread has started to compute its value, but + # before the first thread's stale cache has expired. + time.sleep(.25) # t = 2 + thread_4.start() + + thread_0.join() + thread_1.join() + thread_2.join() + thread_3.join() + thread_4.join() + + self.assertEqual(results, { + 0: 'a', + 1: None, + 2: 'a', + 3: 'd', + 4: 'a' + }) + def assertMaxConnection(self, cache, max_num): for client in cache.clients.values(): self.assertLessEqual(client.connection_pool._created_connections, max_num) @@ -581,21 +642,21 @@ def test_persist_expire_to_persist(self): self.cache.persist('a') self.assertIsNone(self.cache.ttl('a')) - def test_expire_no_expiry_to_expire(self): + def test_touch_no_expiry_to_expire(self): self.cache.set('a', 'a', timeout=None) - self.cache.expire('a', 10) + self.cache.touch('a', 10) ttl = self.cache.ttl('a') self.assertAlmostEqual(ttl, 10) - def test_expire_less(self): + def test_touch_less(self): self.cache.set('a', 'a', timeout=20) - self.cache.expire('a', 10) + self.cache.touch('a', 10) ttl = self.cache.ttl('a') self.assertAlmostEqual(ttl, 10) - def test_expire_more(self): + def test_touch_more(self): self.cache.set('a', 'a', timeout=10) - self.cache.expire('a', 20) + self.cache.touch('a', 20) ttl = self.cache.ttl('a') self.assertAlmostEqual(ttl, 20)