152 lines
4.3 KiB
Python
152 lines
4.3 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""
|
|
celery.backends.cache
|
|
~~~~~~~~~~~~~~~~~~~~~
|
|
|
|
Memcache and in-memory cache result backend.
|
|
|
|
"""
|
|
from __future__ import absolute_import
|
|
|
|
import sys
|
|
|
|
from kombu.utils import cached_property
|
|
from kombu.utils.encoding import bytes_to_str, ensure_bytes
|
|
|
|
from celery.exceptions import ImproperlyConfigured
|
|
from celery.utils.functional import LRUCache
|
|
|
|
from .base import KeyValueStoreBackend
|
|
|
|
__all__ = ['CacheBackend']
|
|
|
|
_imp = [None]
|
|
|
|
PY3 = sys.version_info[0] == 3
|
|
|
|
REQUIRES_BACKEND = """\
|
|
The memcached backend requires either pylibmc or python-memcached.\
|
|
"""
|
|
|
|
UNKNOWN_BACKEND = """\
|
|
The cache backend {0!r} is unknown,
|
|
Please use one of the following backends instead: {1}\
|
|
"""
|
|
|
|
|
|
def import_best_memcache():
|
|
if _imp[0] is None:
|
|
is_pylibmc, memcache_key_t = False, ensure_bytes
|
|
try:
|
|
import pylibmc as memcache
|
|
is_pylibmc = True
|
|
except ImportError:
|
|
try:
|
|
import memcache # noqa
|
|
except ImportError:
|
|
raise ImproperlyConfigured(REQUIRES_BACKEND)
|
|
if PY3:
|
|
memcache_key_t = bytes_to_str
|
|
_imp[0] = (is_pylibmc, memcache, memcache_key_t)
|
|
return _imp[0]
|
|
|
|
|
|
def get_best_memcache(*args, **kwargs):
|
|
is_pylibmc, memcache, key_t = import_best_memcache()
|
|
Client = _Client = memcache.Client
|
|
|
|
if not is_pylibmc:
|
|
def Client(*args, **kwargs): # noqa
|
|
kwargs.pop('behaviors', None)
|
|
return _Client(*args, **kwargs)
|
|
|
|
return Client, key_t
|
|
|
|
|
|
class DummyClient(object):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
self.cache = LRUCache(limit=5000)
|
|
|
|
def get(self, key, *args, **kwargs):
|
|
return self.cache.get(key)
|
|
|
|
def get_multi(self, keys):
|
|
cache = self.cache
|
|
return dict((k, cache[k]) for k in keys if k in cache)
|
|
|
|
def set(self, key, value, *args, **kwargs):
|
|
self.cache[key] = value
|
|
|
|
def delete(self, key, *args, **kwargs):
|
|
self.cache.pop(key, None)
|
|
|
|
def incr(self, key, delta=1):
|
|
return self.cache.incr(key, delta)
|
|
|
|
|
|
backends = {'memcache': get_best_memcache,
|
|
'memcached': get_best_memcache,
|
|
'pylibmc': get_best_memcache,
|
|
'memory': lambda: (DummyClient, ensure_bytes)}
|
|
|
|
|
|
class CacheBackend(KeyValueStoreBackend):
|
|
servers = None
|
|
supports_autoexpire = True
|
|
supports_native_join = True
|
|
implements_incr = True
|
|
|
|
def __init__(self, app, expires=None, backend=None,
|
|
options={}, url=None, **kwargs):
|
|
super(CacheBackend, self).__init__(app, **kwargs)
|
|
|
|
self.options = dict(self.app.conf.CELERY_CACHE_BACKEND_OPTIONS,
|
|
**options)
|
|
|
|
self.backend = url or backend or self.app.conf.CELERY_CACHE_BACKEND
|
|
if self.backend:
|
|
self.backend, _, servers = self.backend.partition('://')
|
|
self.servers = servers.rstrip('/').split(';')
|
|
self.expires = self.prepare_expires(expires, type=int)
|
|
try:
|
|
self.Client, self.key_t = backends[self.backend]()
|
|
except KeyError:
|
|
raise ImproperlyConfigured(UNKNOWN_BACKEND.format(
|
|
self.backend, ', '.join(backends)))
|
|
self._encode_prefixes() # rencode the keyprefixes
|
|
|
|
def get(self, key):
|
|
return self.client.get(key)
|
|
|
|
def mget(self, keys):
|
|
return self.client.get_multi(keys)
|
|
|
|
def set(self, key, value):
|
|
return self.client.set(key, value, self.expires)
|
|
|
|
def delete(self, key):
|
|
return self.client.delete(key)
|
|
|
|
def _apply_chord_incr(self, header, partial_args, group_id, body, **opts):
|
|
self.client.set(self.get_key_for_chord(group_id), '0', time=86400)
|
|
return super(CacheBackend, self)._apply_chord_incr(
|
|
header, partial_args, group_id, body, **opts
|
|
)
|
|
|
|
def incr(self, key):
|
|
return self.client.incr(key)
|
|
|
|
@cached_property
|
|
def client(self):
|
|
return self.Client(self.servers, **self.options)
|
|
|
|
def __reduce__(self, args=(), kwargs={}):
|
|
servers = ';'.join(self.servers)
|
|
backend = '{0}://{1}/'.format(self.backend, servers)
|
|
kwargs.update(
|
|
dict(backend=backend,
|
|
expires=self.expires,
|
|
options=self.options))
|
|
return super(CacheBackend, self).__reduce__(args, kwargs)
|