From da4e21b5156759fa117f516c9dfd919061b71220 Mon Sep 17 00:00:00 2001 From: Bertrand Bordage Date: Sun, 4 Jun 2017 18:37:39 +0200 Subject: [PATCH] Adds `CACHALOT_DATABASES` and removes dynamic setting support. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dynamic setting support is removed because it’s not thread-safe. --- cachalot/api.py | 6 +- cachalot/apps.py | 116 +++++++++++++++----------- cachalot/monkey_patch.py | 21 ++--- cachalot/panels.py | 9 ++- cachalot/settings.py | 93 ++++++++++++++++++--- cachalot/tests/__init__.py | 9 +++ cachalot/tests/read.py | 7 +- cachalot/tests/settings.py | 162 +++++++++++++++++++++++++++---------- cachalot/utils.py | 21 ++--- docs/limits.rst | 15 ++-- docs/quickstart.rst | 49 +++++------ 11 files changed, 336 insertions(+), 172 deletions(-) diff --git a/cachalot/api.py b/cachalot/api.py index 547a996..af1067c 100644 --- a/cachalot/api.py +++ b/cachalot/api.py @@ -7,9 +7,10 @@ from django.db import connections from django.utils.six import string_types from .cache import cachalot_caches +from .settings import cachalot_settings from .signals import post_invalidation from .transaction import AtomicCache -from .utils import _get_table_cache_key, _invalidate_tables +from .utils import _invalidate_tables __all__ = ('invalidate', 'get_last_invalidation') @@ -107,7 +108,8 @@ def get_last_invalidation(*tables_or_models, **kwargs): last_invalidation = 0.0 for cache_alias, db_alias, tables in _cache_db_tables_iterator( _get_tables(tables_or_models), cache_alias, db_alias): - table_cache_keys = [_get_table_cache_key(db_alias, t) for t in tables] + get_table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN + table_cache_keys = [get_table_cache_key(db_alias, t) for t in tables] invalidations = cachalot_caches.get_cache( cache_alias, db_alias).get_many(table_cache_keys).values() if invalidations: diff --git a/cachalot/apps.py b/cachalot/apps.py index 9f12ae6..1651062 100644 --- a/cachalot/apps.py +++ b/cachalot/apps.py @@ -1,59 +1,79 @@ from django.apps import AppConfig from django.conf import settings -from django.core.checks import register, Tags, Error +from django.core.checks import register, Tags, Warning, Error +from cachalot.utils import ITERABLES from .monkey_patch import patch -from .settings import cachalot_settings +from .settings import ( + cachalot_settings, SUPPORTED_CACHE_BACKENDS, SUPPORTED_DATABASE_ENGINES, + SUPPORTED_ONLY) -VALID_DATABASE_ENGINES = { - 'django.db.backends.sqlite3', - 'django.db.backends.postgresql', - 'django.db.backends.mysql', - # TODO: Remove when we drop Django 1.8 support. - 'django.db.backends.postgresql_psycopg2', - - # GeoDjango - 'django.contrib.gis.db.backends.spatialite', - 'django.contrib.gis.db.backends.postgis', - 'django.contrib.gis.db.backends.mysql', - - # django-transaction-hooks - 'transaction_hooks.backends.sqlite3', - 'transaction_hooks.backends.postgis', - 'transaction_hooks.backends.postgresql_psycopg2', - 'transaction_hooks.backends.mysql', -} +@register(Tags.caches, Tags.compatibility) +def check_cache_compatibility(app_configs, **kwargs): + cache = settings.CACHES[cachalot_settings.CACHALOT_CACHE] + cache_backend = cache['BACKEND'] + if cache_backend not in SUPPORTED_CACHE_BACKENDS: + return [Warning( + 'Cache backend %r is not supported by django-cachalot.' + % cache_backend, + hint='Switch to a supported cache backend ' + 'like Redis or Memcached.', + id='cachalot.W001')] + return [] -VALID_CACHE_BACKENDS = { - 'django.core.cache.backends.dummy.DummyCache', - 'django.core.cache.backends.locmem.LocMemCache', - 'django.core.cache.backends.filebased.FileBasedCache', - 'django_redis.cache.RedisCache', - 'django.core.cache.backends.memcached.MemcachedCache', - 'django.core.cache.backends.memcached.PyLibMCCache', -} - - -@register(Tags.compatibility) -def check_compatibility(app_configs, **kwargs): - cache_alias = cachalot_settings.CACHALOT_CACHE - caches = {cache_alias: settings.CACHES[cache_alias]} - +@register(Tags.database, Tags.compatibility) +def check_databases_compatibility(app_configs, **kwargs): errors = [] - for setting, k, valid_values in ( - (settings.DATABASES, 'ENGINE', VALID_DATABASE_ENGINES), - (caches, 'BACKEND', VALID_CACHE_BACKENDS)): - for config in setting.values(): - value = config[k] - if value not in valid_values: - errors.append( - Error( - '`%s` is not compatible with django-cachalot.' % value, - id='cachalot.E001', - ) - ) + databases = settings.DATABASES + original_enabled_databases = getattr(settings, 'CACHALOT_DATABASES', + SUPPORTED_ONLY) + enabled_databases = cachalot_settings.CACHALOT_DATABASES + if original_enabled_databases == SUPPORTED_ONLY: + if not cachalot_settings.CACHALOT_DATABASES: + errors.append(Warning( + 'None of the configured databases are supported ' + 'by django-cachalot.', + hint='Use a supported database, or remove django-cachalot, or ' + 'put at least one database alias in `CACHALOT_DATABASES` ' + 'to force django-cachalot to use it.', + id='cachalot.W002' + )) + elif enabled_databases.__class__ in ITERABLES: + for db_alias in enabled_databases: + if db_alias in databases: + engine = databases[db_alias]['ENGINE'] + if engine not in SUPPORTED_DATABASE_ENGINES: + errors.append(Warning( + 'Database engine %r is not supported ' + 'by django-cachalot.' % engine, + hint='Switch to a supported database engine.', + id='cachalot.W003' + )) + else: + errors.append(Error( + 'Database alias %r from `CACHALOT_DATABASES` ' + 'is not defined in `DATABASES`.' % db_alias, + hint='Change `CACHALOT_DATABASES` to be compliant with' + '`CACHALOT_DATABASES`', + id='cachalot.E001', + )) + + if not enabled_databases: + errors.append(Warning( + 'Django-cachalot is useless because no database ' + 'is configured in `CACHALOT_DATABASES`.', + hint='Reconfigure django-cachalot or remove it.', + id='cachalot.W004' + )) + else: + errors.append(Error( + "`CACHALOT_DATABASES` must be either %r or a list, tuple, " + "frozenset or set of database aliases." % SUPPORTED_ONLY, + hint='Remove `CACHALOT_DATABASES` or change it.', + id='cachalot.E002', + )) return errors @@ -62,6 +82,8 @@ class CachalotConfig(AppConfig): patched = False def ready(self): + cachalot_settings.load() + if not self.patched: patch() self.patched = True diff --git a/cachalot/monkey_patch.py b/cachalot/monkey_patch.py index 3e4f7f2..b0b2c54 100644 --- a/cachalot/monkey_patch.py +++ b/cachalot/monkey_patch.py @@ -16,10 +16,10 @@ from django.utils.six import binary_type from .api import invalidate from .cache import cachalot_caches -from .settings import cachalot_settings +from .settings import cachalot_settings, ITERABLES from .utils import ( - _get_query_cache_key, _get_table_cache_keys, _get_tables_from_sql, - UncachableQuery, TUPLE_OR_LIST, is_cachable, filter_cachable, + _get_table_cache_keys, _get_tables_from_sql, + UncachableQuery, is_cachable, filter_cachable, ) @@ -53,7 +53,7 @@ def _get_result_or_execute_query(execute_query_func, cache, return result result = execute_query_func() - if isinstance(result, Iterable) and result.__class__ not in TUPLE_OR_LIST: + if result.__class__ not in ITERABLES and isinstance(result, Iterable): result = list(result) cache.set(cache_key, (time(), result), cachalot_settings.CACHALOT_TIMEOUT) @@ -66,19 +66,21 @@ def _patch_compiler(original): @_unset_raw_connection def inner(compiler, *args, **kwargs): execute_query_func = lambda: original(compiler, *args, **kwargs) + db_alias = compiler.using if not cachalot_settings.CACHALOT_ENABLED \ + or db_alias not in cachalot_settings.CACHALOT_DATABASES \ or isinstance(compiler, WRITE_COMPILERS): return execute_query_func() try: - cache_key = _get_query_cache_key(compiler) + cache_key = cachalot_settings.CACHALOT_QUERY_KEYGEN(compiler) table_cache_keys = _get_table_cache_keys(compiler) except (EmptyResultSet, UncachableQuery): return execute_query_func() return _get_result_or_execute_query( execute_query_func, - cachalot_caches.get_cache(db_alias=compiler.using), + cachalot_caches.get_cache(db_alias=db_alias), cache_key, table_cache_keys) return inner @@ -109,7 +111,8 @@ def _patch_cursor(): @wraps(original) def inner(cursor, sql, *args, **kwargs): out = original(cursor, sql, *args, **kwargs) - if getattr(cursor.db, 'raw', True) \ + connection = cursor.db + if getattr(connection, 'raw', True) \ and cachalot_settings.CACHALOT_INVALIDATE_RAW: if isinstance(sql, binary_type): sql = sql.decode('utf-8') @@ -117,9 +120,9 @@ def _patch_cursor(): if 'update' in sql or 'insert' in sql or 'delete' in sql \ or 'alter' in sql or 'create' in sql or 'drop' in sql: tables = filter_cachable( - set(_get_tables_from_sql(cursor.db, sql))) + _get_tables_from_sql(connection, sql)) if tables: - invalidate(*tables, db_alias=cursor.db.alias, + invalidate(*tables, db_alias=connection.alias, cache_alias=cachalot_settings.CACHALOT_CACHE) return out diff --git a/cachalot/panels.py b/cachalot/panels.py index 8eb2d04..c995cf7 100644 --- a/cachalot/panels.py +++ b/cachalot/panels.py @@ -11,7 +11,7 @@ from django.utils.translation import ugettext_lazy as _ from django.utils.timesince import timesince from .cache import cachalot_caches -from .utils import _get_table_cache_key +from .settings import cachalot_settings class CachalotPanel(Panel): @@ -45,9 +45,10 @@ class CachalotPanel(Panel): data = defaultdict(list) cache = cachalot_caches.get_cache() for db_alias in settings.DATABASES: - model_cache_keys = dict( - [(_get_table_cache_key(db_alias, model._meta.db_table), model) - for model in models]) + get_table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN + model_cache_keys = { + get_table_cache_key(db_alias, model._meta.db_table): model + for model in models} for cache_key, timestamp in cache.get_many( model_cache_keys.keys()).items(): invalidation = datetime.fromtimestamp(timestamp) diff --git a/cachalot/settings.py b/cachalot/settings.py index 63227f7..3e56554 100644 --- a/cachalot/settings.py +++ b/cachalot/settings.py @@ -1,9 +1,45 @@ from django.conf import settings +from django.utils.module_loading import import_string + + +SUPPORTED_DATABASE_ENGINES = { + 'django.db.backends.sqlite3', + 'django.db.backends.postgresql', + 'django.db.backends.mysql', + # TODO: Remove when we drop Django 1.8 support. + 'django.db.backends.postgresql_psycopg2', + + # GeoDjango + 'django.contrib.gis.db.backends.spatialite', + 'django.contrib.gis.db.backends.postgis', + 'django.contrib.gis.db.backends.mysql', + + # django-transaction-hooks + 'transaction_hooks.backends.sqlite3', + 'transaction_hooks.backends.postgis', + 'transaction_hooks.backends.postgresql_psycopg2', + 'transaction_hooks.backends.mysql', +} + +SUPPORTED_CACHE_BACKENDS = { + 'django.core.cache.backends.dummy.DummyCache', + 'django.core.cache.backends.locmem.LocMemCache', + 'django.core.cache.backends.filebased.FileBasedCache', + 'django_redis.cache.RedisCache', + 'django.core.cache.backends.memcached.MemcachedCache', + 'django.core.cache.backends.memcached.PyLibMCCache', +} + +SUPPORTED_ONLY = 'supported_only' +ITERABLES = {tuple, list, frozenset, set} class Settings(object): + converters = {} + CACHALOT_ENABLED = True CACHALOT_CACHE = 'default' + CACHALOT_DATABASES = 'supported_only' CACHALOT_TIMEOUT = None CACHALOT_CACHE_RANDOM = False CACHALOT_INVALIDATE_RAW = True @@ -12,16 +48,55 @@ class Settings(object): CACHALOT_QUERY_KEYGEN = 'cachalot.utils.get_query_cache_key' CACHALOT_TABLE_KEYGEN = 'cachalot.utils.get_table_cache_key' - def __getattribute__(self, item): - if hasattr(settings, item): - return getattr(settings, item) - return super(Settings, self).__getattribute__(item) + @classmethod + def add_converter(cls, setting): + def inner(func): + cls.converters[setting] = func - def __setattr__(self, key, value): - raise AttributeError( - "Don't modify `cachalot_settings`, use " - "`django.test.utils.override_settings` or " - "`django.conf.settings` instead.") + return inner + + @classmethod + def get_names(cls): + return {name for name in cls.__dict__ + if name[:2] != '__' and name.isupper()} + + def load(self): + for name in self.get_names(): + value = getattr(settings, name, getattr(self.__class__, name)) + converter = self.converters.get(name) + if converter is not None: + value = converter(value) + setattr(self, name, value) + + +@Settings.add_converter('CACHALOT_DATABASES') +def convert(value): + if value == SUPPORTED_ONLY: + value = {alias for alias, setting in settings.DATABASES.items() + if setting['ENGINE'] in SUPPORTED_DATABASE_ENGINES} + if value.__class__ in ITERABLES: + return frozenset(value) + return value + + +@Settings.add_converter('CACHALOT_ONLY_CACHABLE_TABLES') +def convert(value): + return frozenset(value) + + +@Settings.add_converter('CACHALOT_ONLY_UNCACHABLE_TABLES') +def convert(value): + return frozenset(value) + + +@Settings.add_converter('CACHALOT_QUERY_KEYGEN') +def convert(value): + return import_string(value) + + +@Settings.add_converter('CACHALOT_TABLE_KEYGEN') +def convert(value): + return import_string(value) cachalot_settings = Settings() diff --git a/cachalot/tests/__init__.py b/cachalot/tests/__init__.py index 0f6501f..1f08614 100644 --- a/cachalot/tests/__init__.py +++ b/cachalot/tests/__init__.py @@ -1,3 +1,7 @@ +from django.core.signals import setting_changed +from django.dispatch import receiver + +from ..settings import cachalot_settings from .read import ReadTestCase, ParameterTypeTestCase from .write import WriteTestCase, DatabaseCommandTestCase from .transaction import AtomicTestCase @@ -8,3 +12,8 @@ from .api import APITestCase, CommandTestCase from .signals import SignalsTestCase from .postgres import PostgresReadTestCase from .debug_toolbar import DebugToolbarTestCase + + +@receiver(setting_changed) +def reload_settings(sender, **kwargs): + cachalot_settings.load() diff --git a/cachalot/tests/read.py b/cachalot/tests/read.py index c4da75e..25ae896 100644 --- a/cachalot/tests/read.py +++ b/cachalot/tests/read.py @@ -18,7 +18,8 @@ from django.test import ( TransactionTestCase, skipUnlessDBFeature, override_settings) from pytz import UTC -from ..utils import _get_table_cache_key, UncachableQuery +from ..settings import cachalot_settings +from ..utils import UncachableQuery from .models import Test, TestChild from .test_utils import TestUtilsMixin @@ -634,8 +635,8 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase): self.assert_tables(qs, 'cachalot_test') self.assert_query_cached(qs) - table_cache_key = _get_table_cache_key(connection.alias, - Test._meta.db_table) + table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN( + connection.alias, Test._meta.db_table) cache.delete(table_cache_key) self.assert_query_cached(qs) diff --git a/cachalot/tests/settings.py b/cachalot/tests/settings.py index 100ecc6..eb2ab74 100644 --- a/cachalot/tests/settings.py +++ b/cachalot/tests/settings.py @@ -7,12 +7,13 @@ from unittest import skipIf from django.conf import settings from django.contrib.auth.models import User from django.core.cache import DEFAULT_CACHE_ALIAS -from django.core.checks import run_checks, Error, Tags +from django.core.checks import run_checks, Tags, Warning, Error from django.db import connection from django.test import TransactionTestCase from django.test.utils import override_settings from ..api import invalidate +from ..settings import SUPPORTED_ONLY, SUPPORTED_DATABASE_ENGINES from .models import Test, TestParent, TestChild from .test_utils import TestUtilsMixin @@ -48,8 +49,8 @@ class SettingsTestCase(TestUtilsMixin, TransactionTestCase): data = list(Test.objects.all()) self.assertListEqual(data, [t]) - @skipIf(len(settings.CACHES) == 1, - 'We can’t change the cache used since there’s only one configured') + @skipIf(len(settings.CACHES) == 1, 'We can’t change the cache used ' + 'since there’s only one configured.') def test_cache(self): other_cache_alias = next(alias for alias in settings.CACHES if alias != DEFAULT_CACHE_ALIAS) @@ -70,6 +71,24 @@ class SettingsTestCase(TestUtilsMixin, TransactionTestCase): with self.settings(CACHALOT_CACHE=other_cache_alias): self.assert_query_cached(qs, before=0) + def test_databases(self): + qs = Test.objects.all() + with self.settings(CACHALOT_DATABASES=SUPPORTED_ONLY): + self.assert_query_cached(qs) + + invalidate(Test) + + engine = connection.settings_dict['ENGINE'] + SUPPORTED_DATABASE_ENGINES.remove(engine) + with self.settings(CACHALOT_DATABASES=SUPPORTED_ONLY): + self.assert_query_cached(qs, after=1) + SUPPORTED_DATABASE_ENGINES.add(engine) + with self.settings(CACHALOT_DATABASES=SUPPORTED_ONLY): + self.assert_query_cached(qs) + + with self.settings(CACHALOT_DATABASES=[]): + self.assert_query_cached(qs, after=1) + def test_cache_timeout(self): qs = Test.objects.all() @@ -156,52 +175,111 @@ class SettingsTestCase(TestUtilsMixin, TransactionTestCase): self.assert_query_cached(TestParent.objects.all()) self.assert_query_cached(User.objects.all(), after=1) - def test_compatibility(self): - """ - Checks that an error is raised: - - if an incompatible database is configured - - if an incompatible cache is configured as ``CACHALOT_CACHE`` - """ - def get_error(object_path): - return Error('`%s` is not compatible with django-cachalot.' - % object_path, id='cachalot.E001') - - incompatible_database = { - 'ENGINE': 'django.db.backends.oracle', - 'NAME': 'non_existent_db', + def test_cache_compatibility(self): + compatible_cache = { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', } incompatible_cache = { 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', 'LOCATION': 'cache_table' } - with self.settings(DATABASES={'default': incompatible_database}): - errors = run_checks(tags=[Tags.compatibility]) - self.assertListEqual(errors, - [get_error(incompatible_database['ENGINE'])]) - with self.settings(CACHES={'default': incompatible_cache}): - errors = run_checks(tags=[Tags.compatibility]) - self.assertListEqual(errors, - [get_error(incompatible_cache['BACKEND'])]) - with self.settings(DATABASES={'default': incompatible_database}, - CACHES={'default': incompatible_cache}): - errors = run_checks(tags=[Tags.compatibility]) - self.assertListEqual(errors, - [get_error(incompatible_database['ENGINE']), - get_error(incompatible_cache['BACKEND'])]) - compatible_database = { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': 'non_existent_db.sqlite3', - } - compatible_cache = { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - } - with self.settings(DATABASES={'default': compatible_database, - 'secondary': incompatible_database}): - errors = run_checks(tags=[Tags.compatibility]) - self.assertListEqual(errors, - [get_error(incompatible_database['ENGINE'])]) with self.settings(CACHES={'default': compatible_cache, 'secondary': incompatible_cache}): errors = run_checks(tags=[Tags.compatibility]) self.assertListEqual(errors, []) + + warning001 = Warning( + "Cache backend 'django.core.cache.backends.db.DatabaseCache' " + "is not supported by django-cachalot.", + hint='Switch to a supported cache backend ' + 'like Redis or Memcached.', + id='cachalot.W001') + with self.settings(CACHES={'default': incompatible_cache}): + errors = run_checks(tags=[Tags.compatibility]) + self.assertListEqual(errors, [warning001]) + with self.settings(CACHES={'default': compatible_cache, + 'secondary': incompatible_cache}, + CACHALOT_CACHE='secondary'): + errors = run_checks(tags=[Tags.compatibility]) + self.assertListEqual(errors, [warning001]) + + def test_database_compatibility(self): + compatible_database = { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': 'non_existent_db.sqlite3', + } + incompatible_database = { + 'ENGINE': 'django.db.backends.oracle', + 'NAME': 'non_existent_db', + } + + warning002 = Warning( + 'None of the configured databases are supported ' + 'by django-cachalot.', + hint='Use a supported database, or remove django-cachalot, or ' + 'put at least one database alias in `CACHALOT_DATABASES` ' + 'to force django-cachalot to use it.', + id='cachalot.W002' + ) + warning003 = Warning( + "Database engine 'django.db.backends.oracle' is not supported " + "by django-cachalot.", + hint='Switch to a supported database engine.', + id='cachalot.W003' + ) + warning004 = Warning( + 'Django-cachalot is useless because no database ' + 'is configured in `CACHALOT_DATABASES`.', + hint='Reconfigure django-cachalot or remove it.', + id='cachalot.W004' + ) + error001 = Error( + "Database alias 'secondary' from `CACHALOT_DATABASES` " + "is not defined in `DATABASES`.", + hint='Change `CACHALOT_DATABASES` to be compliant with' + '`CACHALOT_DATABASES`', + id='cachalot.E001', + ) + error002 = Error( + "`CACHALOT_DATABASES` must be either %r or a list, tuple, " + "frozenset or set of database aliases." % SUPPORTED_ONLY, + hint='Remove `CACHALOT_DATABASES` or change it.', + id='cachalot.E002', + ) + + with self.settings(DATABASES={'default': incompatible_database}): + errors = run_checks(tags=[Tags.compatibility]) + self.assertListEqual(errors, [warning002]) + + with self.settings(DATABASES={'default': compatible_database, + 'secondary': incompatible_database}): + errors = run_checks(tags=[Tags.compatibility]) + self.assertListEqual(errors, []) + with self.settings(DATABASES={'default': incompatible_database, + 'secondary': compatible_database}): + errors = run_checks(tags=[Tags.compatibility]) + self.assertListEqual(errors, []) + + with self.settings(DATABASES={'default': incompatible_database}, + CACHALOT_DATABASES=['default']): + errors = run_checks(tags=[Tags.compatibility]) + self.assertListEqual(errors, [warning003]) + + with self.settings(DATABASES={'default': incompatible_database}, + CACHALOT_DATABASES=[]): + errors = run_checks(tags=[Tags.compatibility]) + self.assertListEqual(errors, [warning004]) + + with self.settings(DATABASES={'default': incompatible_database}, + CACHALOT_DATABASES=['secondary']): + errors = run_checks(tags=[Tags.compatibility]) + self.assertListEqual(errors, [error001]) + with self.settings(DATABASES={'default': compatible_database}, + CACHALOT_DATABASES=['default', 'secondary']): + errors = run_checks(tags=[Tags.compatibility]) + self.assertListEqual(errors, [error001]) + + with self.settings(CACHALOT_DATABASES='invalid value'): + errors = run_checks(tags=[Tags.compatibility]) + self.assertListEqual(errors, [error002]) diff --git a/cachalot/utils.py b/cachalot/utils.py index 39428bc..de7b38b 100644 --- a/cachalot/utils.py +++ b/cachalot/utils.py @@ -13,10 +13,9 @@ from django.db.models import QuerySet from django.db.models.sql import Query from django.db.models.sql.where import ( ExtraWhere, SubqueryConstraint, WhereNode) -from django.utils.module_loading import import_string from django.utils.six import text_type, binary_type, PY2 -from .settings import cachalot_settings +from .settings import ITERABLES, cachalot_settings from .transaction import AtomicCache @@ -28,8 +27,6 @@ class IsRawQuery(Exception): pass -TUPLE_OR_LIST = {tuple, list} - CACHABLE_PARAM_TYPES = { bool, int, float, Decimal, bytearray, binary_type, text_type, type(None), datetime.date, datetime.time, datetime.datetime, datetime.timedelta, UUID, @@ -63,7 +60,7 @@ def check_parameter_types(params): for p in params: cl = p.__class__ if cl not in CACHABLE_PARAM_TYPES: - if cl in TUPLE_OR_LIST: + if cl in ITERABLES: check_parameter_types(p) elif cl is dict: check_parameter_types(p.items()) @@ -106,14 +103,6 @@ def get_table_cache_key(db_alias, table): return sha1(cache_key.encode('utf-8')).hexdigest() -def _get_query_cache_key(compiler): - return import_string(cachalot_settings.CACHALOT_QUERY_KEYGEN)(compiler) - - -def _get_table_cache_key(db_alias, table): - return import_string(cachalot_settings.CACHALOT_TABLE_KEYGEN)(db_alias, table) - - def _get_tables_from_sql(connection, lowercased_sql): return {t for t in connection.introspection.django_table_names() if t in lowercased_sql} @@ -188,7 +177,8 @@ def _get_tables(db_alias, query): def _get_table_cache_keys(compiler): db_alias = compiler.using - return [_get_table_cache_key(db_alias, t) + get_table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN + return [get_table_cache_key(db_alias, t) for t in _get_tables(db_alias, compiler.query)] @@ -197,8 +187,9 @@ def _invalidate_tables(cache, db_alias, tables): if not tables: return now = time() + get_table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN cache.set_many( - {_get_table_cache_key(db_alias, t): now for t in tables}, + {get_table_cache_key(db_alias, t): now for t in tables}, cachalot_settings.CACHALOT_TIMEOUT) if isinstance(cache, AtomicCache): diff --git a/docs/limits.rst b/docs/limits.rst index 7975b70..5e1ce57 100644 --- a/docs/limits.rst +++ b/docs/limits.rst @@ -99,22 +99,17 @@ Raw SQL queries those potential issues. By default, django-cachalot tries to invalidate its cache after a raw query. -It detects if the raw query contains ``UPDATE``, ``INSERT`` or ``DELETE``, -and then invalidates the tables contained in that query by comparing -with models registered by Django. +It detects if the raw query contains ``UPDATE``, ``INSERT``, ``DELETE``, +``ALTER``, ``CREATE`` or ``DROP`` and then invalidates the tables contained +in that query by comparing with models registered by Django. This is quite robust, so if a query is not invalidated automatically by this system, please :ref:`send a bug report `. In the meantime, you can use :ref:`the API ` to manually invalidate the tables where data has changed. -However, this simple system can be too efficient in some cases and lead to -unwanted extra invalidations. -In such cases, you may want to partially disable this behaviour by -:ref:`dynamically overriding settings ` to set -:ref:`CACHALOT_INVALIDATE_RAW` to ``False``. -After that, use :ref:`the API ` to manually invalidate the tables -you modified. +However, this simple system can be too efficient in some very rare cases +and lead to unwanted extra invalidations. .. _Multiple servers: diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 9e8669e..3089cda 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -8,11 +8,11 @@ Requirements - Python 2.7, 3.4, 3.5 or 3.6 - a cache configured as ``'default'`` with one of these backends: - - `django-redis `_ - - `memcached `_ + - `django-redis `_ + - `memcached `_ (using either python-memcached or pylibmc) - - `filebased `_ - - `locmem `_ + - `filebased `_ + - `locmem `_ (but it’s not shared between processes, see :ref:`locmem limits `) - one of these databases: @@ -67,7 +67,20 @@ Settings change this setting, you end up on a cache that may contain stale data. .. |CACHES| replace:: ``CACHES`` -.. _CACHES: https://docs.djangoproject.com/en/1.7/ref/settings/#std:setting-CACHES +.. _CACHES: https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-CACHES + +``CACHALOT_DATABASES`` +~~~~~~~~~~~~~~~~~~~~~~ + +:Default: ``'supported_only'`` +:Description: + List, tuple, set or frozenset of database aliases from |DATABASES|_ against + which django-cachalot will do caching. By default, the special value + ``'supported_only'`` enables django-cachalot only on supported database + engines. + +.. |DATABASES| replace:: ``DATABASES`` +.. _DATABASES: https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-DATABASES ``CACHALOT_TIMEOUT`` ~~~~~~~~~~~~~~~~~~~~ @@ -115,7 +128,6 @@ Settings can be cached: it disables this setting, so any table can be cached. :ref:`CACHALOT_UNCACHABLE_TABLES` has more weight than this: if you add a table to both settings, it will never be cached. - Use a frozenset over other sequence types for a tiny performance boost. Run ``./manage.py invalidate_cachalot`` after changing this setting. @@ -130,7 +142,6 @@ Settings Queries using a table mentioned in this setting will not be cached. Always keep ``'django_migrations'`` in it, otherwise you may face some issues, especially during tests. - Use a frozenset over other sequence types for a tiny performance boost. Run ``./manage.py invalidate_cachalot`` after changing this setting. ``CACHALOT_QUERY_KEYGEN`` @@ -152,30 +163,6 @@ Settings to use ``./manage.py invalidate_cachalot``). -.. _Dynamic overriding: - -Dynamic overriding -~~~~~~~~~~~~~~~~~~ - -Django-cachalot is built so that its settings can be dynamically changed. -For example: - -.. code:: python - - from django.conf import settings - from django.test.utils import override_settings - - with override_settings(CACHALOT_ENABLED=False): - # SQL queries are not cached in this block - - @override_settings(CACHALOT_CACHE='another_alias') - def your_function(): - # What’s in this function uses another cache - - # Globally disables SQL caching until you set it back to True - settings.CACHALOT_ENABLED = False - - .. _Command: ``manage.py`` command