From e51748612ffd2ec59bbfd52c54e92832e46e3a37 Mon Sep 17 00:00:00 2001 From: David Jean Louis Date: Tue, 28 Feb 2017 14:54:04 +0100 Subject: [PATCH 01/20] Make sure tenant delete() method is called, even when using django querysets bulk delete method (fixes issue #446). --- tenant_schemas/models.py | 21 +++++++++++++++++++- tenant_schemas/tests/test_tenants.py | 29 ++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/tenant_schemas/models.py b/tenant_schemas/models.py index c507fcf..63cef96 100644 --- a/tenant_schemas/models.py +++ b/tenant_schemas/models.py @@ -6,6 +6,24 @@ from tenant_schemas.signals import post_schema_sync from tenant_schemas.utils import get_public_schema_name, schema_exists +class TenantQueryset(models.QuerySet): + """ + QuerySet for instances that inherit from the TenantMixin. + """ + def delete(self): + """ + Make sure we call the delete method of each object in the queryset so + that safety checks and schema deletion (if requested) are executed + even when using bulk delete. + """ + counter, counter_dict = 0, {} + for obj in self: + current_counter, current_counter_dict = obj.delete() + counter += current_counter + counter_dict.update(current_counter_dict) + return counter, counter_dict + + class TenantMixin(models.Model): """ All tenant models must inherit this class. @@ -27,6 +45,7 @@ class TenantMixin(models.Model): domain_url = models.CharField(max_length=128, unique=True) schema_name = models.CharField(max_length=63, unique=True, validators=[_check_schema_name]) + objects = TenantQueryset.as_manager() class Meta: abstract = True @@ -69,7 +88,7 @@ class TenantMixin(models.Model): cursor = connection.cursor() cursor.execute('DROP SCHEMA IF EXISTS %s CASCADE' % self.schema_name) - super(TenantMixin, self).delete(*args, **kwargs) + return super(TenantMixin, self).delete(*args, **kwargs) def create_schema(self, check_if_exists=False, sync_schema=True, verbosity=1): diff --git a/tenant_schemas/tests/test_tenants.py b/tenant_schemas/tests/test_tenants.py index 5dd27e7..2bf26d8 100644 --- a/tenant_schemas/tests/test_tenants.py +++ b/tenant_schemas/tests/test_tenants.py @@ -99,6 +99,35 @@ class TenantDataAndSettingsTest(BaseTestCase): self.assertFalse(schema_exists(tenant.schema_name)) Tenant.auto_drop_schema = False + def test_auto_drop_schema_bulk_delete(self): + """ + When bulk deleting tenants, it should also drop the schemas of + tenants that have auto_drop_schema set to True. + """ + Tenant.auto_drop_schema = True + schemas = ['auto_drop_schema1', 'auto_drop_schema2'] + for schema in schemas: + self.assertFalse(schema_exists(schema)) + tenant = Tenant( + domain_url='%s.test.com' % schema, + schema_name=schema + ) + tenant.save(verbosity=BaseTestCase.get_verbosity()) + self.assertTrue(schema_exists(tenant.schema_name)) + + # Force pending trigger events to be executed + cursor = connection.cursor() + cursor.execute('SET CONSTRAINTS ALL IMMEDIATE') + + # get a queryset of our 2 tenants and do a bulk delete + Tenant.objects.filter(schema_name__in=schemas).delete() + + # verify that the schemas where deleted + for schema in schemas: + self.assertFalse(schema_exists(schema)) + + Tenant.auto_drop_schema = False + def test_switching_search_path(self): tenant1 = Tenant(domain_url='something.test.com', schema_name='tenant1') From a7b3805f11908fcaddf5d4b73fc34da0af21c4f3 Mon Sep 17 00:00:00 2001 From: David Jean Louis Date: Tue, 28 Feb 2017 15:07:31 +0100 Subject: [PATCH 02/20] Fix compatibility issue with django < 1.9 --- tenant_schemas/models.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tenant_schemas/models.py b/tenant_schemas/models.py index 63cef96..d51aa34 100644 --- a/tenant_schemas/models.py +++ b/tenant_schemas/models.py @@ -18,10 +18,13 @@ class TenantQueryset(models.QuerySet): """ counter, counter_dict = 0, {} for obj in self: - current_counter, current_counter_dict = obj.delete() - counter += current_counter - counter_dict.update(current_counter_dict) - return counter, counter_dict + result = obj.delete() + if result is not None: + current_counter, current_counter_dict = result + counter += current_counter + counter_dict.update(current_counter_dict) + if counter: + return counter, counter_dict class TenantMixin(models.Model): From 3223661559bbef7f1647a667e07cc9cd693603c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Ricardo=20Louren=C3=A7o?= Date: Wed, 8 Mar 2017 18:04:23 +0000 Subject: [PATCH 03/20] Add BaseTenantMiddleware for easier custom middleware support --- tenant_schemas/middleware.py | 75 +++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 26 deletions(-) diff --git a/tenant_schemas/middleware.py b/tenant_schemas/middleware.py index fa84094..646d0c8 100644 --- a/tenant_schemas/middleware.py +++ b/tenant_schemas/middleware.py @@ -13,38 +13,39 @@ if django.VERSION >= (1, 10, 0): else: MIDDLEWARE_MIXIN = object +""" +These middlewares should be placed at the very top of the middleware stack. +Selects the proper database schema using request information. Can fail in +various ways which is better than corrupting or revealing data. + +Extend BaseTenantMiddleware for a custom tenant selection strategy, +such as inspecting the header, or extracting it from some OAuth token. +""" + +class BaseTenantMiddleware(MIDDLEWARE_MIXIN): -class TenantMiddleware(MIDDLEWARE_MIXIN): """ - This middleware should be placed at the very top of the middleware stack. - Selects the proper database schema using the request host. Can fail in - various ways which is better than corrupting or revealing data. + Subclass and override this to achieve desired behaviour. Given a + request, return the tenant to use. Tenant should be an instance + of TENANT_MODEL. """ - TENANT_NOT_FOUND_EXCEPTION = Http404 + def get_tenant(self, request): + raise Exception("You have not implemented get_tenant. Don't know which tenant to use!") - def hostname_from_request(self, request): - """ Extracts hostname from request. Used for custom requests filtering. - By default removes the request's port and common prefixes. - """ - return remove_www(request.get_host().split(':')[0]).lower() - - def get_tenant(self, model, hostname, request): - return model.objects.get(domain_url=hostname) def process_request(self, request): # Connection needs first to be at the public schema, as this is where # the tenant metadata is stored. connection.set_schema_to_public() - hostname = self.hostname_from_request(request) - TenantModel = get_tenant_model() + # get_tenant must be implemented by extending this class. + tenant = self.get_tenant(request) - try: - request.tenant = self.get_tenant(TenantModel, hostname, request) - connection.set_tenant(request.tenant) - except TenantModel.DoesNotExist: - raise self.TENANT_NOT_FOUND_EXCEPTION( - 'No tenant for hostname "%s"' % hostname) + if tenant is None: + raise Exception("Trying to set current tenant to None!") + + request.tenant = tenant + connection.set_tenant(request.tenant) # Content type can no longer be cached as public and tenant schemas # have different models. If someone wants to change this, the cache @@ -59,6 +60,28 @@ class TenantMiddleware(MIDDLEWARE_MIXIN): if hasattr(settings, 'PUBLIC_SCHEMA_URLCONF') and request.tenant.schema_name == get_public_schema_name(): request.urlconf = settings.PUBLIC_SCHEMA_URLCONF +class TenantMiddleware(BaseTenantMiddleware): + """ + Selects the proper database schema using the request host. E.g. . + """ + TENANT_NOT_FOUND_EXCEPTION = Http404 + + def hostname_from_request(self, request): + """ Extracts hostname from request. Used for custom requests filtering. + By default removes the request's port and common prefixes. + """ + return remove_www(request.get_host().split(':')[0]).lower() + + def get_tenant(self, request): + hostname = self.hostname_from_request(request) + + TenantModel = get_tenant_model() + try: + return TenantModel.objects.get(domain_url=hostname) + except TenantModel.DoesNotExist: + raise self.TENANT_NOT_FOUND_EXCEPTION( + 'No tenant for hostname "%s"' % hostname) + class SuspiciousTenantMiddleware(TenantMiddleware): """ @@ -87,13 +110,13 @@ class DefaultTenantMiddleware(SuspiciousTenantMiddleware): """ DEFAULT_SCHEMA_NAME = None - def get_tenant(self, model, hostname, request): + def get_tenant(self, request): try: - return super(DefaultTenantMiddleware, self).get_tenant( - model, hostname, request) - except model.DoesNotExist: + return super(DefaultTenantMiddleware, self).get_tenant(request) + except self.TENANT_NOT_FOUND_EXCEPTION: schema_name = self.DEFAULT_SCHEMA_NAME if not schema_name: schema_name = get_public_schema_name() - return model.objects.get(schema_name=schema_name) + TenantModel = get_tenant_model() + return TenantModel.objects.get(schema_name=schema_name) From 540f32cab6bf756a76887461a8432d49f09ef217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Ricardo=20Louren=C3=A7o?= Date: Wed, 8 Mar 2017 19:03:20 +0000 Subject: [PATCH 04/20] Raise NotImplementedError in get_tenant --- tenant_schemas/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tenant_schemas/middleware.py b/tenant_schemas/middleware.py index 646d0c8..1b5c95b 100644 --- a/tenant_schemas/middleware.py +++ b/tenant_schemas/middleware.py @@ -30,7 +30,7 @@ class BaseTenantMiddleware(MIDDLEWARE_MIXIN): of TENANT_MODEL. """ def get_tenant(self, request): - raise Exception("You have not implemented get_tenant. Don't know which tenant to use!") + raise NotImplementedError def process_request(self, request): From f532cbacc9249274b6b4cff98dffa17b4c3d5b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Ricardo=20Louren=C3=A7o?= Date: Wed, 8 Mar 2017 20:15:55 +0000 Subject: [PATCH 05/20] Replicate old behavior where we raise self.TENANT_NOT_FOUND_EXCEPTION for DefaultTenantMiddleware --- tenant_schemas/middleware.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tenant_schemas/middleware.py b/tenant_schemas/middleware.py index 1b5c95b..f99612b 100644 --- a/tenant_schemas/middleware.py +++ b/tenant_schemas/middleware.py @@ -113,10 +113,13 @@ class DefaultTenantMiddleware(SuspiciousTenantMiddleware): def get_tenant(self, request): try: return super(DefaultTenantMiddleware, self).get_tenant(request) - except self.TENANT_NOT_FOUND_EXCEPTION: + except self.TENANT_NOT_FOUND_EXCEPTION, e: schema_name = self.DEFAULT_SCHEMA_NAME if not schema_name: schema_name = get_public_schema_name() TenantModel = get_tenant_model() - return TenantModel.objects.get(schema_name=schema_name) + try: + return TenantModel.objects.get(schema_name=schema_name) + except TenantModel.DoesNotExist: + raise e # Raise the same exception (we don't have a tenant) From 06a1b9d570753818a18112855fc6c3c89923d0db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Ricardo=20Louren=C3=A7o?= Date: Wed, 8 Mar 2017 20:19:07 +0000 Subject: [PATCH 06/20] Support Python 2.6+ --- tenant_schemas/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tenant_schemas/middleware.py b/tenant_schemas/middleware.py index f99612b..1e4e41f 100644 --- a/tenant_schemas/middleware.py +++ b/tenant_schemas/middleware.py @@ -113,7 +113,7 @@ class DefaultTenantMiddleware(SuspiciousTenantMiddleware): def get_tenant(self, request): try: return super(DefaultTenantMiddleware, self).get_tenant(request) - except self.TENANT_NOT_FOUND_EXCEPTION, e: + except self.TENANT_NOT_FOUND_EXCEPTION as e: schema_name = self.DEFAULT_SCHEMA_NAME if not schema_name: schema_name = get_public_schema_name() From d80ac3ace58446b27e2a437269d434b1b216b144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Ricardo=20Louren=C3=A7o?= Date: Thu, 9 Mar 2017 16:33:28 +0000 Subject: [PATCH 07/20] Make self.get_tenant backwards compatible. Handle exceptions within process_request --- tenant_schemas/middleware.py | 58 +++++++++++++++++------------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/tenant_schemas/middleware.py b/tenant_schemas/middleware.py index 1e4e41f..c77c5ea 100644 --- a/tenant_schemas/middleware.py +++ b/tenant_schemas/middleware.py @@ -23,26 +23,41 @@ such as inspecting the header, or extracting it from some OAuth token. """ class BaseTenantMiddleware(MIDDLEWARE_MIXIN): + TENANT_NOT_FOUND_EXCEPTION = Http404 """ Subclass and override this to achieve desired behaviour. Given a request, return the tenant to use. Tenant should be an instance - of TENANT_MODEL. + of TENANT_MODEL. We have three parameters for backwards compatibility + (the request would be enough). """ - def get_tenant(self, request): + def get_tenant(self, model, hostname, request): raise NotImplementedError + def hostname_from_request(self, request): + """ Extracts hostname from request. Used for custom requests filtering. + By default removes the request's port and common prefixes. + """ + return remove_www(request.get_host().split(':')[0]).lower() def process_request(self, request): # Connection needs first to be at the public schema, as this is where # the tenant metadata is stored. connection.set_schema_to_public() - # get_tenant must be implemented by extending this class. - tenant = self.get_tenant(request) + hostname = self.hostname_from_request(request) + TenantModel = get_tenant_model() - if tenant is None: - raise Exception("Trying to set current tenant to None!") + try: + # get_tenant must be implemented by extending this class. + tenant = self.get_tenant(TenantModel, hostname, request) + assert isinstance(request.tenant, TenantModel) + except TenantModel.DoesNotExist: + raise self.TENANT_NOT_FOUND_EXCEPTION( + 'No tenant for {!r}'.format(request.get_host())) + except AssertionError: + raise self.TENANT_NOT_FOUND_EXCEPTION( + 'Invalid tenant {!r}'.format(request.tenant)) request.tenant = tenant connection.set_tenant(request.tenant) @@ -64,23 +79,9 @@ class TenantMiddleware(BaseTenantMiddleware): """ Selects the proper database schema using the request host. E.g. . """ - TENANT_NOT_FOUND_EXCEPTION = Http404 - def hostname_from_request(self, request): - """ Extracts hostname from request. Used for custom requests filtering. - By default removes the request's port and common prefixes. - """ - return remove_www(request.get_host().split(':')[0]).lower() - - def get_tenant(self, request): - hostname = self.hostname_from_request(request) - - TenantModel = get_tenant_model() - try: - return TenantModel.objects.get(domain_url=hostname) - except TenantModel.DoesNotExist: - raise self.TENANT_NOT_FOUND_EXCEPTION( - 'No tenant for hostname "%s"' % hostname) + def get_tenant(self, model, hostname, request): + return model.objects.get(domain_url=hostname) class SuspiciousTenantMiddleware(TenantMiddleware): @@ -110,16 +111,13 @@ class DefaultTenantMiddleware(SuspiciousTenantMiddleware): """ DEFAULT_SCHEMA_NAME = None - def get_tenant(self, request): + def get_tenant(self, model, hostname, request): try: - return super(DefaultTenantMiddleware, self).get_tenant(request) - except self.TENANT_NOT_FOUND_EXCEPTION as e: + return super(DefaultTenantMiddleware, self).get_tenant( + model, hostname, request) + except model.DoesNotExist: schema_name = self.DEFAULT_SCHEMA_NAME if not schema_name: schema_name = get_public_schema_name() - TenantModel = get_tenant_model() - try: - return TenantModel.objects.get(schema_name=schema_name) - except TenantModel.DoesNotExist: - raise e # Raise the same exception (we don't have a tenant) + return model.objects.get(schema_name=schema_name) From b0dc8bdb547452a8fdc4f76ca85550a7d7b6fa0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Ricardo=20Louren=C3=A7o?= Date: Thu, 9 Mar 2017 16:33:57 +0000 Subject: [PATCH 08/20] Add an advanced usage section to the documentation --- docs/advanced_usage.rst | 61 +++++++++++++++++++++++++++++++++++++++++ docs/conf.py | 2 +- docs/index.rst | 3 +- docs/install.rst | 2 ++ 4 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 docs/advanced_usage.rst diff --git a/docs/advanced_usage.rst b/docs/advanced_usage.rst new file mode 100644 index 0000000..85be259 --- /dev/null +++ b/docs/advanced_usage.rst @@ -0,0 +1,61 @@ +============== +Advanced Usage +============== + +Custom tenant strategies (custom middleware support) +==================================================== +By default, ``django-tenant-schemas``'s strategies for determining the correct tenant involve extracting it from the URL (e.g. ``mytenant.mydomain.com``). This is done through a middleware, typically ``TenantMiddleware``. + +In some situations, it might be useful to use **alternative tenant selection strategies**. For example, consider a website with a fixed URL. An approach for this website might be to pass the tenant through a special header, or to determine it in some other manner based on the request (e.g. using an OAuth token mapped to a tenant). ``django-tenant-schemas`` offer an **easily extensible way to provide your own middleware** with minimal code changes. + +To add custom tenant selection strategies, you need to **subclass the** ``BaseTenantMiddleware`` **class and implement its** ``get_tenant`` **method**. This method accepts the current ``request`` object through which you can determine the tenant to use. In addition, for backwards-compatibility reasons, the method also accepts the tenant model class (``TENANT_MODEL``) and the ``hostname`` of the current request. **You should return an instance of your** ``TENANT_MODEL`` **class** from this function. +After creating your middleware, you should make it the top-most middleware in your list. You should only have one subclass of ``BaseTenantMiddleware`` per project. + +Note that you might also wish to extend the other provided middleware classes, such as ``TenantMiddleware``. For example, you might want to chain several strategies together, and you could do so by subclassing the original strategies and manipulating the call to ``super``'s ``get_tenant``. + + +Example: Determine tenant from HTTP header +------------------------------------------ +Suppose you wanted to determine the current tenant based on a request header (``X-DTS-SCHEMA``). You might implement a simple middleware such as: + +.. code-block:: python + + class XHeaderTenantMiddleware(BaseTenantMiddleware): + """ + Determines tenant by the value of the ``X-DTS-SCHEMA`` HTTP header. + """ + def get_tenant(self, model, hostname, request): + schema_name = request.META.get('HTTP_X_DTS_SCHEMA', get_public_schema_name()) + return model.objects.get(schema_name=schema_name) + +Your application could now specify the tenant with the ``X-DTS-SCHEMA`` HTTP header. In scenarios where you are configuring individual tenant websites by yourself, each with its own ``nginx`` configuration to redirect to the right tenant, you could use a configuration such as the one below: + + +.. code-block:: nginx + + # /etc/nginx/conf.d/multitenant.conf + + upstream web { + server localhost:8000; + } + + server { + listen 80 default_server; + server_name _; + + location / { + proxy_pass http://web; + proxy_set_header Host $host; + } + } + + server { + listen 80; + server_name example.com www.example.com; + + location / { + proxy_pass http://web; + proxy_set_header Host $host; + proxy_set_header X-DTS-SCHEMA example; # triggers XHeaderTenantMiddleware + } + } diff --git a/docs/conf.py b/docs/conf.py index 3b032f3..be89624 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,7 +24,7 @@ import datetime # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.autosectionlabel',] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/index.rst b/docs/index.rst index d47d9b4..e890259 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -48,9 +48,10 @@ Contents .. toctree:: :maxdepth: 2 - + install use + advanced_usage examples templates test diff --git a/docs/install.rst b/docs/install.rst index 26f18fc..4b64894 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -38,6 +38,8 @@ If you'd like to raise ``DisallowedHost`` and a HTTP 400 response instead, use t If you'd like to serve the public tenant for unrecognised hostnames instead, use ``tenant_schemas.middleware.DefaultTenantMiddleware``. To use a tenant other than the public tenant, create a subclass and register it instead. +If you'd like a different tenant selection technique (e.g. using an HTTP Header), you can define a custom middleware. See :ref:`Advanced Usage`. + .. code-block:: python from tenant_schemas.middleware import DefaultTenantMiddleware From 2630ecbee498fecb4cf62dac279111948acb9f94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Ricardo=20Louren=C3=A7o?= Date: Thu, 9 Mar 2017 16:37:44 +0000 Subject: [PATCH 09/20] Fix assert --- tenant_schemas/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tenant_schemas/middleware.py b/tenant_schemas/middleware.py index c77c5ea..639e686 100644 --- a/tenant_schemas/middleware.py +++ b/tenant_schemas/middleware.py @@ -51,7 +51,7 @@ class BaseTenantMiddleware(MIDDLEWARE_MIXIN): try: # get_tenant must be implemented by extending this class. tenant = self.get_tenant(TenantModel, hostname, request) - assert isinstance(request.tenant, TenantModel) + assert isinstance(tenant, TenantModel) except TenantModel.DoesNotExist: raise self.TENANT_NOT_FOUND_EXCEPTION( 'No tenant for {!r}'.format(request.get_host())) From 323cfcd38a60d61a9ba9b96eb01fa4a9a2e30d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateu=20C=C3=A0naves?= Date: Fri, 10 Mar 2017 16:06:03 +0100 Subject: [PATCH 10/20] Add migration executors --- setup.py | 2 + .../management/commands/__init__.py | 3 + .../management/commands/migrate_schemas.py | 33 +++------- .../migration_executors/__init__.py | 15 +++++ tenant_schemas/migration_executors/base.py | 64 +++++++++++++++++++ .../migration_executors/parallel.py | 30 +++++++++ .../migration_executors/standard.py | 9 +++ 7 files changed, 132 insertions(+), 24 deletions(-) create mode 100644 tenant_schemas/migration_executors/__init__.py create mode 100644 tenant_schemas/migration_executors/base.py create mode 100644 tenant_schemas/migration_executors/parallel.py create mode 100644 tenant_schemas/migration_executors/standard.py diff --git a/setup.py b/setup.py index c201fd6..6b8613a 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from os.path import exists + from version import get_git_version try: @@ -15,6 +16,7 @@ setup( author_email='carneiro.be@gmail.com', packages=[ 'tenant_schemas', + 'tenant_schemas.migration_executors', 'tenant_schemas.postgresql_backend', 'tenant_schemas.management', 'tenant_schemas.management.commands', diff --git a/tenant_schemas/management/commands/__init__.py b/tenant_schemas/management/commands/__init__.py index 54c4711..c3c28ed 100644 --- a/tenant_schemas/management/commands/__init__.py +++ b/tenant_schemas/management/commands/__init__.py @@ -149,11 +149,14 @@ class SyncCommon(BaseCommand): help=('Database state will be brought to the state after that ' 'migration. Use the name "zero" to unapply all migrations.')) parser.add_argument("-s", "--schema", dest="schema_name") + parser.add_argument('--executor', action='store', dest='executor', default=None, + help='Executor for running migrations [standard (default)|parallel]') def handle(self, *args, **options): self.sync_tenant = options.get('tenant') self.sync_public = options.get('shared') self.schema_name = options.get('schema_name') + self.executor = options.get('executor') self.installed_apps = settings.INSTALLED_APPS self.args = args self.options = options diff --git a/tenant_schemas/management/commands/migrate_schemas.py b/tenant_schemas/management/commands/migrate_schemas.py index 7878629..7e131c2 100644 --- a/tenant_schemas/management/commands/migrate_schemas.py +++ b/tenant_schemas/management/commands/migrate_schemas.py @@ -1,11 +1,10 @@ import django - -from django.conf import settings from django.core.management.commands.migrate import Command as MigrateCommand from django.db import connection from tenant_schemas.management.commands import SyncCommon -from tenant_schemas.utils import get_tenant_model, get_public_schema_name, schema_exists +from tenant_schemas.migration_executors import get_executor +from tenant_schemas.utils import get_public_schema_name, get_tenant_model, schema_exists if django.VERSION >= (1, 9, 0): from django.db.migrations.exceptions import MigrationSchemaMissing @@ -34,35 +33,21 @@ class Command(SyncCommon): super(Command, self).handle(*args, **options) self.PUBLIC_SCHEMA_NAME = get_public_schema_name() + executor = get_executor(codename=self.executor)(self.args, self.options) + if self.sync_public and not self.schema_name: self.schema_name = self.PUBLIC_SCHEMA_NAME if self.sync_public: - self.run_migrations(self.schema_name, settings.SHARED_APPS) + executor.run_migrations(tenants=[self.schema_name]) if self.sync_tenant: if self.schema_name and self.schema_name != self.PUBLIC_SCHEMA_NAME: if not schema_exists(self.schema_name): raise MigrationSchemaMissing('Schema "{}" does not exist'.format( self.schema_name)) else: - self.run_migrations(self.schema_name, settings.TENANT_APPS) + tenants = [self.schema_name] else: - all_tenants = get_tenant_model().objects.exclude(schema_name=get_public_schema_name()) - for tenant in all_tenants: - self.run_migrations(tenant.schema_name, settings.TENANT_APPS) - - def run_migrations(self, schema_name, included_apps): - if int(self.options.get('verbosity', 1)) >= 1: - self._notice("=== Running migrate for schema %s" % schema_name) - - if not schema_exists(schema_name): - raise MigrationSchemaMissing('Schema "{}" does not exist'.format( - schema_name)) - - connection.set_schema(schema_name) - command = MigrateCommand() - command.execute(*self.args, **self.options) - connection.set_schema_to_public() - - def _notice(self, output): - self.stdout.write(self.style.NOTICE(output)) + tenants = get_tenant_model().objects.exclude(schema_name=get_public_schema_name()).values_list( + 'schema_name', flat=True) + executor.run_migrations(tenants=tenants) diff --git a/tenant_schemas/migration_executors/__init__.py b/tenant_schemas/migration_executors/__init__.py new file mode 100644 index 0000000..b83db18 --- /dev/null +++ b/tenant_schemas/migration_executors/__init__.py @@ -0,0 +1,15 @@ +import os + +from tenant_schemas.migration_executors.base import MigrationExecutor +from tenant_schemas.migration_executors.parallel import ParallelExecutor +from tenant_schemas.migration_executors.standard import StandardExecutor + + +def get_executor(codename=None): + codename = codename or os.environ.get('EXECUTOR', StandardExecutor.codename) + + for klass in MigrationExecutor.__subclasses__(): + if klass.codename == codename: + return klass + + raise NotImplementedError('No executor with codename %s' % codename) diff --git a/tenant_schemas/migration_executors/base.py b/tenant_schemas/migration_executors/base.py new file mode 100644 index 0000000..5ce6c59 --- /dev/null +++ b/tenant_schemas/migration_executors/base.py @@ -0,0 +1,64 @@ +import sys + +from django.core.management.commands.migrate import Command as MigrateCommand +from django.db import transaction + +from tenant_schemas.utils import get_public_schema_name + + +def run_migrations(args, options, executor_codename, schema_name, allow_atomic=True): + from django.core.management import color + from django.core.management.base import OutputWrapper + from django.db import connection + + style = color.color_style() + + def style_func(msg): + return '[%s:%s] %s' % ( + style.NOTICE(executor_codename), + style.NOTICE(schema_name), + msg + ) + + stdout = OutputWrapper(sys.stdout) + stdout.style_func = style_func + stderr = OutputWrapper(sys.stderr) + stderr.style_func = style_func + if int(options.get('verbosity', 1)) >= 1: + stdout.write(style.NOTICE("=== Running migrate for schema %s" % schema_name)) + + connection.set_schema(schema_name) + MigrateCommand(stdout=stdout, stderr=stderr).execute(*args, **options) + + try: + transaction.commit() + connection.close() + connection.connection = None + except transaction.TransactionManagementError: + if not allow_atomic: + raise + + # We are in atomic transaction, don't close connections + pass + + connection.set_schema_to_public() + + +class MigrationExecutor(object): + codename = None + + def __init__(self, args, options): + self.args = args + self.options = options + + def run_migrations(self, tenants): + public_schema_name = get_public_schema_name() + + if public_schema_name in tenants: + run_migrations(self.args, self.options, self.codename, public_schema_name) + tenants.pop(tenants.index(public_schema_name)) + + self.run_tenant_migrations(tenants) + + def run_tenant_migrations(self, tenant): + raise NotImplementedError diff --git a/tenant_schemas/migration_executors/parallel.py b/tenant_schemas/migration_executors/parallel.py new file mode 100644 index 0000000..9228378 --- /dev/null +++ b/tenant_schemas/migration_executors/parallel.py @@ -0,0 +1,30 @@ +import functools +import multiprocessing + +from django.conf import settings + +from tenant_schemas.migration_executors.base import MigrationExecutor, run_migrations + + +class ParallelExecutor(MigrationExecutor): + codename = 'parallel' + + def run_tenant_migrations(self, tenants): + if tenants: + processes = getattr(settings, 'TENANT_PARALLEL_MIGRATION_MAX_PROCESSES', 2) + chunks = getattr(settings, 'TENANT_PARALLEL_MIGRATION_CHUNKS', 2) + + from django.db import connection + + connection.close() + connection.connection = None + + run_migrations_p = functools.partial( + run_migrations, + self.args, + self.options, + self.codename, + allow_atomic=False + ) + p = multiprocessing.Pool(processes=processes) + p.map(run_migrations_p, tenants, chunks) diff --git a/tenant_schemas/migration_executors/standard.py b/tenant_schemas/migration_executors/standard.py new file mode 100644 index 0000000..be5eb2d --- /dev/null +++ b/tenant_schemas/migration_executors/standard.py @@ -0,0 +1,9 @@ +from tenant_schemas.migration_executors.base import MigrationExecutor, run_migrations + + +class StandardExecutor(MigrationExecutor): + codename = 'standard' + + def run_tenant_migrations(self, tenants): + for schema_name in tenants: + run_migrations(self.args, self.options, self.codename, schema_name) From c5c1f06d71a90617880c121daf3724c6136c60c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateu=20C=C3=A0naves?= Date: Fri, 10 Mar 2017 16:09:59 +0100 Subject: [PATCH 11/20] Added parallel migration executor to tox and travis --- .travis.yml | 25 +++++++++++++------------ tox.ini | 16 +++++++++++++--- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index d83ff55..b831b2c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,22 @@ +sudo: false language: python python: -- 2.7 -- 3.5 + - 2.7 + - 3.5 services: -- postgresql + - postgresql addons: postgresql: '9.4' -install: -- pip install tox -before_script: -- psql -c "CREATE DATABASE dts_test_project;" -U postgres -script: -- tox -e py${TRAVIS_PYTHON_VERSION/./}-dj${DJANGO/./} +install: pip install -q tox-travis env: -- DJANGO=1.8 -- DJANGO=1.9 -- DJANGO=1.10 + - DJANGO=1.8 + - DJANGO=1.9 + - DJANGO=1.10 +matrix: + fast_finish: true +script: tox +before_script: psql -c "CREATE DATABASE dts_test_project;" -U postgres + deploy: provider: pypi user: bcarneiro diff --git a/tox.ini b/tox.ini index fc4f8c5..882e2ae 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,16 @@ [tox] -envlist = py{27,35}-dj{18,19,110} +envlist = py{27,35}-dj{18,19,110}-{standard,parallel} + +[travis:env] +DJANGO = + 1.8: dj18-{standard,parallel} + 1.9: dj19-{standard,parallel} + 1.10: dj110-{standard,parallel} [testenv] usedevelop = True -deps = +deps = coverage mock tblib @@ -16,6 +22,10 @@ changedir = dts_test_project passenv = PG_NAME PG_USER PG_PASSWORD PG_HOST PG_PORT -commands = +setenv = + standard: MIGRATION_EXECUTOR=standard + parallel: MIGRATION_EXECUTOR=parallel + +commands = coverage run manage.py test --noinput {posargs:tenant_schemas} coverage report -m --include=../tenant_schemas/* From 45157bf459c5df0f11d8c327505bf2129e7ce2e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateu=20C=C3=A0naves?= Date: Fri, 10 Mar 2017 16:11:15 +0100 Subject: [PATCH 12/20] Update docs with new command info --- docs/use.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/use.rst b/docs/use.rst index da4cf13..a551485 100644 --- a/docs/use.rst +++ b/docs/use.rst @@ -95,6 +95,26 @@ The options given to ``migrate_schemas`` are also passed to every ``migrate``. H ``migrate_schemas`` raises an exception when an tenant schema is missing. +migrate_schemas in parallel +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Once the number of tenants grow, migrating all the tenants can become a bottleneck. To speed up this process, you can run tenant migrations in parallel like this: + +.. code-block:: bash + + python manage.py migrate_schemas --executor=parallel + +In fact, you can write your own executor which will run tenant migrations in +any way you want, just take a look at ``tenant_schemas/migration_executors``. + +The ``parallel`` executor accepts the following settings: + +* ``TENANT_PARALLEL_MIGRATION_MAX_PROCESSES`` (default: 2) - maximum number of + processes for migration pool (this is to avoid exhausting the database + connection pool) +* ``TENANT_PARALLEL_MIGRATION_CHUNKS`` (default: 2) - number of migrations to be + sent at once to every worker + tenant_command ~~~~~~~~~~~~~~ From 8f1480ad6251c3d49ac210f2e602de2aab34bb87 Mon Sep 17 00:00:00 2001 From: viatrak Date: Fri, 24 Mar 2017 11:56:00 -0500 Subject: [PATCH 13/20] Add IDs to system checks. Can now be used in SILENCED_SYSTEM_CHECKS --- tenant_schemas/apps.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tenant_schemas/apps.py b/tenant_schemas/apps.py index d644a96..b7f254b 100644 --- a/tenant_schemas/apps.py +++ b/tenant_schemas/apps.py @@ -51,12 +51,14 @@ def best_practice(app_configs, **kwargs): obj="django.conf.settings", hint="This is necessary to overwrite built-in django " "management commands with their schema-aware " - "implementations.")) + "implementations.", + id="tenant_schemas.W001")) if not settings.TENANT_APPS: errors.append( Error("TENANT_APPS is empty.", - hint="Maybe you don't need this app?")) + hint="Maybe you don't need this app?", + id="tenant_schemas.E001")) if hasattr(settings, 'PG_EXTRA_SEARCH_PATHS'): if get_public_schema_name() in settings.PG_EXTRA_SEARCH_PATHS: @@ -74,25 +76,29 @@ def best_practice(app_configs, **kwargs): if not settings.SHARED_APPS: errors.append( - Warning("SHARED_APPS is empty.")) + Warning("SHARED_APPS is empty.", + id="tenant_schemas.W002")) if not set(settings.TENANT_APPS).issubset(INSTALLED_APPS): delta = set(settings.TENANT_APPS).difference(INSTALLED_APPS) errors.append( Error("You have TENANT_APPS that are not in INSTALLED_APPS", - hint=[a for a in settings.TENANT_APPS if a in delta])) + hint=[a for a in settings.TENANT_APPS if a in delta], + id="tenant_schemas.E002")) if not set(settings.SHARED_APPS).issubset(INSTALLED_APPS): delta = set(settings.SHARED_APPS).difference(INSTALLED_APPS) errors.append( Error("You have SHARED_APPS that are not in INSTALLED_APPS", - hint=[a for a in settings.SHARED_APPS if a in delta])) + hint=[a for a in settings.SHARED_APPS if a in delta], + id="tenant_schemas.E003")) if not isinstance(default_storage, TenantStorageMixin): errors.append(Warning( "Your default storage engine is not tenant aware.", hint="Set settings.DEFAULT_FILE_STORAGE to " "'tenant_schemas.storage.TenantFileSystemStorage'", + id="tenant_schemas.W003" )) return errors From 8cb4d174a670e8ce71c3762e10598418bb808b37 Mon Sep 17 00:00:00 2001 From: viatrak Date: Fri, 24 Mar 2017 12:19:52 -0500 Subject: [PATCH 14/20] Update tests to reflect new IDs in Errors and Warnings --- tenant_schemas/apps.py | 2 +- tenant_schemas/tests/test_apps.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tenant_schemas/apps.py b/tenant_schemas/apps.py index b7f254b..6b16b62 100644 --- a/tenant_schemas/apps.py +++ b/tenant_schemas/apps.py @@ -77,7 +77,7 @@ def best_practice(app_configs, **kwargs): if not settings.SHARED_APPS: errors.append( Warning("SHARED_APPS is empty.", - id="tenant_schemas.W002")) + id="tenant_schemas.W002")) if not set(settings.TENANT_APPS).issubset(INSTALLED_APPS): delta = set(settings.TENANT_APPS).difference(INSTALLED_APPS) diff --git a/tenant_schemas/tests/test_apps.py b/tenant_schemas/tests/test_apps.py index a14abd7..e0df917 100644 --- a/tenant_schemas/tests/test_apps.py +++ b/tenant_schemas/tests/test_apps.py @@ -63,7 +63,8 @@ class AppConfigTests(TestCase): obj="django.conf.settings", hint="This is necessary to overwrite built-in django " "management commands with their schema-aware " - "implementations."), + "implementations.", + id="tenant_schemas.W001"), ]) @override_settings(INSTALLED_APPS=[ @@ -83,7 +84,8 @@ class AppConfigTests(TestCase): def test_tenant_apps_empty(self): self.assertBestPractice([ Error("TENANT_APPS is empty.", - hint="Maybe you don't need this app?"), + hint="Maybe you don't need this app?", + id="tenant_schemas.E001"), ]) @override_settings(PG_EXTRA_SEARCH_PATHS=['public', 'demo1', 'demo2']) @@ -101,7 +103,8 @@ class AppConfigTests(TestCase): @override_settings(SHARED_APPS=()) def test_shared_apps_empty(self): self.assertBestPractice([ - Warning("SHARED_APPS is empty."), + Warning("SHARED_APPS is empty.", + id="tenant_schemas.W002"), ]) @override_settings(TENANT_APPS=( @@ -111,7 +114,8 @@ class AppConfigTests(TestCase): def test_tenant_app_missing_from_install_apps(self): self.assertBestPractice([ Error("You have TENANT_APPS that are not in INSTALLED_APPS", - hint=['django.contrib.flatpages']), + hint=['django.contrib.flatpages'], + id="tenant_schemas.E002"), ]) @override_settings(SHARED_APPS=( @@ -127,5 +131,6 @@ class AppConfigTests(TestCase): def test_shared_app_missing_from_install_apps(self): self.assertBestPractice([ Error("You have SHARED_APPS that are not in INSTALLED_APPS", - hint=['django.contrib.flatpages']), + hint=['django.contrib.flatpages'], + id="tenant_schemas.E003"), ]) From 1cb00dc6db5466b07d9154ba4569cdd18a78b277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Raimbault?= Date: Wed, 12 Apr 2017 07:06:19 +0200 Subject: [PATCH 15/20] Django 1.11 support (#463) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added Django 1.11 to Travis - related to #435 - 😉 to tomturner/django-tenants#120 * Replace direct use of import_module by proper load_backend Handle new 'django.db.backends.postgresql' name (Django 1.9) * Fix 'default' already present in _fields in Django 1.11 * Fix not existing access to original_backend.DatabaseError (Django 1.11) Removed exports because django.db.utils.DatabaseError and django.db.utils.IntegrityError must be used instead. This change must be stated in release notes. * The signature of DatabaseWrapper._cursor has changed in Django 1.11 Use the public API * Add missing allowed hosts for tests * Don't override default user or blocks Unix socket DB connection * Updated tox.ini for Django 1.11rc1 * Properly add '.test.com' to ALLOWED_HOSTS in test cases * Internal tests don't use TenantTestCase so set allowed hosts manually * Drop Django 1.9 support Django 1.8 is still within LTS until the end of 2017. --- .travis.yml | 2 +- dts_test_project/dts_test_project/settings.py | 8 +-- tenant_schemas/postgresql_backend/base.py | 51 ++++++++++--------- .../postgresql_backend/introspection.py | 6 ++- tenant_schemas/test/cases.py | 19 ++++++- tenant_schemas/tests/testcases.py | 9 ++++ tox.ini | 7 +-- 7 files changed, 69 insertions(+), 33 deletions(-) diff --git a/.travis.yml b/.travis.yml index d83ff55..3769b5a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,8 @@ script: - tox -e py${TRAVIS_PYTHON_VERSION/./}-dj${DJANGO/./} env: - DJANGO=1.8 -- DJANGO=1.9 - DJANGO=1.10 +- DJANGO=1.11 deploy: provider: pypi user: bcarneiro diff --git a/dts_test_project/dts_test_project/settings.py b/dts_test_project/dts_test_project/settings.py index 58716a7..245b017 100644 --- a/dts_test_project/dts_test_project/settings.py +++ b/dts_test_project/dts_test_project/settings.py @@ -82,10 +82,10 @@ DATABASES = { 'default': { 'ENGINE': 'tenant_schemas.postgresql_backend', 'NAME': os.environ.get('PG_NAME', 'dts_test_project'), - 'USER': os.environ.get('PG_USER', 'postgres'), - 'PASSWORD': os.environ.get('PG_PASSWORD', 'root'), - 'HOST': os.environ.get('PG_HOST', 'localhost'), - 'PORT': int(os.environ.get('PG_PORT', '5432')), + 'USER': os.environ.get('PG_USER'), + 'PASSWORD': os.environ.get('PG_PASSWORD'), + 'HOST': os.environ.get('PG_HOST'), + 'PORT': int(os.environ.get('PG_PORT')) if os.environ.get('PG_PORT') else None, } } diff --git a/tenant_schemas/postgresql_backend/base.py b/tenant_schemas/postgresql_backend/base.py index fb3cf1f..08555aa 100644 --- a/tenant_schemas/postgresql_backend/base.py +++ b/tenant_schemas/postgresql_backend/base.py @@ -1,21 +1,17 @@ import re import warnings -from django.conf import settings -try: - # Django versions >= 1.9 - from django.utils.module_loading import import_module -except ImportError: - # Django versions < 1.9 - from django.utils.importlib import import_module -from django.core.exceptions import ImproperlyConfigured, ValidationError -from tenant_schemas.utils import get_public_schema_name, get_limit_set_calls -from tenant_schemas.postgresql_backend.introspection import DatabaseSchemaIntrospection -import django.db.utils import psycopg2 -ORIGINAL_BACKEND = getattr(settings, 'ORIGINAL_BACKEND', 'django.db.backends.postgresql_psycopg2') +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured, ValidationError +import django.db.utils -original_backend = import_module(ORIGINAL_BACKEND + '.base') +from tenant_schemas.utils import get_public_schema_name, get_limit_set_calls +from tenant_schemas.postgresql_backend.introspection import DatabaseSchemaIntrospection + +ORIGINAL_BACKEND = getattr(settings, 'ORIGINAL_BACKEND', 'django.db.backends.postgresql_psycopg2') +# Django 1.9+ takes care to rename the default backend to 'django.db.backends.postgresql' +original_backend = django.db.utils.load_backend(ORIGINAL_BACKEND) EXTRA_SEARCH_PATHS = getattr(settings, 'PG_EXTRA_SEARCH_PATHS', []) @@ -109,12 +105,16 @@ class DatabaseWrapper(original_backend.DatabaseWrapper): category=DeprecationWarning) return self.tenant - def _cursor(self): + def _cursor(self, name=None): """ Here it happens. We hope every Django db operation using PostgreSQL must go through this to get the cursor handle. We change the path. """ - cursor = super(DatabaseWrapper, self)._cursor() + if name: + # Only supported and required by Django 1.11 (server-side cursor) + cursor = super(DatabaseWrapper, self)._cursor(name=name) + else: + cursor = super(DatabaseWrapper, self)._cursor() # optionally limit the number of executions - under load, the execution # of `set search_path` can be quite time consuming @@ -137,16 +137,28 @@ class DatabaseWrapper(original_backend.DatabaseWrapper): search_paths = [self.schema_name] search_paths.extend(EXTRA_SEARCH_PATHS) + + if name: + # Named cursor can only be used once + cursor_for_search_path = self.connection.cursor() + else: + # Reuse + cursor_for_search_path = cursor + # In the event that an error already happened in this transaction and we are going # to rollback we should just ignore database error when setting the search_path # if the next instruction is not a rollback it will just fail also, so # we do not have to worry that it's not the good one try: - cursor.execute('SET search_path = {0}'.format(','.join(search_paths))) + cursor_for_search_path.execute('SET search_path = {0}'.format(','.join(search_paths))) except (django.db.utils.DatabaseError, psycopg2.InternalError): self.search_path_set = False else: self.search_path_set = True + + if name: + cursor_for_search_path.close() + return cursor @@ -157,10 +169,3 @@ class FakeTenant: """ def __init__(self, schema_name): self.schema_name = schema_name - -if ORIGINAL_BACKEND == "django.contrib.gis.db.backends.postgis": - DatabaseError = django.db.utils.DatabaseError - IntegrityError = psycopg2.IntegrityError -else: - DatabaseError = original_backend.DatabaseError - IntegrityError = original_backend.IntegrityError diff --git a/tenant_schemas/postgresql_backend/introspection.py b/tenant_schemas/postgresql_backend/introspection.py index 4128e1e..ca445d4 100644 --- a/tenant_schemas/postgresql_backend/introspection.py +++ b/tenant_schemas/postgresql_backend/introspection.py @@ -7,7 +7,11 @@ from django.db.backends.base.introspection import ( ) from django.utils.encoding import force_text -FieldInfo = namedtuple('FieldInfo', FieldInfo._fields + ('default',)) +fields = FieldInfo._fields +if 'default' not in fields: + fields += ('default',) + +FieldInfo = namedtuple('FieldInfo', fields) class DatabaseSchemaIntrospection(BaseDatabaseIntrospection): diff --git a/tenant_schemas/test/cases.py b/tenant_schemas/test/cases.py index beb489c..cd05c6b 100644 --- a/tenant_schemas/test/cases.py +++ b/tenant_schemas/test/cases.py @@ -1,15 +1,30 @@ from django.core.management import call_command +from django.conf import settings from django.db import connection from django.test import TestCase from tenant_schemas.utils import get_public_schema_name from tenant_schemas.utils import get_tenant_model +ALLOWED_TEST_DOMAIN = '.test.com' + class TenantTestCase(TestCase): + @classmethod + def add_allowed_test_domain(cls): + # ALLOWED_HOSTS is a special setting of Django setup_test_environment so we can't modify it with helpers + if ALLOWED_TEST_DOMAIN not in settings.ALLOWED_HOSTS: + settings.ALLOWED_HOSTS += [ALLOWED_TEST_DOMAIN] + + @classmethod + def remove_allowed_test_domain(cls): + if ALLOWED_TEST_DOMAIN in settings.ALLOWED_HOSTS: + settings.ALLOWED_HOSTS.remove(ALLOWED_TEST_DOMAIN) + @classmethod def setUpClass(cls): cls.sync_shared() + cls.add_allowed_test_domain() tenant_domain = 'tenant.test.com' cls.tenant = get_tenant_model()(domain_url=tenant_domain, schema_name='test') cls.tenant.save(verbosity=0) # todo: is there any way to get the verbosity from the test command here? @@ -21,6 +36,7 @@ class TenantTestCase(TestCase): connection.set_schema_to_public() cls.tenant.delete() + cls.remove_allowed_test_domain() cursor = connection.cursor() cursor.execute('DROP SCHEMA IF EXISTS test CASCADE') @@ -33,10 +49,10 @@ class TenantTestCase(TestCase): class FastTenantTestCase(TenantTestCase): - @classmethod def setUpClass(cls): cls.sync_shared() + cls.add_allowed_test_domain() tenant_domain = 'tenant.test.com' TenantModel = get_tenant_model() @@ -51,3 +67,4 @@ class FastTenantTestCase(TenantTestCase): @classmethod def tearDownClass(cls): connection.set_schema_to_public() + cls.remove_allowed_test_domain() diff --git a/tenant_schemas/tests/testcases.py b/tenant_schemas/tests/testcases.py index 18531da..558eafa 100644 --- a/tenant_schemas/tests/testcases.py +++ b/tenant_schemas/tests/testcases.py @@ -20,6 +20,8 @@ class BaseTestCase(TestCase): 'django.contrib.contenttypes', 'django.contrib.auth', ) settings.INSTALLED_APPS = settings.SHARED_APPS + settings.TENANT_APPS + if '.test.com' not in settings.ALLOWED_HOSTS: + settings.ALLOWED_HOSTS += ['.test.com'] # Django calls syncdb by default for the test database, but we want # a blank public schema for this set of tests. @@ -29,6 +31,13 @@ class BaseTestCase(TestCase): % (get_public_schema_name(), get_public_schema_name())) super(BaseTestCase, cls).setUpClass() + @classmethod + def tearDownClass(cls): + super(BaseTestCase, cls).tearDownClass() + + if '.test.com' in settings.ALLOWED_HOSTS: + settings.ALLOWED_HOSTS.remove('.test.com') + def setUp(self): connection.set_schema_to_public() super(BaseTestCase, self).setUp() diff --git a/tox.ini b/tox.ini index fc4f8c5..6071ab3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,21 +1,22 @@ [tox] -envlist = py{27,35}-dj{18,19,110} +envlist = py{27,35}-dj{18,19,110,111} [testenv] usedevelop = True -deps = +deps = coverage mock tblib dj18: Django~=1.8.0 dj19: Django~=1.9.0 dj110: Django~=1.10.0 + dj111: Django~=1.11rc1 changedir = dts_test_project passenv = PG_NAME PG_USER PG_PASSWORD PG_HOST PG_PORT -commands = +commands = coverage run manage.py test --noinput {posargs:tenant_schemas} coverage report -m --include=../tenant_schemas/* From 8faa5a74bb6fff2ca0aed2d289a62c4441b52cc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Raimbault?= Date: Wed, 12 Apr 2017 07:08:55 +0200 Subject: [PATCH 16/20] Fix documentation about custom management commands (#464) --- docs/test.rst | 7 ++++--- docs/use.rst | 13 ++++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/test.rst b/docs/test.rst index 459b0c4..65968c6 100644 --- a/docs/test.rst +++ b/docs/test.rst @@ -1,6 +1,7 @@ -================== +===== Tests -================== +===== + Running the tests ----------------- Run these tests from the project ``dts_test_project``, it comes prepacked with the correct settings file and extra apps to enable tests to ensure different apps can exist in ``SHARED_APPS`` and ``TENANT_APPS``. @@ -27,7 +28,7 @@ Because django will not create tenants for you during your tests, we have packed class BaseSetup(TenantTestCase): def setUp(self): self.c = TenantClient(self.tenant) - + def test_user_profile_view(self): response = self.c.get(reverse('user_profile')) self.assertEqual(response.status_code, 200) diff --git a/docs/use.rst b/docs/use.rst index da4cf13..4373d09 100644 --- a/docs/use.rst +++ b/docs/use.rst @@ -1,6 +1,7 @@ =========================== Using django-tenant-schemas =========================== + Supported versions ------------------ You can use ``django-tenant-schemas`` with currently maintained versions of Django -- see the `Django's release process `_ and the present list of `Supported Versions `_. @@ -9,7 +10,7 @@ It is necessary to use a PostgreSQL database. ``django-tenant-schemas`` will ens Creating a Tenant ----------------- -Creating a tenant works just like any other model in django. The first thing we should do is to create the ``public`` tenant to make our main website available. We'll use the previous model we defined for ``Client``. +Creating a tenant works just like any other model in Django. The first thing we should do is to create the ``public`` tenant to make our main website available. We'll use the previous model we defined for ``Client``. .. code-block:: python @@ -43,7 +44,7 @@ Any call to the methods ``filter``, ``get``, ``save``, ``delete`` or any other f Management commands ------------------- -Every command except tenant_command runs by default on all tenants. You can also create your own commands that run on every tenant by inheriting ``BaseTenantCommand``. +By default, base commands run on the public tenant but you can also own commands that run on a specific tenant by inheriting ``BaseTenantCommand``. For example, if you have the following ``do_foo`` command in the ``foo`` app: @@ -57,7 +58,7 @@ For example, if you have the following ``do_foo`` command in the ``foo`` app: def handle(self, *args, **options): do_foo() -You could create a wrapper command ``tenant_do_foo`` by using ``BaseTenantCommand`` like so: +You could create a wrapper command by using ``BaseTenantCommand``: ``foo/management/commands/tenant_do_foo.py`` @@ -68,11 +69,13 @@ You could create a wrapper command ``tenant_do_foo`` by using ``BaseTenantComman class Command(BaseTenantCommand): COMMAND_NAME = 'do_foo' -To run only a particular schema, there is an optional argument called ``--schema``. +To run the command on a particular schema, there is an optional argument called ``--schema``. .. code-block:: bash - ./manage.py migrate_schemas --schema=customer1 + ./manage.py tenant_command do_foo --schema=customer1 + +If you omit the ``schema`` argument, the interactive shell will ask you to select one. migrate_schemas ~~~~~~~~~~~~~~~ From 42ef0234367b54bbe3a73397e9b609b668d25360 Mon Sep 17 00:00:00 2001 From: Gary Reynolds Date: Sun, 16 Apr 2017 22:35:33 +1000 Subject: [PATCH 17/20] Updates for Django 1.11 --- docs/conf.py | 4 ++-- docs/use.rst | 4 ++-- tox.ini | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 3b032f3..971871a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -94,8 +94,8 @@ pygments_style = 'sphinx' intersphinx_mapping = { 'django': ( - 'https://docs.djangoproject.com/en/1.10/', - 'https://docs.djangoproject.com/en/1.10/_objects/'), + 'https://docs.djangoproject.com/en/1.11/', + 'https://docs.djangoproject.com/en/1.11/_objects/'), } # -- Options for HTML output --------------------------------------------------- diff --git a/docs/use.rst b/docs/use.rst index 4373d09..4085fba 100644 --- a/docs/use.rst +++ b/docs/use.rst @@ -4,9 +4,9 @@ Using django-tenant-schemas Supported versions ------------------ -You can use ``django-tenant-schemas`` with currently maintained versions of Django -- see the `Django's release process `_ and the present list of `Supported Versions `_. +You can use ``django-tenant-schemas`` with currently maintained versions of Django -- see the `Django's release process `_ and the present list of `Supported Versions `_. -It is necessary to use a PostgreSQL database. ``django-tenant-schemas`` will ensure compatibility with the minimum required version of the latest Django release. At this time that is PostgreSQL 9.2, the minimum for Django 1.10. +It is necessary to use a PostgreSQL database. ``django-tenant-schemas`` will ensure compatibility with the minimum required version of the latest Django release. At this time that is PostgreSQL 9.3, the minimum for Django 1.11. Creating a Tenant ----------------- diff --git a/tox.ini b/tox.ini index 6071ab3..ffea785 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{27,35}-dj{18,19,110,111} +envlist = py{27,35}-dj{18,110,111} [testenv] usedevelop = True From 71019cded91ca0ab00627ae067001cc4c07564a0 Mon Sep 17 00:00:00 2001 From: Gary Reynolds Date: Sun, 16 Apr 2017 22:57:06 +1000 Subject: [PATCH 18/20] =?UTF-8?q?Bump=20version:=201.7.0=20=E2=86=92=201.8?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 72e6b8f..fb542ed 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.7.0 +current_version = 1.8.0 commit = True tag = True From 400fdc26f3aa612129c5c618e2fa65acfb39841c Mon Sep 17 00:00:00 2001 From: Bernardo Pires Date: Fri, 5 May 2017 16:15:10 -0500 Subject: [PATCH 19/20] Create install.rst Remove outdated setting TENANT_CREATION_FAKES_MIGRATIONS from the docs. Fixes #458. --- docs/install.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index 4b64894..e0c2124 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -175,12 +175,6 @@ Optional Settings The schema name that will be treated as ``public``, that is, where the ``SHARED_APPS`` will be created. -.. attribute:: TENANT_CREATION_FAKES_MIGRATIONS - - :Default: ``'True'`` - - Sets if the models will be synced directly to the last version and all migration subsequently faked. Useful in the cases where migrations can not be faked and need to be ran individually. Be aware that setting this to `False` may significantly slow down the process of creating tenants. - Tenant View-Routing ------------------- From 4a49516c4ab56639f7ede34e2b2d8967f7d2fd2c Mon Sep 17 00:00:00 2001 From: Bernardo Pires Date: Sat, 6 May 2017 07:21:48 -0500 Subject: [PATCH 20/20] Updated docs regarding tests --- docs/test.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/test.rst b/docs/test.rst index 65968c6..2f5d8b8 100644 --- a/docs/test.rst +++ b/docs/test.rst @@ -36,9 +36,7 @@ Because django will not create tenants for you during your tests, we have packed Running tests faster -------------------- -Using the ``TenantTestCase`` can make running your tests really slow quite early in your project. This is due to the fact that it drops, recreates the test schema and runs migrations for every ``TenantTestCase`` you have. If you want to gain speed, there's a ``FastTenantTestCase`` where the test schema will be created and migrations ran only one time. The gain in speed is noticiable but be aware that by using this you will be perpertraiting state between your test cases, please make sure your they wont be affected by this. - -Running tests using ``TenantTestCase`` can start being a bottleneck once the number of tests grow. If you do not care that the state between tests is kept, an alternative is to use the class ``FastTenantTestCase``. Unlike ``TenantTestCase``, the test schema and its migrations will only be created and ran once. This is a significant improvement in speed coming at the cost of shared state. +Running tests using ``TenantTestCase`` can start being a bottleneck once the number of tests grow. ``TenantTestCase`` drops, recreates and executes migrations for the test schema every time for every ``TenantTestCase`` you have. If you do not care that the state between tests is kept, an alternative is to use the class ``FastTenantTestCase``. Unlike ``TenantTestCase``, the test schema and its migrations will only be created and ran once. This is a significant improvement in speed coming at the cost of shared state. .. code-block:: python