entrouvert/wcs
entrouvert
/
wcs
14
1
Fork 0
Code Issues Pull Requests 36 Releases Activity

general: add support for tenants in a tenants subdirectory (#50828)

This commit is contained in:
Frédéric Péters 2021-05-02 21:59:07 +02:00
parent 675d37b8ab
commit 06b7412dd6
16 changed files with 114 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
tests/test_hobo.py
View File

@ -387,18 +387,20 @@ def test_deploy():
hobo_cmd.execute(
base_options, sub_options, ['http://wcs.example.net/', os.path.join(alt_tempdir, 'hobo.json')]
)
assert os.path.exists(os.path.join(alt_tempdir, 'wcs.example.net'))
assert os.path.exists(os.path.join(alt_tempdir, 'tenants', 'wcs.example.net'))
# update
cleanup()
pub_cfg = pickle.load(open(os.path.join(alt_tempdir, 'wcs.example.net', 'config.pck'), 'rb'))
pub_cfg = pickle.load(open(os.path.join(alt_tempdir, 'tenants', 'wcs.example.net', 'config.pck'), 'rb'))
assert pub_cfg['language'] == {'language': 'fr'}
del pub_cfg['language']
pickle.dump(pub_cfg, open(os.path.join(alt_tempdir, 'wcs.example.net', 'config.pck'), 'wb'))
pickle.dump(pub_cfg, open(os.path.join(alt_tempdir, 'tenants', 'wcs.example.net', 'config.pck'), 'wb'))
hobo_cmd.execute(
base_options, sub_options, ['http://wcs.example.net/', os.path.join(alt_tempdir, 'hobo.json')]
base_options,
sub_options,
['http://wcs.example.net/', os.path.join(alt_tempdir, 'tenants', 'wcs.example.net', 'hobo.json')],
)
pub_cfg = pickle.load(open(os.path.join(alt_tempdir, 'wcs.example.net', 'config.pck'), 'rb'))
pub_cfg = pickle.load(open(os.path.join(alt_tempdir, 'tenants', 'wcs.example.net', 'config.pck'), 'rb'))
assert pub_cfg['language'] == {'language': 'fr'}
@ -420,16 +422,16 @@ def test_configure_postgresql():
hobo_cmd.execute(
base_options, sub_options, ['http://wcs.example.net/', os.path.join(alt_tempdir, 'hobo.json')]
)
assert os.path.exists(os.path.join(alt_tempdir, 'wcs.example.net'))
assert os.path.exists(os.path.join(alt_tempdir, 'tenants', 'wcs.example.net'))
with open(os.path.join(alt_tempdir, 'wcs.example.net', 'site-options.cfg'), 'w') as fd:
with open(os.path.join(alt_tempdir, 'tenants', 'wcs.example.net', 'site-options.cfg'), 'w') as fd:
fd.write('[options]\n')
fd.write('postgresql = true\n')
cleanup()
pub = WcsPublisher.create_publisher(register_tld_names=False)
pub.app_dir = os.path.join(alt_tempdir, 'wcs.example.net')
pub.app_dir = os.path.join(alt_tempdir, 'tenants', 'wcs.example.net')
pub.cfg['postgresql'] = {
'createdb-connection-params': {'user': 'test', 'database': 'postgres'},
'database-template-name': 'tests_wcs_%s',

30
tests/test_publisher.py
View File

@ -21,6 +21,7 @@ from wcs.qommon import get_publisher_class
from wcs.qommon.afterjobs import AfterJob
from wcs.qommon.cron import CronJob
from wcs.qommon.http_request import HTTPRequest
from wcs.qommon.publisher import Tenant
from .utilities import create_temporary_pub
@ -154,10 +155,10 @@ def test_get_tenants():
pub = create_temporary_pub()
open(os.path.join(pub.APP_DIR, 'xxx'), 'w').close()
os.mkdir(os.path.join(pub.APP_DIR, 'plop.invalid'))
tenants = list(pub.__class__.get_tenants())
assert 'example.net' in tenants
assert 'xxx' not in tenants
assert 'plop.invalid' not in tenants
hostnames = [x.hostname for x in pub.__class__.get_tenants()]
assert 'example.net' in hostnames
assert 'xxx' not in hostnames
assert 'plop.invalid' not in hostnames
def test_register_cronjobs():
@ -203,9 +204,16 @@ def test_cron_command():
with mock.patch('tempfile.gettempdir') as gettempdir:
gettempdir.side_effect = lambda: pub.app_dir
hostnames = ['example.net', 'foo.bar', 'something.com']
for hostname in hostnames:
if not os.path.exists(os.path.join(pub.APP_DIR, hostname)):
os.mkdir(os.path.join(pub.APP_DIR, hostname))
with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker:
with mock.patch('wcs.qommon.publisher.QommonPublisher.get_tenants') as mock_tenants:
mock_tenants.return_value = ['example.net', 'foo.bar', 'something.com']
mock_tenants.return_value = [
Tenant(os.path.join(pub.app_dir, x)) for x in ('example.net', 'foo.bar', 'something.com')
]
call_command('cron')
assert cron_worker.call_count == 3
cron_worker.reset_mock()
@ -229,24 +237,24 @@ def test_cron_command():
# verify that the lock is released
with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker:
call_command('cron')
call_command('cron', domain='example.net')
assert cron_worker.call_count == 1
# simulate a cron crash
with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker:
cron_worker.side_effect = NotImplementedError
with pytest.raises(NotImplementedError):
call_command('cron')
call_command('cron', domain='example.net')
assert cron_worker.call_count == 1
# verify that the lock is released
with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker:
call_command('cron')
call_command('cron', domain='example.net')
assert cron_worker.call_count == 1
# disable cron system
with override_settings(DISABLE_CRON_JOBS=True):
with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker:
call_command('cron')
call_command('cron', domain='example.net')
assert cron_worker.call_count == 0
# run a specific job
@ -269,10 +277,10 @@ def test_cron_command():
with mock.patch('wcs.publisher.WcsPublisher.register_cronjobs', register_test_cronjobs):
get_publisher_class().cronjobs = []
call_command('cron', job_name='job1')
call_command('cron', job_name='job1', domain='example.net')
assert jobs == []
get_publisher_class().cronjobs = []
call_command('cron', job_name='job2')
call_command('cron', job_name='job2', domain='example.net')
assert jobs == ['job2']

26
wcs/ctl/check_hobos.py
View File

@ -32,6 +32,7 @@ from wcs.qommon import force_str
from ..qommon import misc
from ..qommon.ctl import Command, make_option
from ..qommon.publisher import UnknownTenantError
from ..qommon.storage import atomic_write
@ -72,7 +73,7 @@ class CmdCheckHobos(Command):
if sub_options.redeploy:
sub_options.ignore_timestamp = True
for tenant in publisher.WcsPublisher.get_tenants():
hobo_json_path = os.path.join(publisher.WcsPublisher.APP_DIR, tenant, 'hobo.json')
hobo_json_path = os.path.join(tenant.directory, 'hobo.json')
if not os.path.exists(hobo_json_path):
continue
hobo_json = json.load(open(hobo_json_path))
@ -99,6 +100,7 @@ class CmdCheckHobos(Command):
pub = publisher.WcsPublisher.create_publisher(register_tld_names=False)
global_app_dir = pub.app_dir
global_tenants_dir = os.path.join(global_app_dir, 'tenants')
base_url = args[0]
if args[1] == '-':
@ -111,7 +113,9 @@ class CmdCheckHobos(Command):
service = [
x
for x in self.all_services.get('services', [])
if x.get('service-id') == 'wcs' and x.get('base_url') == base_url and not x.get('secondary')
if x.get('service-id') == 'wcs'
and x.get('base_url') in (base_url, base_url.rstrip('/'))
and not x.get('secondary')
][0]
except IndexError:
return
@ -120,10 +124,16 @@ class CmdCheckHobos(Command):
if base_url.endswith('/'): # wcs doesn't expect a trailing slash
service['base_url'] = base_url[:-1]
pub.app_dir = os.path.join(global_app_dir, self.get_instance_path(service))
if not os.path.exists(pub.app_dir):
print('initializing instance in', pub.app_dir)
os.mkdir(pub.app_dir)
try:
pub.set_tenant_by_hostname(self.get_instance_path(service), skip_sql=True)
except UnknownTenantError:
if not os.path.exists(global_tenants_dir):
os.mkdir(global_tenants_dir)
tenant_app_dir = os.path.join(global_tenants_dir, self.get_instance_path(service))
if not os.path.exists(tenant_app_dir):
print('initializing instance in', tenant_app_dir)
os.mkdir(tenant_app_dir)
pub.set_tenant_by_hostname(self.get_instance_path(service))
if service.get('template_name'):
skeleton_filepath = os.path.join(global_app_dir, 'skeletons', service.get('template_name'))
@ -148,7 +158,9 @@ class CmdCheckHobos(Command):
self.update_profile(self.all_services.get('profile', {}), pub)
# Store hobo.json
atomic_write(os.path.join(pub.app_dir, 'hobo.json'), force_bytes(json.dumps(self.all_services)))
atomic_write(
os.path.join(pub.tenant.directory, 'hobo.json'), force_bytes(json.dumps(self.all_services))
)
def update_configuration(self, service, pub):
if not pub.cfg.get('misc'):

5
wcs/ctl/delete_tenant.py
View File

@ -41,10 +41,7 @@ class CmdDeleteTenant(Command):
publisher.WcsPublisher.configure(self.config)
pub = publisher.WcsPublisher.create_publisher(register_tld_names=False)
hostname = args[0]
pub.app_dir = os.path.join(pub.app_dir, hostname)
pub.set_config()
pub.set_tenant_by_hostname(args[0])
self.delete_tenant(pub, sub_options, args)
def delete_tenant(self, pub, options, args):

9
wcs/ctl/hobo_notify.py
View File

@ -46,13 +46,10 @@ class CmdHoboNotify(Command):
publisher.WcsPublisher.configure(self.config)
pub = publisher.WcsPublisher.create_publisher(register_tld_names=False)
global_app_dir = pub.app_dir
for hostname in publisher.WcsPublisher.get_tenants():
app_dir = os.path.join(global_app_dir, hostname)
if not os.path.exists(os.path.join(app_dir, 'config.pck')):
for tenant in publisher.WcsPublisher.get_tenants():
if not os.path.exists(os.path.join(tenant.directory, 'config.pck')):
continue
pub.app_dir = app_dir
pub.set_config()
pub.set_tenant(tenant)
self.process_notification(notification, pub)
@classmethod

8
wcs/ctl/management/commands/__init__.py
View File

@ -18,16 +18,16 @@ import os
from django.core.management.base import BaseCommand, CommandError
from wcs.qommon.publisher import get_publisher_class
from wcs.qommon.publisher import UnknownTenantError, get_publisher_class
class TenantCommand(BaseCommand):
def init_tenant_publisher(self, domain, **kwargs):
publisher = get_publisher_class().create_publisher(**kwargs)
if not domain in publisher.get_tenants():
try:
publisher.set_tenant_by_hostname(domain)
except UnknownTenantError:
raise CommandError('unknown tenant')
publisher.app_dir = os.path.join(publisher.APP_DIR, domain)
publisher.set_config()
publisher.install_lang()
publisher.substitutions.feed(publisher)
return publisher

11
wcs/ctl/management/commands/convert_to_sql.py
View File

@ -26,7 +26,7 @@ from django.utils.encoding import force_bytes
from wcs import sql
from wcs.formdef import FormDef
from wcs.qommon.misc import localstrftime
from wcs.qommon.publisher import get_publisher_class
from wcs.qommon.publisher import UnknownTenantError, get_publisher_class
from wcs.qommon.storage import atomic_write
from wcs.users import User
@ -59,12 +59,11 @@ class Command(BaseCommand):
def get_publisher(self, domain):
publisher_class = get_publisher_class()
if domain not in publisher_class.get_tenants():
raise CommandError('unknown tenant')
publisher = publisher_class.create_publisher()
publisher.app_dir = os.path.join(publisher.app_dir, domain)
publisher.set_config()
try:
publisher.set_tenant_by_hostname(domain)
except UnknownTenantError:
raise CommandError('unknown tenant')
return publisher
def setup_connection(self, **kwargs):

3
wcs/ctl/management/commands/dbshell.py
View File

@ -14,6 +14,7 @@
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.
from django.core.management.base import CommandError
from django.db.backends.postgresql.client import DatabaseClient
from . import TenantCommand
@ -24,5 +25,7 @@ class Command(TenantCommand):
parser.add_argument('-d', '--domain', '--vhost', metavar='DOMAIN')
def handle(self, *args, **options):
if not options['domain']:
raise CommandError('missing hostname')
pub = self.init_tenant_publisher(options['domain'], register_tld_names=False)
DatabaseClient.runshell_db(conn_params=pub.cfg['postgresql'])

2
wcs/ctl/management/commands/runscript.py
View File

@ -48,7 +48,7 @@ class Command(TenantCommand):
if domain:
domains = [domain]
else:
domains = get_publisher_class().get_tenants()
domains = [x.hostname for x in get_publisher_class().get_tenants()]
fullpath = os.path.dirname(os.path.abspath(args[0]))
sys.path.insert(0, fullpath)
module_name = os.path.splitext(os.path.basename(args[0]))[0].encode('utf-8')

8
wcs/ctl/rebuild_indexes.py
View File

@ -14,8 +14,6 @@
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.
import os
from ..qommon.ctl import Command, make_option
@ -62,14 +60,12 @@ class CmdRebuildIndexes(Command):
publisher.WcsPublisher.configure(self.config)
pub = publisher.WcsPublisher.create_publisher(register_tld_names=False)
app_dir = pub.app_dir
if sub_options.all:
hostnames = publisher.WcsPublisher.get_tenants()
hostnames = [x.domain for x in publisher.WcsPublisher.get_tenants()]
else:
hostnames = args
for hostname in hostnames:
pub.app_dir = os.path.join(app_dir, hostname)
pub.set_config()
pub.set_tenant(hostname)
rebuild_vhost_indexes(pub, destroy=sub_options.destroy)

3
wcs/ctl/runscript.py
View File

@ -42,8 +42,7 @@ class CmdRunScript(Command):
self.config.remove_option('main', 'error_log')
publisher.WcsPublisher.configure(self.config)
publisher = publisher.WcsPublisher.create_publisher(register_tld_names=False)
publisher.app_dir = os.path.join(publisher.app_dir, sub_options.vhost)
publisher.set_config()
publisher.set_tenant_by_hostname(sub_options.vhost)
fullpath = os.path.dirname(os.path.abspath(args[0]))
sys.path.insert(0, fullpath)
module_name = os.path.splitext(os.path.basename(args[0]))[0]

4
wcs/ctl/wipe_data.py
View File

@ -14,7 +14,6 @@
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.
import os
import sys
from ..qommon.ctl import Command, make_option
@ -41,8 +40,7 @@ class CmdWipeData(Command):
publisher.WcsPublisher.configure(self.config)
pub = publisher.WcsPublisher.create_publisher(register_tld_names=False)
pub.app_dir = os.path.join(pub.app_dir, sub_options.vhost)
pub.set_config()
pub.set_tenant_by_hostname(sub_options.vhost)
self.wipe(pub, sub_options, args)
def wipe(self, pub, options, args):

5
wcs/qommon/cron.py
View File

@ -37,11 +37,6 @@ class CronJob:
def cron_worker(publisher, now, job_name=None):
try:
publisher.set_config()
except Exception:
return
# reindex user and formdata if needed (should only be run once)
if publisher.is_using_postgresql():
publisher.reindex_sql()

5
wcs/qommon/management/commands/cron.py
View File

@ -50,7 +50,7 @@ class Command(BaseCommand):
if options.get('domain'):
domains = [options.get('domain')]
else:
domains = get_publisher_class().get_tenants()
domains = [x.hostname for x in get_publisher_class().get_tenants()]
try:
with locket.lock_file(lockfile, timeout=0):
if verbosity > 2:
@ -59,11 +59,10 @@ class Command(BaseCommand):
publisher_class = get_publisher_class()
publisher_class.register_cronjobs()
publisher = publisher_class.create_publisher()
app_dir = publisher.app_dir
for hostname in domains:
if verbosity > 1:
print('cron work on %s' % hostname)
publisher.app_dir = os.path.join(app_dir, hostname)
publisher.set_tenant_by_hostname(hostname)
cron_worker(publisher, now, job_name=options.get('job_name'))
if verbosity > 2:
print('cron end (release lock %s)' % lockfile)

13
wcs/qommon/management/commands/migrate.py
View File

@ -14,8 +14,6 @@
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.
import os
import quixote
from django.core.management.base import BaseCommand
@ -29,17 +27,12 @@ class Command(BaseCommand):
Publisher = get_publisher_class()
quixote.cleanup()
pub = Publisher.create_publisher()
base_app_dir = pub.app_dir
for hostname in Publisher.get_tenants():
tenant_path = os.path.join(base_app_dir, hostname)
if not os.path.exists(os.path.join(tenant_path, 'config.pck')):
continue
for tenant in Publisher.get_tenants():
pub = Publisher.create_publisher()
pub.app_dir = tenant_path
pub.set_config()
pub.set_tenant(tenant)
if pub.is_using_postgresql():
if verbosity:
print('Running migrations for', hostname)
print('Running migrations for', tenant.hostname)
pub.migrate_sql()
pub.cleanup()
quixote.cleanup()

57
wcs/qommon/publisher.py
View File

@ -58,6 +58,16 @@ class ImmediateRedirectException(Exception):
self.location = location
class UnknownTenantError(Exception):
pass
class Tenant:
def __init__(self, directory):
self.directory = directory
self.hostname = os.path.basename(directory)
class QommonPublisher(Publisher):
APP_NAME = None
APP_DIR = None
@ -448,14 +458,10 @@ class QommonPublisher(Publisher):
"""
self.site_options = None # reset at the beginning of a request
canonical_hostname = request.get_server(clean=False).lower().split(':')[0].rstrip('.')
script_name = request.get_header('SCRIPT_NAME', '').strip('/')
self.app_dir = os.path.join(self.APP_DIR, canonical_hostname)
if script_name:
script_name = script_name.replace('/', '+')
self.app_dir += '+' + script_name
if not os.path.exists(self.app_dir):
try:
self.set_tenant_by_hostname(canonical_hostname, request=request)
except UnknownTenantError:
if self.missing_appdir_redirect:
raise ImmediateRedirectException(self.missing_appdir_redirect)
raise Http404()
@ -468,7 +474,6 @@ class QommonPublisher(Publisher):
def init_publish(self, request):
self.set_app_dir(request)
self.set_config(request)
self._http_adapter = None
request.language = self.get_site_language()
@ -947,18 +952,36 @@ class QommonPublisher(Publisher):
cls.extra_sources = []
cls.extra_sources.append(source)
def set_tenant(self, tenant, **kwargs):
self.tenant = tenant
self.app_dir = tenant.directory
self.set_config(**kwargs)
def set_tenant_by_hostname(self, hostname, **kwargs):
for base_dir in (os.path.join(self.APP_DIR, 'tenants'), self.APP_DIR):
tenant_dir = os.path.join(base_dir, hostname)
if os.path.exists(tenant_dir):
self.set_tenant(Tenant(tenant_dir), **kwargs)
break
else:
raise UnknownTenantError(hostname)
@classmethod
def get_tenants(cls):
for tenant in sorted(os.listdir(cls.APP_DIR)):
if tenant in ('collectstatic', 'scripts', 'skeletons', 'spooler'):
for base_dir in (cls.APP_DIR, os.path.join(cls.APP_DIR, 'tenants')):
if not os.path.exists(base_dir):
continue
if tenant.endswith('.invalid'):
continue
if not os.path.isdir(os.path.join(cls.APP_DIR, tenant)):
continue
if not os.access(os.path.join(cls.APP_DIR, tenant), os.W_OK):
continue
yield tenant
for tenant in sorted(os.listdir(base_dir)):
if tenant in ('collectstatic', 'scripts', 'skeletons', 'spooler', 'tenants'):
continue
if tenant.endswith('.invalid'):
continue
tenant_dir = os.path.join(base_dir, tenant)
if not os.path.isdir(tenant_dir):
continue
if not os.access(tenant_dir, os.W_OK):
continue
yield Tenant(tenant_dir)
def get_cfg(key, default=None):