From 951bd3387f41db54c58de091ca10ff7fcb92bda7 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Sat, 7 Aug 2021 20:45:04 +0200 Subject: [PATCH] misc: use uwsgi spooler to launch wcs-olap (#56040) --- bijoe/uwsgi.py | 72 +++++++++++++++++++++++++++++++++++++++ bijoe/wsgi.py | 9 ++++- debian/bijoe.cron.d | 2 -- debian/bijoe.install | 1 - debian/debian_config.py | 4 +++ debian/import-wcs-data.sh | 13 ------- 6 files changed, 84 insertions(+), 17 deletions(-) create mode 100644 bijoe/uwsgi.py delete mode 100755 debian/import-wcs-data.sh diff --git a/bijoe/uwsgi.py b/bijoe/uwsgi.py new file mode 100644 index 0000000..f8f0153 --- /dev/null +++ b/bijoe/uwsgi.py @@ -0,0 +1,72 @@ +# bijoe - BI dashboard +# Copyright (C) 2021 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import contextlib +import logging +import os +import subprocess + +from django.conf import settings + +from uwsgidecorators import cron, spool + +# existing loggers are disabled by Django on django.setup() due do +# @'disable_existing_loggers': True@, this module must be loaded after +# django.setup() (or get_wsgi_application()) for the logger to work +logger = logging.getLogger(__name__) + + +@contextlib.contextmanager +def log_exception(name): + try: + yield + except Exception: + logger.exception('Exception during %s', name) + + +@spool(pass_arguments=True) +@log_exception('wcs-olap job') +def wcs_olap(wcs_olap_ini_path): + launch_wcs_olap(wcs_olap_ini_path) + + +@cron(0, 3, -1, -1, -1, target='spooler') +@log_exception('enqueuing of wcs-olap jobs') +def cron_enqueue_wcs_olap(num): + enqueue_wcs_olap() + + +def launch_wcs_olap(wcs_olap_ini_path): + logger.info('start wcs-olap on %s', wcs_olap_ini_path) + subprocess.run( + [ + settings.WCS_OLAP_COMMAND, + '--all', + wcs_olap_ini_path, + ], + check=False) + logger.info('finished wcs-olap on %s', wcs_olap_ini_path) + + +def enqueue_wcs_olap(): + from hobo.multitenant.middleware import TenantMiddleware + + for tenant in TenantMiddleware.get_tenants(): + wcs_olap_ini_path = os.path.join(tenant.get_directory(), 'wcs-olap.ini') + if os.path.exists(wcs_olap_ini_path): + logger.info('enqueuing wcs-olap job for %s', wcs_olap_ini_path) + wcs_olap(wcs_olap_ini_path) diff --git a/bijoe/wsgi.py b/bijoe/wsgi.py index be839e4..ba5a2e5 100644 --- a/bijoe/wsgi.py +++ b/bijoe/wsgi.py @@ -24,7 +24,14 @@ https://docs.djangoproject.com/en/1.7/howto/deployment/wsgi/ """ import os +import sys + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "bijoe.settings") -from django.core.wsgi import get_wsgi_application +from django.core.wsgi import get_wsgi_application # noqa: E402 + application = get_wsgi_application() + +# load uwsgi tasks +if 'uwsgi' in sys.modules: + import bijoe.uwsgi # noqa: F401 diff --git a/debian/bijoe.cron.d b/debian/bijoe.cron.d index 03d7402..0514874 100644 --- a/debian/bijoe.cron.d +++ b/debian/bijoe.cron.d @@ -2,5 +2,3 @@ PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin MAILTO=root 0 * * * * bijoe bijoe-manage tenant_command clearsessions --all-tenants -1 0 * * * bijoe /usr/lib/bijoe/import-wcs-data.sh - diff --git a/debian/bijoe.install b/debian/bijoe.install index 524032d..93c8a60 100644 --- a/debian/bijoe.install +++ b/debian/bijoe.install @@ -4,5 +4,4 @@ debian/bijoe.service /lib/systemd/system debian/settings.py /etc/bijoe debian/uwsgi.ini /etc/bijoe debian/debian_config.py /usr/lib/bijoe -debian/import-wcs-data.sh /usr/lib/bijoe README.rst /usr/share/doc/bijoe diff --git a/debian/debian_config.py b/debian/debian_config.py index d75262a..ee7bed8 100644 --- a/debian/debian_config.py +++ b/debian/debian_config.py @@ -29,6 +29,10 @@ MELLON_IDENTITY_PROVIDERS = [] # Override default hobo agent INSTALLED_APPS = ('bijoe.hobo_agent',) + INSTALLED_APPS + +WCS_OLAP_COMMAND = '/usr/bin/wcs-olap' + + # # local settings # diff --git a/debian/import-wcs-data.sh b/debian/import-wcs-data.sh deleted file mode 100755 index ad5075e..0000000 --- a/debian/import-wcs-data.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -LOG=`tempfile` - -trap "rm $LOG" EXIT - -for tenant in /var/lib/bijoe/tenants/*; do - if [ -n "${tenant##*.invalid*}" -a -f $tenant/wcs-olap.ini ]; then - wcs-olap --all $tenant/wcs-olap.ini >$LOG 2>&1 || cat $LOG - fi -done - -test -d /etc/bijoe/wcs-olap/post-sync.d/ && run-parts --exit-on-error --new-session /etc/bijoe/wcs-olap/post-sync.d/