misc: use uwsgi spooler to launch wcs-olap (#56040)

This commit is contained in:
Benjamin Dauvergne 2021-08-07 20:45:04 +02:00
parent bee5061ace
commit 951bd3387f
6 changed files with 84 additions and 17 deletions

72
bijoe/uwsgi.py Normal file
View File

@ -0,0 +1,72 @@
# bijoe - BI dashboard
# Copyright (C) 2021 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import contextlib
import logging
import os
import subprocess
from django.conf import settings
from uwsgidecorators import cron, spool
# existing loggers are disabled by Django on django.setup() due do
# @'disable_existing_loggers': True@, this module must be loaded after
# django.setup() (or get_wsgi_application()) for the logger to work
logger = logging.getLogger(__name__)
@contextlib.contextmanager
def log_exception(name):
try:
yield
except Exception:
logger.exception('Exception during %s', name)
@spool(pass_arguments=True)
@log_exception('wcs-olap job')
def wcs_olap(wcs_olap_ini_path):
launch_wcs_olap(wcs_olap_ini_path)
@cron(0, 3, -1, -1, -1, target='spooler')
@log_exception('enqueuing of wcs-olap jobs')
def cron_enqueue_wcs_olap(num):
enqueue_wcs_olap()
def launch_wcs_olap(wcs_olap_ini_path):
logger.info('start wcs-olap on %s', wcs_olap_ini_path)
subprocess.run(
[
settings.WCS_OLAP_COMMAND,
'--all',
wcs_olap_ini_path,
],
check=False)
logger.info('finished wcs-olap on %s', wcs_olap_ini_path)
def enqueue_wcs_olap():
from hobo.multitenant.middleware import TenantMiddleware
for tenant in TenantMiddleware.get_tenants():
wcs_olap_ini_path = os.path.join(tenant.get_directory(), 'wcs-olap.ini')
if os.path.exists(wcs_olap_ini_path):
logger.info('enqueuing wcs-olap job for %s', wcs_olap_ini_path)
wcs_olap(wcs_olap_ini_path)

View File

@ -24,7 +24,14 @@ https://docs.djangoproject.com/en/1.7/howto/deployment/wsgi/
"""
import os
import sys
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "bijoe.settings")
from django.core.wsgi import get_wsgi_application
from django.core.wsgi import get_wsgi_application # noqa: E402
application = get_wsgi_application()
# load uwsgi tasks
if 'uwsgi' in sys.modules:
import bijoe.uwsgi # noqa: F401

2
debian/bijoe.cron.d vendored
View File

@ -2,5 +2,3 @@ PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
MAILTO=root
0 * * * * bijoe bijoe-manage tenant_command clearsessions --all-tenants
1 0 * * * bijoe /usr/lib/bijoe/import-wcs-data.sh

View File

@ -4,5 +4,4 @@ debian/bijoe.service /lib/systemd/system
debian/settings.py /etc/bijoe
debian/uwsgi.ini /etc/bijoe
debian/debian_config.py /usr/lib/bijoe
debian/import-wcs-data.sh /usr/lib/bijoe
README.rst /usr/share/doc/bijoe

View File

@ -29,6 +29,10 @@ MELLON_IDENTITY_PROVIDERS = []
# Override default hobo agent
INSTALLED_APPS = ('bijoe.hobo_agent',) + INSTALLED_APPS
WCS_OLAP_COMMAND = '/usr/bin/wcs-olap'
#
# local settings
#

View File

@ -1,13 +0,0 @@
#!/bin/bash
LOG=`tempfile`
trap "rm $LOG" EXIT
for tenant in /var/lib/bijoe/tenants/*; do
if [ -n "${tenant##*.invalid*}" -a -f $tenant/wcs-olap.ini ]; then
wcs-olap --all $tenant/wcs-olap.ini >$LOG 2>&1 || cat $LOG
fi
done
test -d /etc/bijoe/wcs-olap/post-sync.d/ && run-parts --exit-on-error --new-session /etc/bijoe/wcs-olap/post-sync.d/