misc: regroup wcs-olap runs by idp (#68185)

bijoe/uwsgi.py

@@ -21,6 +21,7 @@ import os
 import subprocess
 
 from django.conf import settings
+from django.db import connection
 from uwsgidecorators import cron, spool
 
 # existing loggers are disabled by Django on django.setup() due do
@@ -39,8 +40,9 @@ def log_exception(name):
 
 @spool(pass_arguments=True)
 @log_exception('wcs-olap job')
-def wcs_olap(wcs_olap_ini_path):
-    launch_wcs_olap(wcs_olap_ini_path)
+def wcs_olap(wcs_olap_ini_paths):
+    for wcs_olap_ini_path in wcs_olap_ini_paths:
+        launch_wcs_olap(wcs_olap_ini_path)
 
 
 @cron(0, 3, -1, -1, -1, target='spooler')
@@ -67,8 +69,24 @@ def launch_wcs_olap(wcs_olap_ini_path):
 def enqueue_wcs_olap():
     from hobo.multitenant.middleware import TenantMiddleware
 
+    tenant_by_idp = {}
     for tenant in TenantMiddleware.get_tenants():
         wcs_olap_ini_path = os.path.join(tenant.get_directory(), 'wcs-olap.ini')
-        if os.path.exists(wcs_olap_ini_path):
-            logger.info('enqueuing wcs-olap job for %s', wcs_olap_ini_path)
-            wcs_olap(wcs_olap_ini_path)
+        if not os.path.exists(wcs_olap_ini_path):
+            continue
+        connection.set_tenant(tenant)
+        if getattr(settings, 'TENANT_DISABLE_CRON_JOBS', False):
+            continue
+        try:
+            idp_url = list(settings.KNOWN_SERVICES['authentic'].values())[0]['url']
+        except (KeyError, IndexError):
+            # add tenant with a unique value as idp
+            tenant_by_idp[id(tenant)] = [wcs_olap_ini_path]
+        else:
+            if idp_url not in tenant_by_idp:
+                tenant_by_idp[idp_url] = []
+            tenant_by_idp[idp_url].append(wcs_olap_ini_path)
+
+    for wcs_olap_ini_paths in tenant_by_idp.values():
+        logger.info('enqueuing wcs-olap job for %s', wcs_olap_ini_paths)
+        wcs_olap(wcs_olap_ini_paths)

debian/uwsgi.ini

@@ -9,7 +9,7 @@ http-socket = /run/bijoe/bijoe.sock
 chmod-socket = 666
 vacuum = true
 
-spooler-processes = 1
+spooler-processes = 3
 spooler-python-import = hobo.provisionning.spooler
 spooler-max-tasks = 20