chrono/agendas/management/commands/sync_desks_timeperiod_exceptions_from_settings.py

@@ -27,4 +27,4 @@ class Command(BaseCommand):
     def handle(self, **options):
         translation.activate(settings.LANGUAGE_CODE)
         for desk in Desk.objects.all():
-            desk.import_timeperiod_exceptions_from_settings(spool=False)
+            desk.import_timeperiod_exceptions_from_settings()

chrono/agendas/models.py

@@ -22,7 +22,6 @@ import functools
 import itertools
 import logging
 import math
-import sys
 import uuid
 from contextlib import contextmanager
 
@@ -36,7 +35,7 @@ from django.contrib.humanize.templatetags.humanize import ordinal
 from django.contrib.postgres.fields import ArrayField
 from django.core.exceptions import ValidationError
 from django.core.validators import MaxValueValidator, MinValueValidator
-from django.db import connection, models, transaction
+from django.db import models
 from django.db.models import (
     Count,
     Exists,
@@ -75,6 +74,7 @@ from django.utils.translation import gettext_lazy as _
 from django.utils.translation import ngettext, pgettext_lazy
 
 from chrono.interval import Interval, IntervalSet
+from chrono.utils import spooler, transaction
 from chrono.utils.date import get_weekday_index
 from chrono.utils.db import ArraySubquery, SumCardinality
 from chrono.utils.misc import AgendaImportError, ICSError, clean_import_data, generate_slug
@@ -1599,18 +1599,7 @@ class Event(models.Model):
         self.async_notify_checked()
 
     def async_notify_checked(self):
-        if 'uwsgi' in sys.modules:
-            from chrono.utils.spooler import event_notify_checked
-
-            tenant = getattr(connection, 'tenant', None)
-            transaction.on_commit(
-                lambda: event_notify_checked.spool(
-                    event_id=str(self.pk), domain=getattr(tenant, 'domain_url', None)
-                )
-            )
-            return
-
-        self.notify_checked()
+        transaction.on_commit(lambda: spooler.run(self.notify_checked))
 
     def notify_checked(self):
         for booking in self.booking_set.filter(user_was_present__isnull=False):
@@ -2517,7 +2506,7 @@ class Desk(models.Model):
 
         return [OpeningHour(*time_range) for time_range in (openslots - exceptions)]
 
-    def import_timeperiod_exceptions_from_settings(self, enable=False, spool=True):
+    def import_timeperiod_exceptions_from_settings(self, enable=False):
         start_update = now()
         for slug, source_info in settings.EXCEPTIONS_SOURCES.items():
             label = source_info['label']
@@ -2530,7 +2519,7 @@ class Desk(models.Model):
             source.settings_label = _(label)
             source.save()
             if enable or source.enabled:  # if already enabled, update anyway
-                source.enable(spool=spool)
+                source.enable()
         TimePeriodExceptionSource.objects.filter(
             desk=self, settings_slug__isnull=False, last_update__lt=start_update
         ).delete()  # source was not in settings anymore
@@ -2653,22 +2642,14 @@ class TimePeriodExceptionSource(models.Model):
 
         return new_source
 
-    def enable(self, spool=True):
+    def enable(self):
         self.enabled = True
         self.save()
-
-        if spool and 'uwsgi' in sys.modules:
-            from chrono.utils.spooler import refresh_exceptions_from_settings
-
-            tenant = getattr(connection, 'tenant', None)
-            transaction.on_commit(
-                lambda: refresh_exceptions_from_settings.spool(
-                    source_id=str(self.pk), domain=getattr(tenant, 'domain_url', None)
-                )
-            )
-            return
-
-        self.refresh_from_settings()
+        # the spooler needs to pickle self.refresh_from_settings but self
+        # contains lazy strings (slugify() of gettext_lazy() results), to clear
+        # that we call refresh_from_db()
+        self.refresh_from_db()
+        transaction.on_commit(lambda: spooler.run(self.refresh_from_settings))
 
     def refresh_from_settings(self):
         if not self.enabled:
@@ -2701,6 +2682,10 @@ class TimePeriodExceptionSource(models.Model):
     def render_ics_url(self):
         return Template(self.ics_url).render(Context(settings.TEMPLATE_VARS))
 
+    @cached_property
+    def ics_content(self):
+        return self._check_ics_content()
+
     def _check_ics_content(self):
         if self.ics_url:
             ics_url = self.render_ics_url()
@@ -2726,7 +2711,9 @@ class TimePeriodExceptionSource(models.Model):
                 pass
             data = response.text
         else:
-            data = force_str(self.ics_file.read())
+            # open file and/or reposition file offset before reading
+            with self.ics_file.open() as fd:
+                data = force_str(fd.read())
 
         try:
             parsed = vobject.readOne(data)
@@ -2750,26 +2737,17 @@ class TimePeriodExceptionSource(models.Model):
             return force_str(vevent.contents['summary'][0].value)
         return _('Exception')
 
-    def refresh_timeperiod_exceptions(self, data=None):
-        if 'uwsgi' in sys.modules:
-            from chrono.utils.spooler import refresh_exception_source
+    def refresh_timeperiod_exceptions_in_spooler(self):
+        transaction.on_commit(lambda: spooler.run(self.refresh_timeperiod_exceptions_from_ics_ignore_errors))
 
-            tenant = getattr(connection, 'tenant', None)
-            transaction.on_commit(
-                lambda: refresh_exception_source.spool(
-                    source_id=str(self.pk), domain=getattr(tenant, 'domain_url', None)
-                )
-            )
-            return
-
-        self.refresh_timeperiod_exceptions_from_ics(data=data)
-
-    def refresh_timeperiod_exceptions_from_ics(self, data=None, recurring_days=600):
-        if data is None:
-            parsed = self._check_ics_content()
-        else:
-            parsed = data
+    def refresh_timeperiod_exceptions_from_ics_ignore_errors(self):
+        try:
+            self.refresh_timeperiod_exceptions_from_ics()
+        except ICSError:
+            pass
 
+    def refresh_timeperiod_exceptions_from_ics(self, recurring_days=600):
+        parsed = self.ics_content
         categories = collections.defaultdict(list)
         with transaction.atomic():
             # delete old exceptions related to this source

chrono/manager/views.py

@@ -3323,8 +3323,7 @@ class DeskImportTimePeriodExceptionsView(ManagedAgendaSubobjectMixin, UpdateView
             ics_file.seek(0)
         elif form.cleaned_data['ics_url']:
             source = desk.timeperiodexceptionsource_set.create(ics_url=form.cleaned_data['ics_url'])
-        parsed = source._check_ics_content()
-        source._parsed = parsed
+        source.ics_content  # pylint: disable=pointless-statement
         return source
 
     def form_valid(self, form):
@@ -3342,12 +3341,8 @@ class DeskImportTimePeriodExceptionsView(ManagedAgendaSubobjectMixin, UpdateView
             form.add_error(None, force_str(e))
             return self.form_invalid(form)
 
-        try:
-            for source in sources:
-                source.refresh_timeperiod_exceptions(data=source._parsed)
-        except ICSError as e:
-            form.add_error(None, force_str(e))
-            return self.form_invalid(form)
+        for source in sources:
+            source.refresh_timeperiod_exceptions_in_spooler()
 
         messages.info(self.request, _('Exceptions will be imported in a few minutes.'))
         return super().form_valid(form)
@@ -3396,7 +3391,7 @@ class TimePeriodExceptionSourceReplaceView(ManagedTimePeriodExceptionMixin, Upda
     def import_file(self, obj, form):
         source = obj.timeperiodexceptionsource_set.filter(ics_filename=self.get_object().ics_filename).first()
         if source is not None:
-            source.refresh_timeperiod_exceptions()
+            source.refresh_timeperiod_exceptions_in_spooler()
 
     def form_valid(self, form):
         try:
@@ -3427,7 +3422,7 @@ class TimePeriodExceptionSourceRefreshView(ManagedTimePeriodExceptionMixin, Deta
     def import_file(self, obj):
         source = obj.timeperiodexceptionsource_set.filter(ics_url=self.get_object().ics_url).first()
         if source is not None:
-            source.refresh_timeperiod_exceptions()
+            source.refresh_timeperiod_exceptions_in_spooler()
 
     def get(self, request, *args, **kwargs):
         try:
@@ -4004,8 +3999,7 @@ class UnavailabilityCalendarImportUnavailabilitiesView(ManagedUnavailabilityCale
             source = unavailabilty_calendar.timeperiodexceptionsource_set.create(
                 ics_url=form.cleaned_data['ics_url']
             )
-        parsed = source._check_ics_content()
-        source._parsed = parsed
+        source.ics_content  # pylint: disable=pointless-statement
         return source
 
     def form_valid(self, form):
@@ -4016,12 +4010,7 @@ class UnavailabilityCalendarImportUnavailabilitiesView(ManagedUnavailabilityCale
             form.add_error(None, force_str(e))
             return self.form_invalid(form)
 
-        try:
-            source.refresh_timeperiod_exceptions(data=source._parsed)
-        except ICSError as e:
-            form.add_error(None, force_str(e))
-            return self.form_invalid(form)
-
+        source.refresh_timeperiod_exceptions_in_spooler()
         messages.info(self.request, _('Exceptions will be imported in a few minutes.'))
         return super().form_valid(form)
 

chrono/settings.py

@@ -209,3 +209,5 @@ if os.path.exists(local_settings_file):
 # update EXCEPTIONS_SOURCES with modified (or not) WORKING_DAY_CALENDAR
 if EXCEPTIONS_SOURCES.get('holidays', {}) == {'class': 'workalendar.europe.France', 'label': N_('Holidays')}:
     EXCEPTIONS_SOURCES['holidays']['class'] = WORKING_DAY_CALENDAR
+
+USE_NEW_SPOOLER = True

chrono/utils/spooler.py

@@ -14,58 +14,9 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from django.db import connection
-from uwsgidecorators import spool  # pylint: disable=import-error
+try:
+    from hobo.multitenant.spooler import run  # pylint: disable=unused-import
+except ImportError:
 
-from chrono.agendas.models import Event, ICSError, TimePeriodExceptionSource
-
-
-def set_connection(domain):
-    from hobo.multitenant.middleware import TenantMiddleware  # pylint: disable=import-error
-
-    tenant = TenantMiddleware.get_tenant_by_hostname(domain)
-    connection.set_tenant(tenant)
-
-
-@spool
-def refresh_exception_source(args):
-    if args.get('domain'):
-        # multitenant installation
-        set_connection(args['domain'])
-
-    try:
-        source = TimePeriodExceptionSource.objects.get(pk=args['source_id'])
-    except TimePeriodExceptionSource.DoesNotExist:
-        return
-    try:
-        source.refresh_timeperiod_exceptions_from_ics()
-    except ICSError:
-        pass
-
-
-@spool
-def refresh_exceptions_from_settings(args):
-    if args.get('domain'):
-        # multitenant installation
-        set_connection(args['domain'])
-
-    try:
-        source = TimePeriodExceptionSource.objects.get(pk=args['source_id'])
-    except TimePeriodExceptionSource.DoesNotExist:
-        return
-
-    source.refresh_from_settings()
-
-
-@spool
-def event_notify_checked(args):
-    if args.get('domain'):
-        # multitenant installation
-        set_connection(args['domain'])
-
-    try:
-        event = Event.objects.get(pk=args['event_id'])
-    except Event.DoesNotExist:
-        return
-
-    event.notify_checked()
+    def run(func, *args, **kwargs):
+        func(*args, **kwargs)

chrono/utils/transaction.py

@@ -0,0 +1,19 @@
+# chrono - agendas system
+# Copyright (C) 2023 Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+# import on_commit() here to permit its monkeypatching in tests/conftest.py
+from django.db.transaction import atomic, on_commit  # pylint: disable=unused-import

debian/control

@@ -16,7 +16,6 @@ Depends: python3-django (>= 2:3.2),
          python3-gadjo,
          python3-publik-django-templatetags,
          python3-requests,
-         python3-uwsgidecorators,
          ${misc:Depends},
          ${python3:Depends},
 Recommends: python3-django-mellon

scripts/fixtures/admin.json

@@ -0,0 +1,20 @@
+[
+  {
+    "model": "auth.user",
+    "pk": 1,
+    "fields": {
+      "password": "pbkdf2_sha256$260000$O4TMYoLPoydHL1h2qE6Rcd$P9/OYa0mPxrgKtYufKWttmIFW+3SfeD8raupkhMbGwo=",
+      "last_login": "2023-04-08T01:21:28.681Z",
+      "is_superuser": true,
+      "username": "admin",
+      "first_name": "",
+      "last_name": "",
+      "email": "admin@example.com",
+      "is_staff": true,
+      "is_active": true,
+      "date_joined": "2023-04-08T01:21:23.751Z",
+      "groups": [],
+      "user_permissions": []
+    }
+  }
+]

scripts/runuwsgi_settings.py

@@ -0,0 +1,46 @@
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql',
+        'NAME': 'chrono-runserver',
+    }
+}
+
+LOGGING = {
+    'version': 1,
+    'formatters': {
+        'stream': {
+            'format': '%(asctime)s %(levelname)s %(name)s: %(message)s',
+            'datefmt': '%Y-%m-%d %a %H:%M:%S',
+        },
+    },
+    'handlers': {
+        'stream': {
+            'level': 'DEBUG',
+            'class': 'logging.FileHandler',
+            'formatter': 'stream',
+            # fd3 is redirected to stdout in scripts/uwsgi.sh
+            # to prevent logs to go to uwsgi log file.
+            'filename': '/dev/fd/3',
+        },
+    },
+    'loggers': {
+        # even when debugging seeing SQL queries is too much, activate it
+        # explicitly using DEBUG_DB
+        'django.db': {
+            'level': 'INFO',
+            'handlers': [],
+            'propagate': True,
+        },
+        'django': {
+            'level': 'INFO',
+            'handlers': [],
+            'propagate': True,
+        },
+        '': {
+            'handlers': ['stream'],
+            'level': 'INFO',
+        },
+    },
+}
+
+MIDDLEWARE = list(MIDDLEWARE) + ['testapp.MaDbgMiddleware']

scripts/uwsgi.sh

@@ -0,0 +1,59 @@
+#!/bin/sh
+
+set -e
+
+exec 3>&1
+
+function silent() {
+	LOG=`mktemp`
+	"$@" >$LOG 2>&1 || (tail $LOG; rm $LOG; exit 1)
+	rm $LOG
+}
+
+mkdir -p $TOX_WORK_DIR/spooler
+createdb chrono-runserver >/dev/null 2>&1 || true
+silent ./manage.py migrate
+silent ./manage.py loaddata scripts/fixtures/admin.json
+
+set -m
+
+UWSGI_INI=$TOX_WORK_DIR/uwsgi.ini
+cat >$UWSGI_INI <<EOF
+[uwsgi]
+plugin = python3
+single-interpreter = true
+pymodule-alias = testapp=scripts/testapp.py
+module = chrono.wsgi:application
+need-app = true
+plugin = logfile
+logger = file:$TOX_WORK_DIR/uwsgi.log
+req-logger = file:/dev/fd/3
+logformat-strftime = true
+log-date = %%Y-%%m-%%d %%a %%H:%%M:%%S
+log-format = %(ftime) INFO %(method) %(uri) (%(rsize) bytes, %(msecs) msecs, status %(status))
+virtualenv = $TOX_ENV_DIR
+auto-procname = true
+procname-prefix-spaced = chrono
+master = true
+enable-threads = true
+processes = 3
+pidfile = $TOX_WORK_DIR/uwsgi.pid
+http-socket = 127.0.0.1:9040
+spooler-python-import = chrono.utils.spooler
+spooler = $TOX_WORK_DIR/spooler
+python-autoreload = 1
+EOF
+
+echo
+
+silent uwsgi $UWSGI_INI &
+
+echo - uwsgi.ini: $UWSGI_INI
+echo - pidfile: $TOX_WORK_DIR/uwsgi.pid
+echo - uwsgi.log: $TOX_WORK_DIR/uwsgi.log
+echo - spooler: $TOX_WORK_DIR/spooler
+echo
+echo
+echo "     Open http://127.0.0.1:9040/ with login/password: admin / admin"
+
+fg >/dev/null

tests/conftest.py

@@ -23,3 +23,14 @@ def nocache(settings):
             'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
         }
     }
+
+
+@pytest.fixture(autouse=True)
+def mock_oncommit(monkeypatch):
+    '''Launch on_commit callbacks immediately, it won't work exactly as in
+    production but at least we will see some side effects.'''
+
+    def on_commit(func):
+        func()
+
+    monkeypatch.setattr('chrono.utils.transaction.on_commit', on_commit)

tox.ini

@@ -63,3 +63,23 @@ deps =
 commands =
   ./getlasso3.sh
   pylint: ./pylint.sh chrono/ tests/
+
+[testenv:runuwsgi]
+usedevelop = True
+passenv =
+  DISPLAY
+  XAUTHORITY
+setenv =
+  DJANGO_SETTINGS_MODULE=chrono.settings
+  CHRONO_SETTINGS_FILE=scripts/runuwsgi_settings.py
+deps =
+  django-filter>=2.4,<2.5
+  git+https://git.entrouvert.org/entrouvert/hobo.git@wip/76423-uwsgidecorators-corriger-le-pass
+  git+https://git.entrouvert.org/entrouvert/gadjo.git
+  git+https://git.entrouvert.org/entrouvert/publik-django-templatetags.git
+  psycopg2-binary
+  madbg
+allowlist_externals=
+  bash
+commands =
+  bash scripts/uwsgi.sh