translation update

- remove all code related to sending/parsing debug logs in
  /var/log/<app>/debug.
- add test on DebugLogFilter.
- configure JournalHandler to take debug logs if settings.DEBUG_LOG is True

MANIFEST.in

@@ -13,7 +13,7 @@ recursive-include hobo/maintenance/templates *.html *.txt
 recursive-include hobo/locale *.po *.mo
 recursive-include hobo/environment/locale *.po *.mo
 recursive-include hobo/agent/authentic2/locale *.po *.mo
-recursive-include tests *.py *.json
+recursive-include tests *.py *.json *.jpeg
 include hobo/multitenant/README
 include MANIFEST.in
 include COPYING

debian/debian_config_common.py

@@ -45,11 +45,6 @@ DISABLE_CRON_JOBS = False
 # mode for newly updated files
 FILE_UPLOAD_PERMISSIONS = 0o644
 
-DEBUG_LOG_PATH = '/var/log/%s/debug' % PROJECT_NAME
-DEBUG_LOG_FORMAT = (
-    '%(asctime)s \x1f%(tenant)s \x1f%(ip)s \x1f%(user)r \x1f%(request_id)s \x1f'
-    '%(levelname)s \x1f%(name)s \x1f%(message)s'
-)
 DEBUG_PROVISIONNING_LOG_PATH = '/var/log/%s/provisionning-debug' % PROJECT_NAME
 
 DISABLE_GLOBAL_HANDLERS = os.environ.get('DISABLE_GLOBAL_HANDLERS') == '1'
@@ -84,9 +79,6 @@ LOGGING = {
             'format': '%(application)s %(levelname)s %(tenant)s %(ip)s %(user)s %(request_id)s'
             ' %(message)s',
         },
-        'debug': {
-            'format': DEBUG_LOG_FORMAT,
-        },
         'syslog_no_filter': {
             'format': '%(levelname)s %(message)s',
         },
@@ -113,16 +105,6 @@ LOGGING = {
         'null': {
             'class': 'logging.NullHandler',
         },
-        'debug': {
-            'level': 'DEBUG',
-            'class': 'hobo.logger.TimedRotatingFileHandler',
-            'formatter': 'debug',
-            'filename': DEBUG_LOG_PATH,
-            'when': 'midnight',
-            'backupCount': 1,
-            'interval': 1,
-            'filters': ['request_context', 'debug_log'],
-        },
     },
     'loggers': {
         'django.db': {
@@ -186,23 +168,27 @@ LOGGING = {
         },
         '': {
             'level': 'DEBUG',
-            'filters': ['request_context'],
-            'handlers': ([] if DISABLE_GLOBAL_HANDLERS else ['syslog']) + ['mail_admins', 'debug'],
+            'filters': ['request_context', 'debug_log'],
+            'handlers': ([] if DISABLE_GLOBAL_HANDLERS else ['syslog']) + ['mail_admins'],
         },
     },
 }
 
 # Journald support
 if os.path.exists('/run/systemd/journal/socket') and not DISABLE_GLOBAL_HANDLERS:
+    systemd = None
     try:
-        from systemd import journal
+        import cysystemd as systemd
     except ImportError:
-        pass
-    else:
+        try:
+            import systemd
+        except ImportError:
+            pass
+    if systemd:
         LOGGING['handlers']['journald'] = {
-            'level': 'INFO',
+            'level': 'DEBUG',
             'class': 'hobo.journal.JournalHandler',
-            'filters': ['request_context'],
+            'filters': ['request_context', 'debug_log'],
             'formatter': 'syslog',
         }
         LOGGING['loggers']['']['handlers'].remove('syslog')

hobo/agent/common/__init__.py

@@ -16,11 +16,11 @@ def notify_agents(data):
     }
     with Celery('hobo', broker=settings.BROKER_URL) as app:
         app.conf.update(
-            CELERY_TASK_SERIALIZER='json',
-            CELERY_ACCEPT_CONTENT=['json'],
-            CELERY_RESULT_SERIALIZER='json',
-            CELERY_QUEUES=(Broadcast('broadcast_tasks'),),
-            CELERY_EVENT_QUEUE_TTL=10,
+            task_serializer='json',
+            accept_content=['json'],
+            result_serializer='json',
+            task_queues=(Broadcast('broadcast_tasks'),),
+            event_queue_ttl=10,
         )
         # see called method in hobo.agent.worker.celery
         app.send_task(

hobo/agent/worker/celery.py

@@ -22,10 +22,10 @@ from . import services, settings
 
 app = Celery('hobo', broker=settings.BROKER_URL)
 app.conf.update(
-    CELERY_TASK_SERIALIZER='json',
-    CELERY_ACCEPT_CONTENT=['json'],
-    CELERY_RESULT_SERIALIZER='json',
-    CELERY_QUEUES=(Broadcast('broadcast_tasks'),),
+    task_serializer='json',
+    accept_content=['json'],
+    result_serializer='json',
+    task_queues=(Broadcast('broadcast_tasks'),),
 )
 
 if hasattr(settings, 'CELERY_SETTINGS'):

hobo/applications/forms.py

@@ -15,15 +15,62 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from django import forms
+from django.utils.timezone import now
 from django.utils.translation import gettext_lazy as _
 
 from hobo.applications.models import Application, Version
 
 
 class GenerateForm(forms.Form):
-    number = forms.CharField(label=_('Version Number'), max_length=100)
+    number = forms.RegexField(
+        label=_('Version Number'),
+        max_length=100,
+        regex=r'^\d+\.\d+$',
+        help_text=_('The version number consists of two numbers separated by a dot. Example: 1.0'),
+    )
     notes = forms.CharField(label=_('Version notes'), widget=forms.Textarea, required=False)
 
+    def __init__(self, *args, **kwargs):
+        self.latest_version = kwargs.pop('latest_version')
+        super().__init__(*args, **kwargs)
+        if self.latest_version:
+            try:
+                old_version = [int(n) for n in self.latest_version.number.split('.')]
+            except ValueError:
+                old_version = None
+            if old_version:
+                self.initial['number'] = '.'.join(str(n) for n in old_version[:2])
+            self.initial['notes'] = self.latest_version.notes
+
+    def clean_number(self):
+        number = self.cleaned_data['number']
+        if not self.latest_version:
+            return number
+        try:
+            old_number = [int(n) for n in self.latest_version.number.split('.')]
+        except ValueError:
+            return number
+        new_number = [int(n) for n in self.cleaned_data['number'].split('.')]
+        if old_number[:2] > new_number:
+            raise forms.ValidationError(
+                _('The version number must be equal to or greater than the previous one.')
+            )
+        return number
+
+    def get_cleaned_number(self):
+        number = [int(n) for n in self.cleaned_data['number'].split('.')]
+        number.append(int(now().strftime('%Y%m%d')))
+        if not self.latest_version:
+            return '%s.%s.%s.0' % tuple(number)
+        try:
+            old_number = [int(n) for n in self.latest_version.number.split('.')]
+        except ValueError:
+            return '%s.%s.%s.0' % tuple(number)
+        if number != old_number[:3]:
+            return '%s.%s.%s.0' % tuple(number)
+        last_part = old_number[3]
+        return '%s.%s.%s.%s' % (*number, last_part + 1)
+
 
 class InstallForm(forms.Form):
     bundle = forms.FileField(label=_('Application'))

hobo/applications/models.py

@@ -132,6 +132,8 @@ class Application(models.Model):
             modules.append('combo')
         if settings.HOBO_APPLICATION_CHRONO_SUPPORT:
             modules.append('chrono')
+        if settings.HOBO_APPLICATION_LINGO_SUPPORT:
+            modules.append('lingo')
         return modules
 
     def save(self, *args, **kwargs):
@@ -417,7 +419,7 @@ class Version(models.Model):
                 if service_objects[service['url']].secondary:
                     continue
                 url = urllib.parse.urljoin(service['url'], target_url)
-                response = requests.put(url, data=bundle_content)
+                response = requests.post(url, files=[('bundle', bundle_content)])
                 if not response.ok:
                     raise DeploymentError(exception_message % (service_id, response.status_code))
                 if not job:
@@ -456,7 +458,7 @@ class Version(models.Model):
                 if service_objects[service['url']].secondary:
                     continue
                 url = urllib.parse.urljoin(service['url'], target_url)
-                response = requests.put(url, data=bundle_content)
+                response = requests.post(url, files=[('bundle', bundle_content)])
                 if not response.ok:
                     raise DeploymentError(exception_message % (service_id, response.status_code))
                 if not job:

hobo/applications/templates/hobo/applications/manifest.html

@@ -31,14 +31,68 @@
 
 {% block content %}
 
+  <div>
+    <label>
+      {% trans "Filter label:" %}
+      <input type="search" id="name-filter">
+    </label>
+    <label>
+      {% trans "Filter type:" %}
+      <select id="type-filter">
+        <option value="">----------</option>
+        {% for component_type in component_types %}
+          {% ifchanged component_type.service.title %}
+            <optgroup label="{{ component_type.service.title }}">
+          {% endifchanged %}
+          <option value="{{ component_type.id }}">{{ component_type.text }}</option>
+        {% endfor %}
+      </select>
+    </label>
+  </div>
+  <script>
+    $('#name-filter, #type-filter').on('change blur keyup', function() {
+      const name_val = $('#name-filter').val().toLowerCase();
+      const type_val = $('#type-filter').val();
+      if (!name_val && !type_val) {
+        $('.application-content li').show();
+      } else {
+        $('.application-content li').each(function(idx, elem) {
+          var slugged_text = $(elem).attr('data-slugged-text');
+          var type = $(elem).attr('data-type');
+          if (name_val && type_val) {
+            if (slugged_text.indexOf(name_val) > -1 && type == type_val) {
+              $(elem).show();
+            } else {
+              $(elem).hide();
+            }
+          } else if (name_val) {
+            if (slugged_text.indexOf(name_val) > -1) {
+              $(elem).show();
+            } else {
+              $(elem).hide();
+            }
+          } else if (type_val) {
+            if (type == type_val) {
+              $(elem).show();
+            } else {
+              $(elem).hide();
+            }
+          } else {
+            $(elem).hide();
+          }
+        });
+      }
+    });
+  </script>
+
   <ul class="objects-list single-links application-content">
     {% for relation in relations %}
-      <li {% if relation.auto_dependency %}class="auto-dependency"{% endif %}>
+      <li data-slugged-text="{{ relation.element.name|slugify }}" data-type="{{ relation.element.type }}" {% if relation.auto_dependency %}class="auto-dependency"{% endif %}>
         <a {% if relation.element.get_redirect_url %}href="{{ relation.element.get_redirect_url }}"{% endif %}>
           {% if relation.error %}<span class="tag tag-error">{{ relation.get_error_status_display }}</span>{% endif %}
           {{ relation.element.name }} <span class="extra-info">- {{ relation.element.type_label }}</span>
         </a>
-        {% if app.editable and not relation.auto_dependency %}<a rel="popup" class="delete" href="{% url 'application-delete-element' app_slug=app.slug pk=relation.id %}">{% trans "remove" %}</a>{% endif %}
+        {% if app.editable and not relation.auto_dependency %}<a rel="popup" class="delete" href="{% url 'application-delete-element' app_slug=app.slug pk=relation.id %}"><span class="sr-only">{% trans "remove" %}</span></a>{% endif %}
       </li>
     {% endfor %}
   </ul>
@@ -96,7 +150,7 @@
             <ul class="objects-list single-links app-parameters--list">
               {% for parameter in parameters_qs %}
                 <li><a rel="popup" href="{% url 'application-edit-parameter' app_slug=app.slug pk=parameter.id %}">{{ parameter.label }}</a>
-                  <a rel="popup" class="delete" href="{% url 'application-delete-parameter' app_slug=app.slug pk=parameter.id %}">{% trans "remove" %}</a>
+                  <a rel="popup" class="delete" href="{% url 'application-delete-parameter' app_slug=app.slug pk=parameter.id %}"><span class="sr-only">{% trans "remove" %}</span></a>
                 </li>
               {% endfor %}
             </ul>

hobo/applications/templates/hobo/applications/version_compare.html

@@ -27,7 +27,7 @@
               {% if not element.real_element %}<span class="tag tag-error">{% trans "Element not found" %}</span> {% endif %}{{ element.name }} <span class="extra-info">- {{ element.type_label }}</span>
               {% if element.real_element.get_redirect_url %}
                 <a class="link-action-icon link" href="{{ element.real_element.get_redirect_url }}?application={{ app.slug }}&version1={{ version1.number }}&version2={{ version2.number }}&compare">
-                  {% trans "see" %}
+                  <span class="sr-only">{% trans "see" %}</span>
                 </a>
               {% endif %}
             </a>

hobo/applications/utils.py

@@ -61,5 +61,7 @@ class Requests(RequestsSession):
 
         query = urlencode(query_params, doseq=True)
         url = urllib.parse.urlunparse((scheme, netloc, path, params, query, fragment))
+        headers = kwargs.pop('headers', {})
+        headers['Accept-Language'] = f'{settings.LANGUAGE_CODE}, {settings.LANGUAGE_CODE[:2]}, en'
 
-        return super().request(method, url, **kwargs)
+        return super().request(method, url, headers=headers, **kwargs)

hobo/applications/views.py

@@ -32,6 +32,7 @@ from django.utils.timezone import localtime, now
 from django.utils.translation import gettext_lazy as _
 from django.views.generic import DetailView, FormView, ListView, RedirectView, TemplateView
 from django.views.generic.edit import CreateView, DeleteView, UpdateView
+from PIL import Image, UnidentifiedImageError
 
 from hobo.environment.models import Variable
 
@@ -91,6 +92,7 @@ class ManifestView(TemplateView):
 
         type_labels = {}
         object_types = get_object_types()
+        context['component_types'] = object_types
         types = [o['id'] for o in object_types]
         for object_type in object_types:
             type_labels[object_type['id']] = object_type['singular']
@@ -390,23 +392,17 @@ class GenerateView(FormView):
     form_class = GenerateForm
     template_name = 'hobo/applications/generate.html'
 
-    def get_initial(self):
+    def get_form_kwargs(self):
+        kwargs = super().get_form_kwargs()
         self.app = get_object_or_404(Application, slug=self.kwargs['app_slug'], editable=True)
-        version = self.app.version_set.order_by('last_update_timestamp').last()
-        if version:
-            self.initial['number'] = version.number
-            self.initial['notes'] = version.notes
-        return super().get_initial()
+        kwargs['latest_version'] = self.app.version_set.order_by('last_update_timestamp').last()
+        return kwargs
 
     def form_valid(self, form):
         app = self.app
 
-        latest_version = app.version_set.order_by('last_update_timestamp').last()
-        if latest_version and latest_version.number == form.cleaned_data['number']:
-            version = latest_version
-        else:
-            version = Version(application=app)
-        version.number = form.cleaned_data['number']
+        version = Version(application=app)
+        version.number = form.get_cleaned_number()
         version.notes = form.cleaned_data['notes']
         version.save()
 
@@ -456,13 +452,20 @@ class Install(FormView):
         tar_io = io.BytesIO(self.request.FILES['bundle'].read())
         try:
             with tarfile.open(fileobj=tar_io) as tar:
-                manifest = json.loads(tar.extractfile('manifest.json').read().decode())
+                try:
+                    manifest = json.loads(tar.extractfile('manifest.json').read().decode())
+                except KeyError:
+                    form.add_error('bundle', _('Invalid tar file, missing manifest.'))
+                    return self.form_invalid(form)
                 if self.application and self.application.slug != manifest.get('slug'):
                     form.add_error(
                         'bundle',
                         _('Can not update this application, wrong slug (%s).') % manifest.get('slug'),
                     )
                     return self.form_invalid(form)
+                icon = manifest.get('icon')
+                if icon:
+                    Image.open(tar.extractfile(icon))
                 app, created = Application.objects.get_or_create(
                     slug=manifest.get('slug'), defaults={'name': manifest.get('application')}
                 )
@@ -475,7 +478,6 @@ class Install(FormView):
                     # overwriting a local application and keep on developing it.
                     app.editable = False
                 app.save()
-                icon = manifest.get('icon')
                 if icon:
                     app.icon.save(icon, tar.extractfile(icon), save=True)
                 else:
@@ -483,6 +485,9 @@ class Install(FormView):
         except tarfile.TarError:
             form.add_error('bundle', _('Invalid tar file.'))
             return self.form_invalid(form)
+        except UnidentifiedImageError:
+            form.add_error('bundle', _('Invalid icon file.'))
+            return self.form_invalid(form)
 
         # always create a new version on install or if previous version has not the same number
         version_number = manifest.get('version_number') or 'unknown'

hobo/deploy/signals.py

@@ -56,10 +56,10 @@ def notify_agents(sender, **kwargs):
     tls.MUST_NOTIFY = False
     with Celery('hobo', broker=settings.BROKER_URL) as app:
         app.conf.update(
-            CELERY_TASK_SERIALIZER='json',
-            CELERY_ACCEPT_CONTENT=['json'],
-            CELERY_RESULT_SERIALIZER='json',
-            CELERY_QUEUES=(Broadcast('broadcast_tasks'),),
+            task_serializer='json',
+            accept_content=['json'],
+            result_serializer='json',
+            task_queues=(Broadcast('broadcast_tasks'),),
         )
         # see called method in hobo.agent.worker.celery
         app.send_task(

hobo/environment/admin.py

@@ -1,2 +0,0 @@
-# noqa pylint: disable=unused-import
-from django.contrib import admin

hobo/journal.py

@@ -22,8 +22,11 @@ import sys as _sys
 import traceback as _traceback
 from syslog import LOG_ALERT, LOG_CRIT, LOG_DEBUG, LOG_ERR, LOG_INFO, LOG_WARNING
 
-# noqa pylint: disable=import-error
-from systemd._journal import sendv
+try:
+    # more recent pythong binding is named cysystemd
+    from cysystemd import journal  # noqa pylint: disable=import-error
+except ImportError:
+    from systemd import journal  # noqa pylint: disable=import-error
 
 _IDENT_CHARACTER = set('ABCDEFGHIJKLMNOPQRTSUVWXYZ_0123456789')
 
@@ -32,61 +35,6 @@ def _valid_field_name(s):
     return not (set(s) - _IDENT_CHARACTER)
 
 
-def _make_line(field, value):
-    if isinstance(value, bytes):
-        return field.encode('utf-8') + b'=' + value
-    elif isinstance(value, str):
-        return field + '=' + value
-    else:
-        return field + '=' + str(value)
-
-
-def send(MESSAGE, MESSAGE_ID=None, CODE_FILE=None, CODE_LINE=None, CODE_FUNC=None, **kwargs):
-    r"""Send a message to the journal.
-
-    >>> from systemd import journal
-    >>> journal.send('Hello world')
-    >>> journal.send('Hello, again, world', FIELD2='Greetings!')
-    >>> journal.send('Binary message', BINARY=b'\xde\xad\xbe\xef')
-
-    Value of the MESSAGE argument will be used for the MESSAGE= field. MESSAGE
-    must be a string and will be sent as UTF-8 to the journal.
-
-    MESSAGE_ID can be given to uniquely identify the type of message. It must be
-    a string or a uuid.UUID object.
-
-    CODE_LINE, CODE_FILE, and CODE_FUNC can be specified to identify the caller.
-    Unless at least on of the three is given, values are extracted from the
-    stack frame of the caller of send(). CODE_FILE and CODE_FUNC must be
-    strings, CODE_LINE must be an integer.
-
-    Additional fields for the journal entry can only be specified as keyword
-    arguments. The payload can be either a string or bytes. A string will be
-    sent as UTF-8, and bytes will be sent as-is to the journal.
-
-    Other useful fields include PRIORITY, SYSLOG_FACILITY, SYSLOG_IDENTIFIER,
-    SYSLOG_PID.
-    """
-
-    args = ['MESSAGE=' + MESSAGE]
-
-    if MESSAGE_ID is not None:
-        id = getattr(MESSAGE_ID, 'hex', MESSAGE_ID)
-        args.append('MESSAGE_ID=' + id)
-
-    if CODE_LINE is CODE_FILE is CODE_FUNC is None:
-        CODE_FILE, CODE_LINE, CODE_FUNC = _traceback.extract_stack(limit=2)[0][:3]
-    if CODE_FILE is not None:
-        args.append('CODE_FILE=' + CODE_FILE)
-    if CODE_LINE is not None:
-        args.append(f'CODE_LINE={CODE_LINE:d}')
-    if CODE_FUNC is not None:
-        args.append('CODE_FUNC=' + CODE_FUNC)
-
-    args.extend(_make_line(key, val) for key, val in kwargs.items())
-    return sendv(*args)
-
-
 class JournalHandler(_logging.Handler):
     """Journal handler class for the Python logging framework.
 
@@ -139,7 +87,7 @@ class JournalHandler(_logging.Handler):
     the `sender_function` parameter.
     """
 
-    def __init__(self, level=_logging.NOTSET, sender_function=send, **kwargs):
+    def __init__(self, level=_logging.NOTSET, **kwargs):
         super().__init__(level)
 
         for name in kwargs:
@@ -148,7 +96,6 @@ class JournalHandler(_logging.Handler):
         if 'SYSLOG_IDENTIFIER' not in kwargs:
             kwargs['SYSLOG_IDENTIFIER'] = _sys.argv[0]
 
-        self.send = sender_function
         self._extra = kwargs
 
     def emit(self, record):
@@ -190,10 +137,13 @@ class JournalHandler(_logging.Handler):
                     continue
                 if key in ['threadName', 'processName', 'pathname', 'lineno', 'funcName']:
                     continue
+                # prevent logging of WSGIRequest object which attached to the log record
+                if not isinstance(value, (str, int)):
+                    continue
                 extras[key.upper()] = value
 
-            self.send(
-                msg,
+            journal.send(
+                MESSAGE=msg,
                 PRIORITY=format(pri),
                 LOGGER=record.name,
                 THREAD_NAME=record.threadName,

hobo/locale/fr/LC_MESSAGES/django.po

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: hobo 0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2023-12-15 10:40+0100\n"
+"POT-Creation-Date: 2024-04-30 09:23+0200\n"
 "PO-Revision-Date: 2023-12-15 10:56+0100\n"
 "Last-Translator: Frederic Peters <fpeters@entrouvert.com>\n"
 "Language: French\n"
@@ -20,10 +20,21 @@ msgstr ""
 msgid "Version Number"
 msgstr "Numéro de version"
 
+#: hobo/applications/forms.py
+msgid ""
+"The version number consists of two numbers separated by a dot. Example: 1.0"
+msgstr ""
+"Le numéro de version est formé de deux nombres séparés par un point. "
+"Exemple: 1.0"
+
 #: hobo/applications/forms.py
 msgid "Version notes"
 msgstr "Notes de version"
 
+#: hobo/applications/forms.py
+msgid "The version number must be equal to or greater than the previous one."
+msgstr "Le numéro de version doit être supérieur ou égal au précédent."
+
 #: hobo/applications/forms.py
 msgid "Application"
 msgstr "Application"
@@ -429,6 +440,14 @@ msgstr "Ces composants seront réinstallés si vous lancez l’installation."
 msgid "See all versions"
 msgstr "Voir toutes les versions"
 
+#: hobo/applications/templates/hobo/applications/manifest.html
+msgid "Filter label:"
+msgstr "Filtrer le libellé :"
+
+#: hobo/applications/templates/hobo/applications/manifest.html
+msgid "Filter type:"
+msgstr "Filtrer le type :"
+
 #: hobo/applications/templates/hobo/applications/manifest.html
 msgid "remove"
 msgstr "supprimer"
@@ -579,6 +598,10 @@ msgstr "Analyse des dépendances"
 msgid "Creating application bundle"
 msgstr "Création de l’application"
 
+#: hobo/applications/views.py
+msgid "Invalid tar file, missing manifest."
+msgstr "Fichier tar invalide, manifest manquant."
+
 #: hobo/applications/views.py
 #, python-format
 msgid "Can not update this application, wrong slug (%s)."
@@ -590,6 +613,10 @@ msgstr ""
 msgid "Invalid tar file."
 msgstr "Fichier tar invalide."
 
+#: hobo/applications/views.py
+msgid "Invalid icon file."
+msgstr "Fichier icon invalide."
+
 #: hobo/applications/views.py
 msgid "Check installation"
 msgstr "Vérification de l’installation"

hobo/logger.py

@@ -14,15 +14,10 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import ast
-import datetime
 import logging
 import logging.handlers
-import os
-import time
 import warnings
 
-import pytz
 from django.conf import settings
 from django.db import connection
 
@@ -124,33 +119,17 @@ class ForceDebugFilter(logging.Filter):
         return super().filter(record)
 
 
-class LogRecord(logging.LogRecord):
-    '''Subclass LogRecord to make multiline log parseable'''
-
-    def getMessage(self):
-        return super().getMessage().replace('\n', '\n ')
-
-
-class TimedRotatingFileHandler(logging.handlers.TimedRotatingFileHandler):
-    def format(self, record):
-        old_class = record.__class__
-        record.__class__ = LogRecord
-        try:
-            return super().format(record)
-        finally:
-            record.__class__ = old_class
-
-
 class DebugLogFilter:
     '''Filter debug log records based on the DEBUG_LOG setting'''
 
     def filter(self, record):
         debug_log = getattr(settings, 'DEBUG_LOG', False)
+        if record.levelno > logging.DEBUG:
+            return True
+
         if not debug_log:
             return False
 
-        # change class to add space after newlines in message
-        record.__class__ = LogRecord
         if debug_log is True:
             return True
         elif hasattr(debug_log, 'encode'):
@@ -164,74 +143,6 @@ class DebugLogFilter:
             return bool(debug_log)
 
 
-class DebugLog:
-    def __init__(self, path):
-        self.path = path
-
-    def _pre_lines(self, cursor=0):
-        if not os.path.exists(self.path):
-            return
-        with open(self.path, 'rb') as fd:
-            accum = ''
-
-            try:
-                fd.seek(cursor)
-            except Exception:
-                return
-
-            for line in fd:
-                size = len(line)
-                cursor += size
-                line = line.decode('utf-8')
-
-                if not accum:
-                    accum = line
-                elif not line.startswith(' '):
-                    yield cursor - size, accum
-                    accum = line
-                else:
-                    accum += line[1:]
-            if accum:
-                yield cursor, accum
-
-    keys = ['tenant', 'ip', 'user', 'request_id', 'level', 'logger']
-
-    def _parse(self, cursor=0):
-        for cursor, line in self._pre_lines(cursor=cursor):
-            if line.endswith('\n'):
-                line = line[:-1]
-            parts = line.split(' \x1f', settings.DEBUG_LOG_FORMAT.count(' \x1f'))
-            try:
-                timestamp = datetime.datetime.strptime(parts[0], '%Y-%m-%d %H:%M:%S,%f')
-                timestamp = pytz.timezone(time.tzname[0]).localize(timestamp)
-            except ValueError:
-                continue
-            message = parts[-1]
-            d = {key: value for key, value in zip(self.keys, parts[1:-1])}
-            if 'user' in d:
-                try:
-                    d['user'] = ast.literal_eval(d['user'])
-                except SyntaxError:
-                    pass
-            d.update(
-                {
-                    'cursor': cursor,
-                    'timestamp': timestamp,
-                    'message': message,
-                }
-            )
-            yield d
-
-    @classmethod
-    def lines(cls, cursor=0):
-        debug_log_path = getattr(settings, 'DEBUG_LOG_PATH', None)
-        if not debug_log_path:
-            return
-        if not os.path.exists(debug_log_path):
-            return
-        yield from cls(debug_log_path)._parse(cursor=cursor)
-
-
 class ClampLogLevel(logging.Filter):
     def __init__(self, level):
         self.levelname = level.upper()

hobo/multitenant/settings_loaders.py

@@ -152,6 +152,10 @@ class TemplateVars(FileBaseSettingsLoader):
                 variables['idp_account_url'] = service.get('base_url') + 'accounts/'
                 variables['idp_registration_url'] = service.get('base_url') + 'register/'
 
+            if service.get('service-id') == 'lingo' and not service.get('secondary'):
+                # always add a trailing /
+                variables['lingo_url'] = urllib.parse.urljoin(service.get('base_url'), '/')
+
             if not service.get('this'):
                 continue
 

hobo/multitenant/utils.py

@@ -13,9 +13,7 @@ def provision_user_groups(user, uuids):
     logger = logging.getLogger(__name__)
 
     existing_pks = user.groups.values_list('pk', flat=True)
-    not_found = set(uuids)
     for role in Role.objects.filter(uuid__in=uuids).exclude(pk__in=existing_pks):
-        not_found.discard(role.uuid)
         if role.pk in existing_pks:
             continue
         user.groups.through.objects.get_or_create(group=role, user=user)
@@ -31,5 +29,6 @@ def provision_user_groups(user, uuids):
             pass
         else:
             logger.info('removed role %s from %s (%s)', rel.group, user, user.pk)
-    for uuid in not_found:
+    existing_uuids = set(Role.objects.values_list('uuid', flat=True))
+    for uuid in set(uuids).difference(existing_uuids):
         logger.warning('role %s of user %s does not exist', uuid, user)

hobo/settings.py

@@ -37,7 +37,6 @@ EMAIL_FROM_ALLOWED_DOMAINS = []
 # Application definition
 
 INSTALLED_APPS = (
-    'django.contrib.admin',
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'django.contrib.sessions',
@@ -250,6 +249,9 @@ HOBO_APPLICATION_COMBO_SUPPORT = False
 # Support for chrono elements in applications
 HOBO_APPLICATION_CHRONO_SUPPORT = False
 
+# Support for lingo elements in applications
+HOBO_APPLICATION_LINGO_SUPPORT = False
+
 # Phone prefixes by country for phone number as authentication identifier
 PHONE_COUNTRY_CODES = {
     '32': {'region': 'BE', 'region_desc': _('Belgium')},

hobo/urls.py

@@ -15,7 +15,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from django.conf import settings
-from django.contrib import admin
 from django.urls import include, path, re_path
 
 from .applications.urls import urlpatterns as applications_urls
@@ -31,8 +30,6 @@ from .theme.urls import urlpatterns as theme_urls
 from .urls_utils import decorated_includes
 from .views import admin_required, health_json, hobo, home, login, login_local, logout, menu_json
 
-admin.autodiscover()
-
 urlpatterns = [
     path('', home, name='home'),
     re_path(r'^sites/', decorated_includes(admin_required, include(environment_urls))),
@@ -48,7 +45,6 @@ urlpatterns = [
     path('api/health/', health_json, name='health-json'),
     re_path(r'^menu.json$', menu_json, name='menu_json'),
     re_path(r'^hobos.json$', hobo),
-    re_path(r'^admin/', admin.site.urls),
 ]
 
 # add authentication patterns

tests/test_application.py

@@ -1,7 +1,8 @@
-import base64
 import copy
+import datetime
 import io
 import json
+import os
 import random
 import re
 import tarfile
@@ -9,6 +10,7 @@ import tarfile
 import httmock
 import pytest
 from django.contrib.contenttypes.models import ContentType
+from django.utils.timezone import now
 from httmock import HTTMock
 from pyquery import PyQuery
 from webtest import Upload
@@ -29,6 +31,8 @@ from .test_manager import login
 
 pytestmark = pytest.mark.django_db
 
+TESTS_DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
+
 
 WCS_AVAILABLE_OBJECTS = {
     'data': [
@@ -216,7 +220,7 @@ def mocked_http(url, request):
 
     if url.path == '/api/export-import/bundle-import/':
         # alter WCS_AVAILABLE_FORMS response with newly installed forms
-        with tarfile.open(mode='r', fileobj=io.BytesIO(request.body)) as tar:
+        with tarfile.open(mode='r', fileobj=io.BytesIO(request.original.files[0][1])) as tar:
             manifest_json = json.load(tar.extractfile('manifest.json'))
         for element in manifest_json.get('elements'):
             if element['type'] not in [x['id'] for x in WCS_AVAILABLE_OBJECTS['data']]:
@@ -323,7 +327,7 @@ def test_create_application(app, admin_user, settings, analyze):
         resp = resp.form.submit().follow()
         assert 'Test Card' in resp.text
         version = Version.objects.latest('pk')
-        assert version.number == '1.0'
+        assert version.number == '1.0.%s.0' % now().strftime('%Y%m%d')
         assert version.notes == 'Foo bar blah.'
         resp = resp.click('Download')
         assert resp.content_type == 'application/x-tar'
@@ -332,59 +336,31 @@ def test_create_application(app, admin_user, settings, analyze):
         assert b'<carddef/>' in resp.content
         assert b'"icon": null' in resp.content
         assert b'"documentation_url": "http://foo.bar"' in resp.content
-        assert b'"version_number": "1.0"' in resp.content
+        assert b'"version_number": "1.0.%s.0"' % now().strftime('%Y%m%d').encode() in resp.content
         assert b'"version_notes": "Foo bar blah."' in resp.content
         assert b'"visible": false' in resp.content
 
         resp = app.get('/applications/manifest/test/versions/')
         versions = [e.text() for e in resp.pyquery('h3').items()]
-        assert versions.count('1.0') == 1
+        assert versions.count('1.0.%s.0' % now().strftime('%Y%m%d')) == 1
         assert resp.text.count('Creating application bundle') == 1
         assert 'Compare' not in resp
         resp = resp.click(href='/applications/manifest/test/download/%s/' % version.pk)
         assert resp.content_type == 'application/x-tar'
-        assert resp.headers['Content-Disposition'] == 'attachment; filename="test-1.0.tar"'
-        assert b'"version_number": "1.0"' in resp.content
-
-        # generate again without changing version number
-        resp = app.get('/applications/manifest/test/generate/')
-        assert resp.form['number'].value == '1.0'  # last one
-        assert resp.form['notes'].value == 'Foo bar blah.'  # last one
-        resp.form['notes'] = 'Foo bar blahha.'
-        resp = resp.form.submit().follow()
-        same_version = Version.objects.latest('pk')
-        assert same_version.number == '1.0'
-        assert same_version.notes == 'Foo bar blahha.'
-        assert same_version.pk == version.pk
-        resp = resp.click('Download')
-        assert resp.content_type == 'application/x-tar'
-        assert b'"version_number": "1.0"' in resp.content
-        assert b'"version_notes": "Foo bar blahha."' in resp.content
-
-        resp = app.get('/applications/manifest/test/versions/')
-        versions = [e.text() for e in resp.pyquery('h3').items()]
-        assert versions.count('1.0') == 1
-        assert resp.text.count('Creating application bundle') == 2
-        assert 'Compare' not in resp
-        resp = resp.click(href='/applications/manifest/test/download/%s/' % same_version.pk)
-        assert resp.content_type == 'application/x-tar'
-        assert resp.headers['Content-Disposition'] == 'attachment; filename="test-1.0.tar"'
-        assert b'"version_number": "1.0"' in resp.content
+        assert resp.headers[
+            'Content-Disposition'
+        ] == 'attachment; filename="test-1.0.%s.0.tar"' % now().strftime('%Y%m%d')
+        assert b'"version_number": "1.0.%s.0"' % now().strftime('%Y%m%d').encode() in resp.content
 
         # add an icon
         resp = app.get('/applications/manifest/test/metadata/')
-        resp.form['icon'] = Upload(
-            'foo.png',
-            base64.decodebytes(
-                b'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQAAAAA3bvkkAAAACklEQVQI12NoAAAAggCB3UNq9AAAAABJRU5ErkJggg=='
-            ),
-            'image/png',
-        )
+        with open(os.path.join(TESTS_DATA_DIR, 'black.jpeg'), mode='rb') as icon_fd:
+            resp.form['icon'] = Upload('foo.jpeg', icon_fd.read(), 'image/jpeg')
         resp.form['documentation_url'] = ''  # and reset documentation_url
         resp.form['visible'] = True
         resp = resp.form.submit().follow()
         application.refresh_from_db()
-        assert re.match(r'applications/icons/foo(_\w+)?.png', application.icon.name)
+        assert re.match(r'applications/icons/foo(_\w+)?.jpeg', application.icon.name)
         assert application.documentation_url == ''
         assert application.visible is True
 
@@ -394,7 +370,7 @@ def test_create_application(app, admin_user, settings, analyze):
         resp = resp.form.submit()
         assert 'The icon must be in JPEG or PNG format' in resp
         application.refresh_from_db()
-        assert re.match(r'applications/icons/foo(_\w+)?.png', application.icon.name)
+        assert re.match(r'applications/icons/foo(_\w+)?.jpeg', application.icon.name)
 
         resp = app.get('/applications/manifest/test/')
         resp = resp.click('Generate application bundle')
@@ -408,68 +384,88 @@ def test_create_application(app, admin_user, settings, analyze):
         assert b'<carddef/>' in resp.content
         assert b'"icon": "foo' in resp.content
         assert b'"documentation_url": ""' in resp.content
-        assert b'"version_number": "2.0"' in resp.content
+        assert b'"version_number": "2.0.%s.0"' % now().strftime('%Y%m%d').encode() in resp.content
         assert b'"version_notes": "Foo bar blah. But with an icon."' in resp.content
         assert b'"visible": true' in resp.content
         version = Version.objects.latest('pk')
-        assert version.number == '2.0'
+        assert version.number == '2.0.%s.0' % now().strftime('%Y%m%d')
         assert version.notes == 'Foo bar blah. But with an icon.'
-        assert version.pk != same_version.pk
 
         resp = app.get('/applications/manifest/test/versions/')
         versions = [e.text() for e in resp.pyquery('h3').items()]
-        assert versions.count('1.0') == 1
-        assert versions.count('2.0') == 1
+        assert versions.count('1.0.%s.0' % now().strftime('%Y%m%d')) == 1
+        assert versions.count('2.0.%s.0' % now().strftime('%Y%m%d')) == 1
         assert resp.text.count('Compare') == 2
-        assert resp.text.count('Creating application bundle') == 3
-        resp = resp.click(href='/applications/manifest/test/download/%s/' % same_version.pk)
+        assert resp.text.count('Creating application bundle') == 2
+        resp = resp.click(
+            href='/applications/manifest/test/download/%s/' % Version.objects.get(number__startswith='1.0').pk
+        )
         assert resp.content_type == 'application/x-tar'
-        assert resp.headers['Content-Disposition'] == 'attachment; filename="test-1.0.tar"'
-        assert b'"version_number": "1.0"' in resp.content
+        assert resp.headers[
+            'Content-Disposition'
+        ] == 'attachment; filename="test-1.0.%s.0.tar"' % now().strftime('%Y%m%d')
+        assert b'"version_number": "1.0.%s.0"' % now().strftime('%Y%m%d').encode() in resp.content
         resp = app.get('/applications/manifest/test/versions/')
-        resp = resp.click(href='/applications/manifest/test/download/%s/' % version.pk)
+        resp = resp.click(
+            href='/applications/manifest/test/download/%s/' % Version.objects.get(number__startswith='2.0').pk
+        )
         assert resp.content_type == 'application/x-tar'
-        assert resp.headers['Content-Disposition'] == 'attachment; filename="test-2.0.tar"'
-        assert b'"version_number": "2.0"' in resp.content
+        assert resp.headers[
+            'Content-Disposition'
+        ] == 'attachment; filename="test-2.0.%s.0.tar"' % now().strftime('%Y%m%d')
+        assert b'"version_number": "2.0.%s.0"' % now().strftime('%Y%m%d').encode() in resp.content
 
-        resp = app.get('/applications/manifest/test/generate/')
-        assert resp.form['number'].value == '2.0'  # last one
-        assert resp.form['notes'].value == 'Foo bar blah. But with an icon.'  # last one
-        resp.form['number'] = '1.0'  # old number
+        resp = app.get('/applications/manifest/test/')
+        resp = resp.click('Generate application bundle')
+        resp.form['number'] = '2.0'
+        resp.form['notes'] = 'Foo bar blah. But with an icon.'
         resp = resp.form.submit().follow()
-        new_version = Version.objects.latest('pk')
-        assert new_version.number == '1.0'
-        assert new_version.notes == 'Foo bar blah. But with an icon.'
-        assert new_version.pk != version.pk  # new version created
+        resp = resp.click('Download')
+        assert resp.content_type == 'application/x-tar'
+        # uncompressed tar, primitive check of contents
+        assert b'<formdef/>' in resp.content
+        assert b'<carddef/>' in resp.content
+        assert b'"icon": "foo' in resp.content
+        assert b'"documentation_url": ""' in resp.content
+        assert b'"version_number": "2.0.%s.1"' % now().strftime('%Y%m%d').encode() in resp.content
+        assert b'"version_notes": "Foo bar blah. But with an icon."' in resp.content
+        assert b'"visible": true' in resp.content
+        version = Version.objects.latest('pk')
+        assert version.number == '2.0.%s.1' % now().strftime('%Y%m%d')
+        assert version.notes == 'Foo bar blah. But with an icon.'
+
         resp = app.get('/applications/manifest/test/versions/')
         versions = [e.text() for e in resp.pyquery('h3').items()]
-        assert versions.count('1.0') == 2
-        assert versions.count('2.0') == 1
-        assert resp.text.count('Creating application bundle') == 4
+        assert resp.text.count('Creating application bundle') == 3
         assert resp.text.count('Compare') == 3
         resp = resp.click('Compare', index=0)
-        assert 'Compare version 1.0 to:' in resp
+        assert 'Compare version 2.0.%s.1 to:' % now().strftime('%Y%m%d') in resp
+        version1 = Version.objects.get(number__startswith='1.0')
+        version2 = Version.objects.filter(number__startswith='2.0').latest('-pk')
+        version3 = Version.objects.filter(number__startswith='2.0').latest('pk')
         assert resp.form['version'].options == [
-            (str(version.pk), False, '2.0'),
-            (str(same_version.pk), False, '1.0'),
+            (str(version2.pk), False, version2.number),
+            (str(version1.pk), False, version1.number),
         ]
         resp = resp.form.submit()
         assert resp.location.endswith(
             '/applications/manifest/test/version/compare/?version1=%s&version2=%s'
-            % (new_version.pk, version.pk)
+            % (version3.pk, version2.pk)
         )
         resp = resp.follow()
-        assert 'Version 2.0' in resp
-        assert 'Version 1.0' in resp
+        assert 'Version %s' % version3.number in resp
+        assert 'Version %s' % version2.number in resp
         resp = resp.click('Compare elements definitions')
-        assert 'Version 2.0' in resp
-        assert 'Version 1.0' in resp
+        assert 'Version %s' % version3.number in resp
+        assert 'Version %s' % version2.number in resp
         assert (
-            'https://wcs.example.invalid/api/export-import/forms/test-form/redirect/?application=test&version1=2.0&version2=1.0&compare'
+            'https://wcs.example.invalid/api/export-import/forms/test-form/redirect/?application=test&version1=%s&version2=%s&compare'
+            % (version2.number, version3.number)
             in resp
         )
         assert (
-            'https://wcs.example.invalid/api/export-import/cards/test-card/redirect/?application=test&version1=2.0&version2=1.0&compare'
+            'https://wcs.example.invalid/api/export-import/cards/test-card/redirect/?application=test&version1=%s&version2=%s&compare'
+            % (version2.number, version3.number)
             in resp
         )
         assert 'Test Form <span class="extra-info">- Form</span>' in resp
@@ -479,16 +475,18 @@ def test_create_application(app, admin_user, settings, analyze):
         Element.objects.filter(type='cards').delete()
         resp = app.get(
             '/applications/manifest/test/version/compare/?version1=%s&version2=%s&mode=elements'
-            % (new_version.pk, version.pk)
+            % (version3.pk, version2.pk)
         )
-        assert 'Version 2.0' in resp
-        assert 'Version 1.0' in resp
+        assert 'Version %s' % version3.number in resp
+        assert 'Version %s' % version2.number in resp
         assert (
-            'https://wcs.example.invalid/api/export-import/forms/test-form/redirect/?application=test&version1=2.0&version2=1.0&compare'
+            'https://wcs.example.invalid/api/export-import/forms/test-form/redirect/?application=test&version1=%s&version2=%s&compare'
+            % (version2.number, version3.number)
             in resp
         )
         assert (
-            'https://wcs.example.invalid/api/export-import/cards/test-card/redirect/?application=test&version1=2.0&version2=1.0&compare'
+            'https://wcs.example.invalid/api/export-import/cards/test-card/redirect/?application=test&version1=%s&version2=%s&compare'
+            % (version2.number, version3.number)
             not in resp
         )
         assert 'Test Form <span class="extra-info">- Form</span>' in resp
@@ -521,6 +519,123 @@ def test_create_application(app, admin_user, settings, analyze):
     app.get('/applications/manifest/test/delete/%s/' % application.relation_set.first().pk, status=404)
 
 
+def test_application_version_number(app, admin_user, settings):
+    Wcs.objects.create(base_url='https://wcs.example.invalid', slug='foobar', title='Foobar')
+
+    settings.KNOWN_SERVICES = {
+        'wcs': {
+            'blah': {
+                # simulate an instance from another collectivity
+                'title': 'Unknown',
+                'url': 'https://unknown.example.invalid/',
+                'orig': 'example.org',
+                'secret': 'xxx',
+            },
+            'foobar': {
+                'title': 'Foobar',
+                'url': 'https://wcs.example.invalid/',
+                'orig': 'example.org',
+                'secret': 'xxx',
+            },
+        }
+    }
+
+    login(app)
+
+    application = Application.objects.create(name='Test', slug='test')
+    resp = app.get('/applications/manifest/test/generate/')
+    resp.form['number'] = 'X'
+    resp = resp.form.submit()
+    assert resp.context['form'].errors == {'number': ['Enter a valid value.']}
+    resp.form['number'] = 'X.0'
+    resp = resp.form.submit()
+    assert resp.context['form'].errors == {'number': ['Enter a valid value.']}
+    resp.form['number'] = '1.1.1'
+    resp = resp.form.submit()
+    assert resp.context['form'].errors == {'number': ['Enter a valid value.']}
+    resp.form['number'] = '1.1'
+    with StatefulHTTMock(mocked_http):
+        resp.form.submit()
+    last_version = Version.objects.latest('pk')
+    assert last_version.application == application
+    assert last_version.number == '1.1.%s.0' % now().strftime('%Y%m%d')
+
+    resp = app.get('/applications/manifest/test/generate/')
+    assert resp.form['number'].value == '1.1'
+    with StatefulHTTMock(mocked_http):
+        resp.form.submit()
+    last_version = Version.objects.latest('pk')
+    assert last_version.application == application
+    assert last_version.number == '1.1.%s.1' % now().strftime('%Y%m%d')
+
+    resp = app.get('/applications/manifest/test/generate/')
+    assert resp.form['number'].value == '1.1'
+    with StatefulHTTMock(mocked_http):
+        resp.form.submit()
+    last_version = Version.objects.latest('pk')
+    assert last_version.application == application
+    assert last_version.number == '1.1.%s.2' % now().strftime('%Y%m%d')
+
+    resp = app.get('/applications/manifest/test/generate/')
+    resp.form['number'] = '1.0'
+    resp = resp.form.submit()
+    assert resp.context['form'].errors == {
+        'number': ['The version number must be equal to or greater than the previous one.']
+    }
+
+    last_version.number = '1.1.%s.2' % (now() - datetime.timedelta(days=1)).strftime('%Y%m%d')
+    last_version.save()
+    resp = app.get('/applications/manifest/test/generate/')
+    assert resp.form['number'].value == '1.1'
+    with StatefulHTTMock(mocked_http):
+        resp.form.submit()
+    last_version = Version.objects.latest('pk')
+    assert last_version.application == application
+    assert last_version.number == '1.1.%s.0' % now().strftime('%Y%m%d')
+
+    last_version.number = 'garbage'
+    last_version.save()
+    resp = app.get('/applications/manifest/test/generate/')
+    assert resp.form['number'].value == ''
+    resp.form['number'] = '1.1'
+    with StatefulHTTMock(mocked_http):
+        resp.form.submit()
+    last_version = Version.objects.latest('pk')
+    assert last_version.application == application
+    assert last_version.number == '1.1.%s.0' % now().strftime('%Y%m%d')
+
+    last_version.number = '1'
+    last_version.save()
+    resp = app.get('/applications/manifest/test/generate/')
+    assert resp.form['number'].value == '1'
+    resp.form['number'] = '1.1'
+    with StatefulHTTMock(mocked_http):
+        resp.form.submit()
+    last_version = Version.objects.latest('pk')
+    assert last_version.application == application
+    assert last_version.number == '1.1.%s.0' % now().strftime('%Y%m%d')
+
+    last_version.number = '1.1'
+    last_version.save()
+    resp = app.get('/applications/manifest/test/generate/')
+    assert resp.form['number'].value == '1.1'
+    with StatefulHTTMock(mocked_http):
+        resp.form.submit()
+    last_version = Version.objects.latest('pk')
+    assert last_version.application == application
+    assert last_version.number == '1.1.%s.0' % now().strftime('%Y%m%d')
+
+    last_version.number = '1.1.%s.1.1' % now().strftime('%Y%m%d')
+    last_version.save()
+    resp = app.get('/applications/manifest/test/generate/')
+    assert resp.form['number'].value == '1.1'
+    with StatefulHTTMock(mocked_http):
+        resp.form.submit()
+    last_version = Version.objects.latest('pk')
+    assert last_version.application == application
+    assert last_version.number == '1.1.%s.2' % now().strftime('%Y%m%d')
+
+
 def test_manifest_ordering(app, admin_user, settings):
     Wcs.objects.create(base_url='https://wcs.example.invalid', slug='foobar', title='Foobar')
 
@@ -913,7 +1028,7 @@ def get_bundle(with_icon=False):
         manifest_json = {
             'application': 'Test',
             'slug': 'test',
-            'icon': 'foo.png' if with_icon else None,
+            'icon': 'foo.jpeg' if with_icon else None,
             'description': '',
             'documentation_url': 'http://foo.bar',
             'version_number': '43.0' if with_icon else '42.0',
@@ -939,12 +1054,10 @@ def get_bundle(with_icon=False):
         tarinfo.size = len(manifest_fd.getvalue())
         tar.addfile(tarinfo, fileobj=manifest_fd)
         if with_icon:
-            icon_fd = io.BytesIO(
-                b'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQAAAAA3bvkkAAAACklEQVQI12NoAAAAggCB3UNq9AAAAABJRU5ErkJggg=='
-            )
-            tarinfo = tarfile.TarInfo('foo.png')
-            tarinfo.size = len(icon_fd.getvalue())
-            tar.addfile(tarinfo, fileobj=icon_fd)
+            with open(os.path.join(TESTS_DATA_DIR, 'black.jpeg'), mode='rb') as icon_fd:
+                tarinfo = tarfile.TarInfo(manifest_json['icon'])
+                tarinfo.size = 558
+                tar.addfile(tarinfo, fileobj=icon_fd)
 
     return tar_io.getvalue()
 
@@ -999,7 +1112,7 @@ def test_deploy_application(app, admin_user, settings, app_bundle, app_bundle_wi
         application = Application.objects.get(slug='test')
         assert application.name == 'Test'
         if bundle == app_bundle_with_icon:
-            assert re.match(r'applications/icons/foo(_\w+)?.png', application.icon.name)
+            assert re.match(r'applications/icons/foo(_\w+)?.jpeg', application.icon.name)
         else:
             assert application.icon.name == ''
         assert application.documentation_url == 'http://foo.bar'
@@ -1182,6 +1295,55 @@ def test_deploy_application(app, admin_user, settings, app_bundle, app_bundle_wi
     resp = resp.form.submit()
     assert resp.context['form'].errors == {'bundle': ['Invalid tar file.']}
 
+    # missing manifest
+    tar_io = io.BytesIO()
+    with tarfile.open(mode='w', fileobj=tar_io) as tar:
+        foo_fd = io.BytesIO(json.dumps({'foo': 'bar'}, indent=2).encode())
+        tarinfo = tarfile.TarInfo('foo.json')
+        tarinfo.size = len(foo_fd.getvalue())
+        tar.addfile(tarinfo, fileobj=foo_fd)
+    resp = app.get('/applications/')
+    if action == 'Update':
+        with StatefulHTTMock(mocked_http2):
+            resp = resp.click(href='manifest/test/')
+    resp = resp.click(action)
+    resp.form['bundle'] = Upload('app.tar', tar_io.getvalue(), 'application/x-tar')
+    resp = resp.form.submit()
+    assert resp.context['form'].errors == {'bundle': ['Invalid tar file, missing manifest.']}
+
+    # bad icon file
+    tar_io = io.BytesIO()
+    with tarfile.open(mode='w', fileobj=tar_io) as tar:
+        manifest_json = {
+            'application': 'Test',
+            'slug': 'test',
+            'icon': 'foo.png',
+            'description': '',
+            'documentation_url': 'http://foo.bar',
+            'version_number': '42.0',
+            'version_notes': 'foo bar blah',
+            'elements': [],
+        }
+        manifest_fd = io.BytesIO(json.dumps(manifest_json, indent=2).encode())
+        tarinfo = tarfile.TarInfo('manifest.json')
+        tarinfo.size = len(manifest_fd.getvalue())
+        tar.addfile(tarinfo, fileobj=manifest_fd)
+        icon_fd = io.BytesIO(b'garbage')
+        tarinfo = tarfile.TarInfo('foo.png')
+        tarinfo.size = len(icon_fd.getvalue())
+        tar.addfile(tarinfo, fileobj=icon_fd)
+
+    bundle = tar_io.getvalue()
+
+    resp = app.get('/applications/')
+    if action == 'Update':
+        with StatefulHTTMock(mocked_http2):
+            resp = resp.click(href='manifest/test/')
+    resp = resp.click(action)
+    resp.form['bundle'] = Upload('app.tar', bundle, 'application/x-tar')
+    resp = resp.form.submit()
+    assert resp.context['form'].errors == {'bundle': ['Invalid icon file.']}
+
 
 def test_update_application(app, admin_user, settings, app_bundle):
     Wcs.objects.create(base_url='https://wcs.example.invalid', slug='foobar', title='Foobar')

tests/test_provision_groups.py

@@ -0,0 +1,34 @@
+import logging
+
+import pytest
+
+from hobo.agent.common.models import Role
+from hobo.multitenant.utils import provision_user_groups
+
+pytestmark = pytest.mark.django_db
+
+
+@pytest.fixture
+def roles():
+    return [Role.objects.create(uuid=str(i) * 8, name='Role %s' % i) for i in range(5)]
+
+
+def test_provision_user_groups(db, roles, admin_user, caplog):
+    caplog.set_level(logging.INFO)
+    roles_uuids = [r.uuid for r in roles]
+    provision_user_groups(admin_user, roles_uuids)
+    assert len(caplog.records) == len(roles)
+    assert list(admin_user.groups.all()) == [r.group_ptr for r in roles]
+
+    caplog.clear()
+    caplog.set_level(logging.WARNING)
+    provision_user_groups(admin_user, roles_uuids)
+    assert list(admin_user.groups.all()) == [r.group_ptr for r in roles]
+    assert len(caplog.records) == 0
+
+    provision_user_groups(admin_user, roles_uuids[:-1])
+    assert list(admin_user.groups.all()) == [r.group_ptr for r in roles[:-1]]
+
+    roles_uuids.append('unknown-uuid')
+    provision_user_groups(admin_user, roles_uuids)
+    assert len(caplog.records) == 1

tests_authentic/settings.py

@@ -48,6 +48,3 @@ SESSION_COOKIE_SECURE = False
 CSRF_COOKIE_SECURE = False
 
 LANGUAGE_CODE = 'en'
-
-# noqa pylint: disable=undefined-variable
-LOGGING['handlers']['debug']['filename'] = 'debug.log'

tests_multitenant/conftest.py

@@ -85,6 +85,13 @@ def make_tenant(tmp_path, transactional_db, settings, request):
                             'template_name': '...portal-user...',
                             'base_url': 'http://portal-user.example.net',
                         },
+                        {
+                            'slug': 'payment',
+                            'title': 'Lingo',
+                            'service-id': 'lingo',
+                            'secret_key': '123456',
+                            'base_url': 'https://payment.example.net',
+                        },
                     ],
                 },
                 fd,

tests_multitenant/test_logger.py

@@ -14,97 +14,42 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import datetime
 import logging
 import logging.config
-import time
 
-import pytest
-import pytz
-from tenant_schemas.utils import tenant_context
-
-from hobo.logger import ClampLogLevel, DebugLog
+from hobo.logger import ClampLogLevel, DebugLogFilter
 
 
-@pytest.fixture
-def debug_log(settings, tmpdir):
-    debug_log_path = str(tmpdir / 'debug.log')
-    settings.DEBUG_LOG_PATH = debug_log_path
+def test_debug_log_filter(settings, tmp_path):
+    log_path = tmp_path / 'debug.log'
 
-    LOGGING = {
-        'version': 1,
-        'disable_existing_loggers': True,
-        'filters': {
-            'debug_log': {
-                '()': 'hobo.logger.DebugLogFilter',
-            },
-            'request_context': {
-                '()': 'hobo.logger.RequestContextFilter',
-            },
-        },
-        'formatters': {
-            'debug': {
-                'format': settings.DEBUG_LOG_FORMAT,
-            },
-        },
-        'handlers': {
-            'debug': {
-                'level': 'DEBUG',
-                'class': 'hobo.logger.TimedRotatingFileHandler',
-                'formatter': 'debug',
-                'filename': debug_log_path,
-                'when': 'midnight',
-                'backupCount': 1,
-                'interval': 1,
-                'filters': ['request_context', 'debug_log'],
-            }
-        },
-        'loggers': {
-            'multitenant': {
-                'level': 'DEBUG',
-                'handlers': ['debug'],
-            },
-        },
-    }
-    logging.config.dictConfig(LOGGING)
+    settings.DEBUG_LOG = False
 
-    logger = logging.getLogger('multitenant')
-    yield logger
-    logger.removeHandler(logger.handlers[0])
+    logger = logging.getLogger('test')
+    handler = logging.FileHandler(str(log_path))
+    handler.setLevel(logging.DEBUG)
+    handler.addFilter(DebugLogFilter())
+    logger.addHandler(handler)
+    logger.propagate = False
+    logger.setLevel(logging.DEBUG)
 
-
-def test_debug_log(tenants, settings, app, rf, debug_log, freezer):
-    freezer.move_to('2020-4-20')
-    request = rf.get('/path/')
-    debug_log.info('test %s is ok', 1, extra={'request': request, 'tenant': 'yes'})
-    lines = list(DebugLog.lines())
-    assert len(lines) == 0
+    logger.info('barfoo')
+    logger.debug('foobar')
+    assert log_path.read_text().count('barfoo') == 1
+    assert log_path.read_text().count('foobar') == 0
 
     settings.DEBUG_LOG = True
-    with tenant_context(tenants[0]):
-        debug_log.info(
-            'log %s is \nok', 2, extra={'request': request, 'tenant': 'tenant1', 'user': 'jean darmette'}
-        )
-        debug_log.debug('log %s is \nok', 3, extra={'request': request})
-    lines = list(DebugLog.lines())
-    assert len(lines) == 2
-    request_id = hex(id(request))[2:].upper()
-    assert lines[0] == {
-        'cursor': 111,
-        'ip': '127.0.0.1',
-        'request_id': 'r:' + request_id,
-        'message': 'log 2 is \nok',
-        'level': 'INFO',
-        'tenant': 'tenant1.example.net',
-        'timestamp': pytz.timezone(time.tzname[0]).localize(datetime.datetime(2020, 4, 20, 2, 0)),
-        'user': '-',
-        'logger': 'multitenant',
-    }
+    logger.debug('foobar')
+    assert log_path.read_text().count('foobar') == 1
 
-    # check that seeking by cursor gives the same lines
-    lines2 = list(DebugLog.lines(cursor=lines[0]['cursor']))
-    assert len(lines2) == 1
-    assert lines[1] == lines2[0]
+    settings.DEBUG_LOG = 'test.foobar,test.foobar2'
+    logger.debug('foobar')
+    assert log_path.read_text().count('foobar') == 1
+
+    logging.getLogger('test.foobar').debug('foobar')
+    logging.getLogger('test.foobar2').debug('foobar')
+    logging.getLogger('test.foobar3').debug('foobar')
+    assert log_path.read_text().count('foobar') == 3
 
 
 def test_clamp_log_level(caplog):

tests_multitenant/test_request_context_filter.py

@@ -1,4 +1,5 @@
 import logging
+from unittest import mock
 
 import pytest
 from _pytest.logging import LogCaptureHandler
@@ -75,13 +76,14 @@ def test_request_context_filter(caplog, settings, tenants, client):
 
 @pytest.fixture
 def sender():
-    yield MockSender()
+    with mock.patch('hobo.journal.journal.send') as sender:
+        yield sender
 
 
 @pytest.fixture
 def journald_handler(sender):
     root_logger = logging.getLogger()
-    journald_handler = JournalHandler(sender_function=sender.send)
+    journald_handler = JournalHandler()
     journald_handler.addFilter(RequestContextFilter())
 
     root_logger.handlers.insert(0, journald_handler)  # head insert
@@ -108,19 +110,29 @@ def test_systemd(settings, tenants, client, journald_handler, sender):
             client.login(username='john.doe', password='john.doe')
         client.get('/', SERVER_NAME=tenant.domain_url, HTTP_X_FORWARDED_FOR='99.99.99.99, 127.0.0.1')
 
-    assert len(sender.calls) == 2
-    for tenant, (args, kwargs) in zip(tenants, sender.calls):
-        assert args == ('wat!',)
-        assert kwargs['IP'] == '99.99.99.99'
-        assert kwargs['TENANT'] == tenant.domain_url
-        assert kwargs['PATH'] == '/'
-        assert kwargs['REQUEST_ID'].startswith('r:')
-        assert kwargs['USER'] == user.username
-        assert kwargs['USER_EMAIL'] == user.email
-        assert kwargs['USER_NAME'] == user.username
-        assert kwargs['USER_DISPLAY_NAME'] == 'John Doe'
-        assert kwargs['USER_UUID'] == 'ab' * 16
-        assert kwargs['APPLICATION'] == 'fake-agent'
+    assert len(sender.mock_calls) == 2
+    for tenant, call in zip(tenants, sender.mock_calls):
+        for key, value in [
+            ('APPLICATION', 'fake-agent'),
+            ('IP', '99.99.99.99'),
+            ('LEVELNAME', 'ERROR'),
+            ('LEVELNO', 40),
+            ('LOGGER', 'hobo.test_urls'),
+            ('MESSAGE', 'wat!'),
+            ('MODULE', 'test_urls'),
+            ('MSG', 'wat!'),
+            ('NAME', 'hobo.test_urls'),
+            ('PATH', '/'),
+            ('PRIORITY', '3'),
+            ('REQUEST_ID', mock.ANY),
+            ('TENANT', tenant.domain_url),
+            ('USER_DISPLAY_NAME', 'John Doe'),
+            ('USER_EMAIL', 'jodn.doe@example.com'),
+            ('USER', 'john.doe'),
+            ('USER_NAME', 'john.doe'),
+            ('USER_UUID', 'abababababababababababababababab'),
+        ]:
+            assert call.kwargs[key] == value
 
 
 def test_debug_log_filter(caplog, settings):

tests_multitenant/test_settings.py

@@ -88,6 +88,7 @@ def test_tenant_template_vars(tenants, settings, client):
                 assert (
                     django.conf.settings.TEMPLATE_VARS['portal_user_url'] == 'http://portal-user.example.net'
                 )
+                assert django.conf.settings.TEMPLATE_VARS['lingo_url'] == 'https://payment.example.net/'
 
         # check it's no longer defined after going back to the public schema
         with pytest.raises(AttributeError):

tests_passerelle/settings.py

@@ -32,4 +32,3 @@ LOGGING['loggers']['suds'] = {
     'handlers': ['mail_admins'],
     'propagate': True,
 }
-LOGGING['handlers']['debug']['filename'] = 'debug.log'