lingo: send the amount paid in pay_invoice() and notify() (#76572)

This commit factorize the creation of the JSON payload in pay_invoice()
and notify(), so that they send the same information.

The pay_invoice method is moved on the Transaction class.

.coveragerc

@@ -0,0 +1,5 @@
+[run]
+dynamic_context = test_function
+
+[html]
+show_contexts = True

.git-blame-ignore-revs

@@ -0,0 +1,12 @@
+# trivial: apply black
+47d67c395ef124ea4534ea7c56d48d4e302db430
+# misc: apply isort (#52797)
+29bc8e66a978b1bbadbc05186599a70ce1b8ef98
+# misc: apply pyupgrade (#55868)
+f4615c506194cf4eace0af551f14f54552f09dc5
+# misc: apply djhtml (#69709)
+4784a3990eec0aadadce99bf1da21c0b531b289d
+# misc: apply django-upgrade (#69798)
+cd498afcb0bcd8d3f2fce98c9747ca1406d9a449
+# misc: apply double-quote-string-fixer (#79788)
+9d8876e155e8a433ebafbd4ccb9de4960d1830b4

.gitignore

@@ -11,8 +11,17 @@ combo.egg-info/
 .sass-cache/
 combo/apps/maps/static/css/combo.map.css
 combo/apps/maps/static/css/combo.map.css.map
+combo/apps/pwa/static/css/combo.manager.pwa.css
+combo/apps/pwa/static/css/combo.manager.pwa.css.map
+combo/apps/family/static/css/combo.weekly_agenda.css
+combo/apps/dataviz/static/css/combo.multiselectwidget.css
+combo/manager/static/css/combo.manager.css
 data/themes/gadjo/static/css/agent-portal.css
 data/themes/gadjo/static/css/agent-portal.css.map
 .cache
 .coverage
 .pytest_cache/
+node_modules/
+coverage/
+package.json
+package-lock.json

.pre-commit-config.yaml

@@ -0,0 +1,36 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+    - id: double-quote-string-fixer
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v3.3.1
+    hooks:
+    - id: pyupgrade
+      args: ['--keep-percent-format', '--py39-plus']
+-   repo: https://github.com/adamchainz/django-upgrade
+    rev: 1.13.0
+    hooks:
+    - id: django-upgrade
+      args: ['--target-version', '3.2']
+-   repo: https://github.com/psf/black
+    rev: 23.1.0
+    hooks:
+    - id: black
+      args: ['--target-version', 'py39', '--skip-string-normalization', '--line-length', '110']
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+    - id: isort
+      args: ['--profile', 'black', '--line-length', '110']
+- repo: https://github.com/rtts/djhtml
+  rev: '3.0.5'
+  hooks:
+    - id: djhtml
+      args: ['--tabwidth', '2']
+-   repo: https://git.entrouvert.org/pre-commit-debian.git
+    rev: v0.3
+    hooks:
+    - id: pre-commit-debian

Jenkinsfile

@@ -1,30 +1,43 @@
-@Library('eo-jenkins-lib@master') import eo.Utils
+@Library('eo-jenkins-lib@main') import eo.Utils
 
 pipeline {
     agent any
-    options { disableConcurrentBuilds() }
+    options {
+        disableConcurrentBuilds()
+        timeout(time: 20, unit: 'MINUTES')
+    }
     stages {
         stage('Unit Tests') {
             steps {
-                sh 'tox -rv'
+                sh 'nox'
             }
             post {
                 always {
                     script {
                         utils = new Utils()
-                        utils.publish_coverage('coverage.xml')
+                        utils.publish_coverage('coverage.xml,coverage/cobertura-coverage.xml')
                         utils.publish_coverage_native('index.html')
                         utils.publish_pylint('pylint.out')
                     }
-                    junit '*_results.xml'
+                    mergeJunitResults()
                 }
             }
         }
         stage('Packaging') {
             steps {
                 script {
-                    if (env.JOB_NAME == 'combo' && env.GIT_BRANCH == 'origin/master') {
-                        sh 'sudo -H -u eobuilder /usr/local/bin/eobuilder combo'
+                    env.SHORT_JOB_NAME=sh(
+                        returnStdout: true,
+                        // given JOB_NAME=gitea/project/PR-46, returns project
+                        // given JOB_NAME=project/main, returns project
+                        script: '''
+                            echo "${JOB_NAME}" | sed "s/gitea\\///" | awk -F/ '{print $1}'
+                        '''
+                    ).trim()
+                    if (env.GIT_BRANCH == 'main' || env.GIT_BRANCH == 'origin/main') {
+                        sh "sudo -H -u eobuilder /usr/local/bin/eobuilder -d bullseye,bookworm ${SHORT_JOB_NAME}"
+                    } else if (env.GIT_BRANCH.startsWith('hotfix/')) {
+                        sh "sudo -H -u eobuilder /usr/local/bin/eobuilder -d bullseye,bookworm --branch ${env.GIT_BRANCH} --hotfix ${SHORT_JOB_NAME}"
                     }
                 }
             }
@@ -34,7 +47,7 @@ pipeline {
         always {
             script {
                 utils = new Utils()
-                utils.mail_notify(currentBuild, env, 'admin+jenkins-combo@entrouvert.com')
+                utils.mail_notify(currentBuild, env, 'ci+jenkins-combo@entrouvert.org')
             }
         }
         success {

MANIFEST.in

@@ -1,14 +1,16 @@
 # locales
-recursive-include combo/locale *.po *.mo
+recursive-include combo/locales/locale *.po *.mo
 
 # static
-recursive-include combo/apps/usersearch/static *.css *.js *.ico *.gif *.png *.jpg
 recursive-include combo/apps/lingo/static *.css *.js *.ico *.gif *.png *.jpg
-recursive-include combo/apps/dataviz/static *.css *.js *.ico *.gif *.png *.jpg
+recursive-include combo/apps/dataviz/static *.css *.js *.ico *.gif *.png *.jpg *.scss
 recursive-include combo/apps/dashboard/static *.js
+recursive-include combo/apps/family/static *.css *.scss *.js
+recursive-include combo/apps/gallery/static *.js
 recursive-include combo/apps/maps/static *.css *.scss *.js
 recursive-include combo/apps/pwa/static *.css *.scss *.js *.svg
-recursive-include combo/manager/static *.css *.js *.ico *.gif *.png *.jpg
+recursive-include combo/apps/wcs/static *.js *.css
+recursive-include combo/manager/static *.scss *.css *.js *.ico *.gif *.png *.jpg
 recursive-include combo/public/static *.css *.js *.ico *.gif *.png *.jpg
 
 # templates
@@ -17,13 +19,13 @@ recursive-include combo/apps/assets/templates *.html
 recursive-include combo/apps/calendar/templates *.html
 recursive-include combo/apps/dashboard/templates *.html
 recursive-include combo/apps/search/templates *.html
-recursive-include combo/apps/usersearch/templates *.html
-recursive-include combo/apps/dataviz/templates *.html
+recursive-include combo/apps/dataviz/templates *.html *.svg
 recursive-include combo/apps/family/templates *.html
 recursive-include combo/apps/fargo/templates *.html
+recursive-include combo/apps/gallery/templates *.html
+recursive-include combo/apps/kb/templates *.html
 recursive-include combo/apps/lingo/templates *.html
 recursive-include combo/apps/maps/templates *.html
-recursive-include combo/apps/momo/templates *.html
 recursive-include combo/apps/newsletters/templates *.html
 recursive-include combo/apps/notifications/templates *.html
 recursive-include combo/apps/pwa/templates *.html *.js *.json

README

@@ -13,7 +13,7 @@ Dependencies can be installed with pip,
  $ pip install -r requirements.txt
 
 It's then required to get the database configured (./manage.py migrate); by
-default it will create a db.sqlite3 file.
+default it will create a postgresqsl DB.
 
 You can then run the Django test server for a quick try (you should refer to
 the Django documentation for production deployments).
@@ -88,11 +88,32 @@ Unit tests are written using py.test, and its pytest-django support library.
 
   DJANGO_SETTINGS_MODULE=combo.settings COMBO_SETTINGS_FILE=tests/settings.py py.test
 
-Tests for w.c.s. cells do require access to the wcsctl script, its location has
-to be given in a WCSCTL environment variable, this give a full command line:
 
-  WCSCTL=$(pwd)/wcs/wcsctl.py \
-  DJANGO_SETTINGS_MODULE=combo.settings COMBO_SETTINGS_FILE=tests/settings.py py.test
+Code Style
+----------
+
+black is used to format the code, using thoses parameters:
+
+    black --target-version py37 --skip-string-normalization --line-length 110
+
+isort is used to format the imports, using those parameters:
+
+    isort --profile black --line-length 110
+
+pyupgrade is used to automatically upgrade syntax, using those parameters:
+
+    pyupgrade --keep-percent-format --py37-plus
+
+djhtml is used to automatically indent html files, using those parameters:
+
+    djhtml --tabwidth 2
+
+django-upgrade is used to automatically upgrade Django syntax, using those parameters:
+
+    django-upgrade --target-version 3.2
+
+There is .pre-commit-config.yaml to use pre-commit to automatically run these tools
+before commits. (execute `pre-commit install` to install the git hook.)
 
 
 License
@@ -120,3 +141,10 @@ Gauge.js
   License: MIT
   Comment:
    From http://bernii.github.io/gauge.js/
+
+Pygal.tooltip.js
+  Files: combo/apps/dataviz/static/js/pygal.tooltip.js
+  Copyright: 2015, Florian Mounier Kozea
+  License: LGPL-3+
+  Comment:
+   From https://github.com/Kozea/pygal.js/

combo/apps/assets/__init__.py

@@ -1,34 +0,0 @@
-# combo - content management system
-# Copyright (C) 2017-2018  Entr'ouvert
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import django.apps
-from django.core.urlresolvers import reverse
-from django.utils.translation import ugettext_lazy as _
-
-
-class AppConfig(django.apps.AppConfig):
-    name = 'combo.apps.assets'
-    verbose_name = _('Assets')
-
-    def get_before_urls(self):
-        from . import urls
-        return urls.urlpatterns
-
-    def get_extra_manager_actions(self):
-        return [{'href': reverse('combo-manager-assets'),
-                'text': _('Assets')}]
-
-default_app_config = 'combo.apps.assets.AppConfig'

combo/apps/assets/api_views.py

@@ -0,0 +1,62 @@
+# combo - content management system
+# Copyright (C) 2019  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import base64
+from io import BytesIO
+
+from django.core.files import File
+from rest_framework import permissions, serializers, status
+from rest_framework.generics import GenericAPIView
+from rest_framework.response import Response
+
+from .models import Asset
+
+
+class FileSerializer(serializers.Serializer):
+    content = serializers.CharField(required=True, allow_blank=False)
+    content_type = serializers.CharField(required=False, allow_null=True)
+    filename = serializers.CharField(required=False, allow_null=True)
+
+    def validate_content(self, value):
+        try:
+            return base64.decodebytes(value.encode('ascii'))
+        except Exception as e:
+            raise serializers.ValidationError('content must be base64 (%r)' % e)
+
+
+class AssetSerializer(serializers.Serializer):
+    asset = FileSerializer(required=True)
+
+
+class Set(GenericAPIView):
+    permission_classes = (permissions.IsAuthenticated,)
+    serializer_class = AssetSerializer
+
+    def post(self, request, key, *args, **kwargs):
+        serializer = self.get_serializer(data=request.data)
+        if not serializer.is_valid():
+            response = {'err': 1, 'err_desc': serializer.errors}
+            return Response(response, status.HTTP_400_BAD_REQUEST)
+        data = serializer.validated_data
+
+        asset, dummy = Asset.objects.get_or_create(key=key)
+        asset.asset = File(BytesIO(data['asset']['content']), name=data['asset'].get('filename'))
+        asset.save()
+        response = {'err': 0, 'url': request.build_absolute_uri(f'/assets/{key}')}
+        return Response(response)
+
+
+view_set = Set.as_view()

combo/apps/assets/apps.py

@@ -0,0 +1,32 @@
+# combo - content management system
+# Copyright (C) 2017-2018  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import django.apps
+from django.urls import reverse
+from django.utils.translation import gettext_lazy as _
+
+
+class AppConfig(django.apps.AppConfig):
+    name = 'combo.apps.assets'
+    verbose_name = _('Assets')
+
+    def get_before_urls(self):
+        from . import urls
+
+        return urls.urlpatterns
+
+    def get_extra_manager_actions(self):
+        return [{'href': reverse('combo-manager-assets'), 'text': _('Assets')}]

combo/apps/assets/forms.py

@@ -14,14 +14,30 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import PIL
 from django import forms
-from django.utils.translation import ugettext_lazy as _
+from django.core.exceptions import ValidationError
+from django.utils.translation import gettext_lazy as _
+
+
+def validate_asset_file(value):
+    try:
+        PIL.Image.open(value.file)
+    except PIL.UnidentifiedImageError:
+        pass  # not an image
+    except PIL.Image.DecompressionBombError as expt:
+        raise ValidationError(
+            _('Uploaded image exceeds size limits: %(detail)s'), params={'detail': str(expt)}
+        )
+    return True
 
 
 class AssetUploadForm(forms.Form):
-    upload = forms.FileField(label=_('File'))
+    upload = forms.FileField(label=_('File'), validators=[validate_asset_file])
 
 
 class AssetsImportForm(forms.Form):
-    assets_file = forms.FileField(label=_('Assets File'))
+    assets_file = forms.FileField(
+        label=_('Assets File'), help_text=_('Archive (.tar) with asset files as content.')
+    )
     overwrite = forms.BooleanField(label=_('Overwrite Existing Files'), required=False)

combo/apps/assets/migrations/0001_initial.py

@@ -1,24 +1,23 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.11.12 on 2018-06-12 11:42
-from __future__ import unicode_literals
 
 from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     initial = True
 
-    dependencies = [
-    ]
+    dependencies = []
 
     operations = [
         migrations.CreateModel(
             name='Asset',
             fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                (
+                    'id',
+                    models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+                ),
                 ('key', models.CharField(max_length=128, unique=True)),
-                ('asset', models.FileField(upload_to=b'assets')),
+                ('asset', models.FileField(upload_to='assets')),
             ],
         ),
     ]

combo/apps/assets/migrations/0002_key_maxlength.py

@@ -0,0 +1,15 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('assets', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='asset',
+            name='key',
+            field=models.CharField(max_length=256, unique=True),
+        ),
+    ]

combo/apps/assets/models.py

@@ -19,6 +19,7 @@ import json
 from django.core import serializers
 from django.db import models
 
+
 class AssetManager(models.Manager):
     def get_by_natural_key(self, key):
         return self.get(key=key)
@@ -27,7 +28,7 @@ class AssetManager(models.Manager):
 class Asset(models.Model):
     objects = AssetManager()
 
-    key = models.CharField(max_length=128, unique=True)
+    key = models.CharField(max_length=256, unique=True)
     asset = models.FileField(upload_to='assets')
 
     @classmethod
@@ -35,8 +36,11 @@ class Asset(models.Model):
         return [x.get_as_serialized_object() for x in Asset.objects.all()]
 
     def get_as_serialized_object(self):
-        serialized_asset = json.loads(serializers.serialize('json', [self],
-            use_natural_foreign_keys=True, use_natural_primary_keys=True))[0]
+        serialized_asset = json.loads(
+            serializers.serialize(
+                'json', [self], use_natural_foreign_keys=True, use_natural_primary_keys=True
+            )
+        )[0]
         del serialized_asset['model']
         del serialized_asset['pk']
         return serialized_asset
@@ -49,7 +53,7 @@ class Asset(models.Model):
     @classmethod
     def load_serialized_object(cls, json_asset):
         json_asset['model'] = 'assets.asset'
-        asset, created = Asset.objects.get_or_create(key=json_asset['fields']['key'])
+        asset, dummy = Asset.objects.get_or_create(key=json_asset['fields']['key'])
         json_asset['pk'] = asset.id
-        asset = [x for x in serializers.deserialize('json', json.dumps([json_asset]))][0]
+        asset = next(serializers.deserialize('json', json.dumps([json_asset]), ignorenonexistent=True))
         asset.save()

combo/apps/assets/templates/combo/manager_asset_confirm_delete.html

@@ -2,16 +2,16 @@
 {% load i18n %}
 
 {% block appbar %}
-<h2>{% trans 'Delete Asset' %}</h2>
+  <h2>{% trans 'Delete Asset' %}</h2>
 {% endblock %}
 
 {% block content %}
-<form method="post">
-  {% csrf_token %}
-  {% blocktrans %}Are you sure you want to delete this?{% endblocktrans %}
-  <div class="buttons">
-    <button class="delete-button">{% trans 'Delete' %}</button>
-    <a class="cancel" href="{% url 'combo-manager-assets' %}">{% trans 'Cancel' %}</a>
-  </div>
-</form>
+  <form method="post">
+    {% csrf_token %}
+    {% blocktrans %}Are you sure you want to delete this?{% endblocktrans %}
+    <div class="buttons">
+      <button class="delete-button">{% trans 'Delete' %}</button>
+      <a class="cancel" href="{% url 'combo-manager-assets' %}">{% trans 'Cancel' %}</a>
+    </div>
+  </form>
 {% endblock %}

combo/apps/assets/templates/combo/manager_asset_overwrite.html

@@ -2,20 +2,20 @@
 {% load i18n %}
 
 {% block appbar %}
-<h2>{% trans "Overwrite Asset" %}</h2>
+  <h2>{% trans "Overwrite Asset" %}</h2>
 {% endblock %}
 
 {% block content %}
 
-<form method="post" enctype="multipart/form-data">
-  {% csrf_token %}
-  <p>
-   {% trans "This will erase the existing file and replace it with a new one." %}
-  </p>
-  {{ form.as_p }}
-  <div class="buttons">
-    <button class="submit-button">{% trans "Upload" %}</button>
-    <a class="cancel" href="{% url 'combo-manager-assets' %}">{% trans 'Cancel' %}</a>
-  </div>
-</form>
+  <form method="post" enctype="multipart/form-data">
+    {% csrf_token %}
+    <p>
+      {% trans "This will erase the existing file and replace it with a new one." %}
+    </p>
+    {{ form.as_p }}
+    <div class="buttons">
+      <button class="submit-button">{% trans "Upload" %}</button>
+      <a class="cancel" href="{% url 'combo-manager-assets' %}">{% trans 'Cancel' %}</a>
+    </div>
+  </form>
 {% endblock %}

combo/apps/assets/templates/combo/manager_asset_upload.html

@@ -2,17 +2,17 @@
 {% load i18n %}
 
 {% block appbar %}
-<h2>{% trans "Asset Upload" %}</h2>
+  <h2>{% trans "Asset Upload" %}</h2>
 {% endblock %}
 
 {% block content %}
 
-<form method="post" enctype="multipart/form-data">
-  {% csrf_token %}
-  {{ form.as_p }}
-  <div class="buttons">
-    <button class="submit-button">{% trans "Upload" %}</button>
-    <a class="cancel" href="{% url 'combo-manager-assets' %}">{% trans 'Cancel' %}</a>
-  </div>
-</form>
+  <form method="post" enctype="multipart/form-data">
+    {% csrf_token %}
+    {{ form.as_p }}
+    <div class="buttons">
+      <button class="submit-button">{% trans "Upload" %}</button>
+      <a class="cancel" href="{% url 'combo-manager-assets' %}">{% trans 'Cancel' %}</a>
+    </div>
+  </form>
 {% endblock %}

combo/apps/assets/templates/combo/manager_assets.html

@@ -2,83 +2,49 @@
 {% load i18n %}
 
 {% block appbar %}
-<h2>{% trans 'Assets' %}</h2>
-<span class="actions">
-<a class="extra-actions-menu-opener"></a>
-<a href="{% url 'combo-manager-asset-upload' %}" rel="popup">{% trans 'Upload' %}</a>
-<ul class="extra-actions-menu">
-  <li><a href="{% url 'combo-manager-assets-export' %}">{% trans 'Export Assets' %}</a></li>
-  <li><a rel="popup" href="{% url 'combo-manager-assets-import' %}">{% trans 'Import Assets' %}</a></li>
-</ul>
-</span>
+  <h2>{% trans 'Assets' %}</h2>
+  <span class="actions">
+    <a class="extra-actions-menu-opener"></a>
+    <a href="{% url 'combo-manager-asset-upload' %}" rel="popup">{% trans 'Upload' %}</a>
+    <ul class="extra-actions-menu">
+      <li><a href="{% url 'combo-manager-assets-export' %}">{% trans 'Export assets as archive' %}</a></li>
+      <li><a rel="popup" href="{% url 'combo-manager-assets-import' %}">{% trans 'Import archive of assets' %}</a></li>
+    </ul>
+  </span>
 {% endblock %}
 
 {% block breadcrumb %}
-{{ block.super }}
-<a href="{% url 'combo-manager-assets' %}">{% trans 'Assets' %}</a>
+  {{ block.super }}
+  <a href="{% url 'combo-manager-assets' %}">{% trans 'Assets' %}</a>
 {% endblock %}
 
 {% block content %}
 
-{% if not object_list and not query %}
-<div class="big-msg-info">
-  {% blocktrans %}
-  This site doesn't have any asset yet. You can add some directly when editing
-  pages, in the "Upload Image" dialog.
-  {% endblocktrans %}
-</div>
+  {% if not object_list and not query %}
+    <div class="big-msg-info">
+      {% blocktrans %}
+        This site doesn't have any asset yet. You can add some directly when editing
+        pages, in the "Upload Image" dialog.
+      {% endblocktrans %}
+    </div>
 
-{% else %}
+  {% else %}
 
-<form>
-  <p><input name="q" type="search" value="{{query}}"> <button>{% trans 'Search' %}</button>
-  <span class="help_text">{% trans "(case insensitive search over filenames)" %}</span>
-  </p>
-</form>
+    <form>
+      <p><input name="q" type="search" value="{{query}}"> <button>{% trans 'Search' %}</button>
+        <span class="help_text">{% trans "(case insensitive search over filenames)" %}</span>
+      </p>
+    </form>
 
-<div id="assets-browser">
-  <div id="assets-listing">
-  <table class="main">
-    <thead>
-      <tr>
-        <th>{% trans "Name" %}</th>
-        <th>{% trans "Size" %}</th>
-        <th></th>
-        <th></th>
-      </tr>
-    </thead>
-    <tbody>
-      {% for asset in object_list %}
-      <tr class="{{ asset.css_classes }}">
-        <td><a href="{{ asset.src }}">{{ asset.name }}</a></td>
-        <td>{% if asset.size %}{{ asset.size|filesizeformat }}{% else %}-{% endif %}</td>
-        <td class="image">{% if asset.is_image %}<img data-href="{{ asset.src }}" src="{{ asset.thumb }}"/>{% endif %}</td>
-        <td class="actions">
-          {% if asset.key %}{# theme asset #}
-          <a href="{% url 'combo-manager-slot-asset-upload' key=asset.key %}"
-             class="overwrite" rel="popup">{% trans 'Overwrite' %}</a>
-          {% if asset.asset %}
-          <a href="{% url 'combo-manager-slot-asset-delete' key=asset.key %}"
-             class="delete" rel="popup">{% trans 'Delete' %}</a>
-          {% endif %}
-          {% else %}
-          <a href="{% url 'combo-manager-asset-overwrite' %}?img={{asset.filepath|iriencode}}"
-             class="overwrite" rel="popup">{% trans 'Overwrite' %}</a>
-          <a href="{% url 'combo-manager-asset-delete' %}?img={{asset.filepath|iriencode}}"
-             class="delete" rel="popup">{% trans 'Delete' %}</a>
-          {% endif %}
-        </td>
-        </td>
-      </tr>
-      {% endfor %}
-    </tbody>
-  </table>
-  </div>
-  <div id="asset-preview"></div>
-</div>
+    <div id="assets-browser">
+      <div id="assets-listing">
+        {% include "combo/manager_assets_fragment.html" %}
+      </div>
+      <div id="asset-preview"></div>
+    </div>
 
-{% include "gadjo/pagination.html" %}
+    {% include "gadjo/pagination.html" %}
 
-{% endif %}
+  {% endif %}
 
 {% endblock %}

combo/apps/assets/templates/combo/manager_assets_browse.html

@@ -0,0 +1,73 @@
+{% extends "gadjo/base.html" %}
+{% load gadjo static i18n %}
+
+{% block css %}
+  <link rel="stylesheet" type="text/css" media="all" href="{% static "css/combo.manager.css" %}?{% start_timestamp %}"/>
+{% endblock %}
+
+{% block extrascripts %}
+  <script src="{% static "js/combo.manager.js" %}?{% start_timestamp %}"></script>
+{% endblock %}
+
+{% block user-links %}{% endblock %}
+{% block sidepage %}{% endblock %}
+{% block site-header %}{% endblock %}
+{% block bodyargs %}class="no-header"{% endblock %}
+{% block footer %}{% endblock %}
+
+{% block appbar %}
+  <h2>{% trans "Browse for the file you want, then click 'Embed File' to continue..." %}</h2>
+{% endblock %}
+
+{% block content %}
+
+  {% if not object_list and not query %}
+    <div class="big-msg-info">
+      {% trans "No files found. Upload files using the 'Image Button' or 'Link Button' dialog's 'Upload' tab." %}
+    </div>
+
+  {% else %}
+
+    <form>
+      {% for k, v in request.GET.items %}
+        {% if k != 'q' %}<input type="hidden" name="{{ k }}" value="{{ v }}" />{% endif %}
+      {% endfor %}
+      <p><input name="q" type="search" value="{{query}}"> <button>{% trans 'Search' %}</button>
+        <span class="help_text">{% trans "(case insensitive search over filenames)" %}</span>
+      </p>
+    </form>
+
+    <div id="assets-browser" class="assets-ckeditor">
+      <div id="assets-listing">
+        {% with asset_for_ckeditor=True %}
+          {% include "combo/manager_assets_fragment.html" %}
+        {% endwith %}
+      </div>
+      <div id="asset-preview"></div>
+    </div>
+
+    {% include "gadjo/pagination.html" %}
+
+    <div class="buttons">
+      <input href="" id="asset-ckeditor-embed" type="submit" name="_embed" value="{% trans "Embed File" %}" disabled />
+    </div>
+
+  {% endif %}
+  <script type="text/javascript">
+    // helper functions
+    function getUrlParam(paramName) {
+      var reParam = new RegExp('(?:[\?&]|&amp;)' + paramName + '=([^&]+)', 'i') ;
+      var match = window.location.search.match(reParam) ;
+
+      return (match && match.length > 1) ? match[1] : '' ;
+    }
+
+    // embedder
+    $(document).on('click', '#asset-ckeditor-embed', function() {
+      var funcNum = getUrlParam('CKEditorFuncNum');
+      var fileUrl = $(this).attr('href');
+      window.opener.CKEDITOR.tools.callFunction(funcNum, fileUrl);
+      window.close();
+    });
+  </script>
+{% endblock %}

combo/apps/assets/templates/combo/manager_assets_export_size.html

@@ -0,0 +1 @@
+({{ size|filesizeformat }})

combo/apps/assets/templates/combo/manager_assets_fragment.html

@@ -0,0 +1,46 @@
+{% load i18n %}
+<table class="main">
+  <thead>
+    <tr>
+      <th>{% trans "Name" %}</th>
+      <th>{% trans "Size" %}</th>
+      <th></th>
+      {% if not asset_for_ckeditor %}
+        <th></th>
+      {% endif %}
+    </tr>
+  </thead>
+  <tbody>
+    {% for asset in object_list %}
+      <tr class="{{ asset.css_classes }}" data-href="{{ asset.src }}">
+        <td>
+          {% if asset_for_ckeditor %}
+            {{ asset.name }}
+          {% elif asset.src %}
+            <a href="{{ asset.src }}">{{ asset.name }}</a>
+          {% else %}
+            {{ asset.name }} <span class="not-defined">({% trans "not defined" %})</span>
+          {% endif %}
+          <td>{% if asset.size %}{{ asset.size|filesizeformat }}{% else %}-{% endif %}</td>
+          <td class="image">{% if asset.is_image %}<img data-href="{{ asset.src }}" src="{{ asset.thumb }}"/>{% endif %}</td>
+          {% if not asset_for_ckeditor %}
+            <td class="actions">
+              {% if asset.key %}{# theme asset #}
+                <a href="{% url 'combo-manager-slot-asset-upload' key=asset.key %}{% if cell_reference %}?cell_reference={{ cell_reference }}{% endif %}"
+                   class="overwrite" rel="popup">{% trans 'Overwrite' %}</a>
+                {% if asset.asset %}
+                  <a href="{% url 'combo-manager-slot-asset-delete' key=asset.key %}{% if cell_reference %}?cell_reference={{ cell_reference }}{% endif %}"
+                     class="delete" rel="popup">{% trans 'Delete' %}</a>
+                {% endif %}
+              {% else %}
+                <a href="{% url 'combo-manager-asset-overwrite' %}?img={{asset.filepath|iriencode}}"
+                   class="overwrite" rel="popup">{% trans 'Overwrite' %}</a>
+                <a href="{% url 'combo-manager-asset-delete' %}?img={{asset.filepath|iriencode}}"
+                   class="delete" rel="popup">{% trans 'Delete' %}</a>
+              {% endif %}
+            </td>
+          {% endif %}
+        </tr>
+    {% endfor %}
+  </tbody>
+</table>

combo/apps/assets/templates/combo/manager_assets_import.html

@@ -2,17 +2,20 @@
 {% load i18n %}
 
 {% block appbar %}
-<h2>{% trans "Assets Import" %}</h2>
+  <h2>{% trans "Assets Archive Import" %}</h2>
 {% endblock %}
 
 {% block content %}
 
-<form method="post" enctype="multipart/form-data">
-  {% csrf_token %}
-  {{ form.as_p }}
-  <div class="buttons">
-    <button class="submit-button">{% trans "Import" %}</button>
-    <a class="cancel" href="{% url 'combo-manager-assets' %}">{% trans 'Cancel' %}</a>
-  </div>
-</form>
+  <form method="post" enctype="multipart/form-data">
+    {% csrf_token %}
+    <p>
+      {% trans "Assets archive import allows you to integrate assets exported from another site." %}
+    </p>
+    {{ form.as_p }}
+    <div class="buttons">
+      <button class="submit-button">{% trans "Import" %}</button>
+      <a class="cancel" href="{% url 'combo-manager-assets' %}">{% trans 'Cancel' %}</a>
+    </div>
+  </form>
 {% endblock %}

combo/apps/assets/templates/combo/manager_slot_assets.html

@@ -0,0 +1,19 @@
+{% extends "combo/manager_base.html" %}
+{% load i18n %}
+
+{% block appbar %}
+  <h2>{% trans 'Assets' %}</h2>
+{% endblock %}
+
+{% block breadcrumb %}
+  {{ block.super }}
+  <a href="{% url 'combo-manager-assets' %}">{% trans 'Assets' %}</a>
+{% endblock %}
+
+{% block content %}
+
+  <div id="assets-listing">
+    {% include "combo/manager_assets_fragment.html" %}
+  </div>
+
+{% endblock %}

combo/apps/assets/templatetags/assets.py

@@ -14,10 +14,10 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
+
 from django import template
 from django.db.models.fields.files import ImageFieldFile
-from django.utils import six
-
 from sorl.thumbnail.shortcuts import get_thumbnail
 
 from ..models import Asset
@@ -37,7 +37,7 @@ def asset_url(*args, **kwargs):
             asset = asset_object
             break
 
-        if isinstance(asset_object, six.string_types):
+        if isinstance(asset_object, str):
             try:
                 asset = Asset.objects.get(key=asset_object).asset
                 break
@@ -51,15 +51,49 @@ def asset_url(*args, **kwargs):
     if not asset:
         return ''
 
+    if not os.path.exists(asset.path):
+        return asset.url
+
     geometry_string = kwargs.pop('size', None)
-    if not geometry_string or asset.file.name.endswith('svg'):
+    if not geometry_string or asset.name.endswith('svg'):
         return asset.url
 
     return get_thumbnail(asset, geometry_string, **kwargs).url
 
 
-@register.assignment_tag
-def get_asset(key):
+@register.simple_tag
+def asset_css_url(*args, **kwargs):
+    url = asset_url(*args, **kwargs)
+    if url:
+        return 'url(%s)' % url
+    else:
+        return 'none'
+
+
+@register.simple_tag(takes_context=True)
+def get_asset(context, *args, **kwargs):
+    if context.get('traverse_cells'):
+        # assets are not required when we are just searching for page placeholders
+        return None
+
+    key = None
+    if 'cell' in kwargs and 'type' in kwargs:
+        cell = kwargs['cell']
+        cell_type = kwargs['type']
+        try:
+            if not cell.can_have_assets():
+                return None
+            key = cell.get_asset_slot_key(cell_type)
+        except AttributeError:
+            return None
+        if hasattr(cell, '_assets'):
+            return cell._assets.get(key)
+    elif len(args) == 1:
+        key = args[0]
+
+    if not key:
+        return None
+
     try:
         return Asset.objects.get(key=key)
     except Asset.DoesNotExist:

combo/apps/assets/urls.py

@@ -14,25 +14,27 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from django.conf.urls import url, include
+from django.urls import include, path, re_path
 
-from combo.urls_utils import decorated_includes, manager_required
+from combo.urls_utils import decorated_includes, staff_required
 
-from . import views
+from . import api_views, views
 
 assets_manager_urls = [
-    url(r'^$', views.assets, name='combo-manager-assets'),
-    url(r'^delete$', views.asset_delete, name='combo-manager-asset-delete'),
-    url(r'^overwrite/$', views.asset_overwrite, name='combo-manager-asset-overwrite'),
-    url(r'^upload/$', views.asset_upload, name='combo-manager-asset-upload'),
-    url(r'^upload/(?P<key>[\w_:-]+)/$', views.slot_asset_upload, name='combo-manager-slot-asset-upload'),
-    url(r'^delete/(?P<key>[\w_:-]+)/$', views.slot_asset_delete, name='combo-manager-slot-asset-delete'),
-    url(r'^export/$', views.assets_export, name='combo-manager-assets-export'),
-    url(r'^import/$', views.assets_import, name='combo-manager-assets-import'),
+    path('', views.assets, name='combo-manager-assets'),
+    re_path(r'^slots/(?P<cell_reference>[\w_-]+)/$', views.slot_assets, name='combo-manager-slot-assets'),
+    path('delete', views.asset_delete, name='combo-manager-asset-delete'),
+    path('overwrite/', views.asset_overwrite, name='combo-manager-asset-overwrite'),
+    path('upload/', views.asset_upload, name='combo-manager-asset-upload'),
+    re_path(r'^upload/(?P<key>[\w_:-]+)/$', views.slot_asset_upload, name='combo-manager-slot-asset-upload'),
+    re_path(r'^delete/(?P<key>[\w_:-]+)/$', views.slot_asset_delete, name='combo-manager-slot-asset-delete'),
+    path('export/', views.assets_export, name='combo-manager-assets-export'),
+    path('import/', views.assets_import, name='combo-manager-assets-import'),
 ]
 
 urlpatterns = [
-    url(r'^assets/(?P<key>[\w_:-]+)$', views.serve_asset),
-    url(r'^manage/assets/', decorated_includes(manager_required,
-        include(assets_manager_urls))),
+    re_path(r'^assets/(?P<key>[\w_:-]+)$', views.serve_asset),
+    re_path(r'^manage/assets/', decorated_includes(staff_required, include(assets_manager_urls))),
+    re_path(r'^api/assets/set/(?P<key>[\w_:-]+)/$', api_views.view_set, name='api-assets-set'),
+    path('ajax/assets-export-size/', views.assets_export_size, name='combo-manager-assets-export-size'),
 ]

combo/apps/assets/utils.py

@@ -0,0 +1,96 @@
+# combo - content management system
+# Copyright (C) 2020  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import os
+import tarfile
+from io import BytesIO
+
+from django.core.files.storage import default_storage
+
+from .models import Asset
+
+ASSET_DIRS = [
+    'assets',
+    'page-pictures',
+    'uploads',
+]
+
+
+def is_asset_dir(basedir):
+    # exclude dirs like cache or applications, which contain non asset files
+    media_prefix = default_storage.path('')
+    asset_basedirs = [os.path.join(media_prefix, ad) for ad in ASSET_DIRS]
+    for adb in asset_basedirs:
+        if basedir.startswith(adb):
+            return True
+    return False
+
+
+def clean_assets_files():
+    media_prefix = default_storage.path('')
+    for basedir, dummy, filenames in os.walk(media_prefix):
+        if not is_asset_dir(basedir):
+            continue
+        for filename in filenames:
+            os.remove('%s/%s' % (basedir, filename))
+
+
+def add_tar_content(tar, filename, content):
+    file = tarfile.TarInfo(filename)
+    fd = BytesIO()
+    fd.write(content.encode('utf-8'))
+    file.size = fd.tell()
+    fd.seek(0)
+    tar.addfile(file, fileobj=fd)
+    fd.close()
+
+
+def untar_assets_files(tar, overwrite=False):
+    media_prefix = default_storage.path('')
+    data = {}
+    for tarinfo in tar.getmembers():
+        filepath = default_storage.path(tarinfo.name)
+        if not overwrite and os.path.exists(filepath):
+            continue
+        if tarinfo.name == '_assets.json':
+            json_assets = tar.extractfile(tarinfo).read()
+            data = json.loads(json_assets.decode('utf-8'))
+        elif tarinfo.name != '_site.json':
+            tar.extract(tarinfo, path=media_prefix)
+    return data
+
+
+def tar_assets_files(tar):
+    media_prefix = default_storage.path('')
+    for basedir, dummy, filenames in os.walk(media_prefix):
+        if not is_asset_dir(basedir):
+            continue
+        for filename in filenames:
+            tar.add(os.path.join(basedir, filename), os.path.join(basedir, filename)[len(media_prefix) :])
+    export = {'assets': Asset.export_all_for_json()}
+    add_tar_content(tar, '_assets.json', json.dumps(export, indent=2))
+
+
+def import_assets(fd, overwrite=False):
+    with tarfile.open(mode='r', fileobj=fd) as tar:
+        data = untar_assets_files(tar, overwrite=overwrite)
+    Asset.load_serialized_objects(data.get('assets') or [])
+
+
+def export_assets(fd):
+    with tarfile.open(mode='w', fileobj=fd) as tar:
+        tar_assets_files(tar)

combo/apps/assets/views.py

@@ -14,30 +14,31 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import tarfile
 import os
-
-from django.conf import settings
-from django.contrib import messages
-from django.core.exceptions import PermissionDenied
-from django.core.files.storage import default_storage
-from django.core.urlresolvers import reverse, reverse_lazy
-from django.http import Http404, HttpResponse
-from django.shortcuts import redirect
-from django.utils.six import BytesIO
-from django.utils.translation import ugettext_lazy as _
-from django.views.generic import TemplateView, ListView, FormView
+import tarfile
+from io import BytesIO
 
 import ckeditor
+from django.conf import settings
+from django.contrib import messages
+from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
+from django.core.files.storage import default_storage
+from django.http import Http404, HttpResponse
+from django.shortcuts import get_object_or_404, redirect
+from django.urls import reverse, reverse_lazy
+from django.utils.translation import gettext_lazy as _
+from django.views.generic import FormView, ListView, TemplateView
 from sorl.thumbnail.shortcuts import get_thumbnail
 
+from combo.apps.assets.utils import export_assets, import_assets
+from combo.apps.maps.models import MapLayer
 from combo.data.models import CellBase
 
-from .forms import AssetUploadForm, AssetsImportForm
+from .forms import AssetsImportForm, AssetUploadForm
 from .models import Asset
 
 
-class CkEditorAsset(object):
+class CkEditorAsset:
     def __init__(self, filepath):
         self.filepath = filepath
         self.name = os.path.basename(filepath)
@@ -58,8 +59,7 @@ class CkEditorAsset(object):
 
     def thumb(self):
         if getattr(settings, 'CKEDITOR_IMAGE_BACKEND', None):
-            thumb = ckeditor.utils.get_media_url(
-                    ckeditor.utils.get_thumb_filename(self.filepath))
+            thumb = ckeditor.utils.get_media_url(ckeditor.utils.get_thumb_filename(self.filepath))
         else:
             thumb = self.src
         return thumb
@@ -68,7 +68,7 @@ class CkEditorAsset(object):
         return ckeditor.views.is_image(self.src)
 
 
-class SlotAsset(object):
+class SlotAsset:
     def __init__(self, key=None, name=None, asset_type='image', asset=None):
         self.key = key
         self.name = name
@@ -80,7 +80,10 @@ class SlotAsset(object):
 
     def size(self):
         if self.asset:
-            return os.stat(self.asset.asset.path).st_size
+            try:
+                return os.stat(self.asset.asset.path).st_size
+            except OSError:
+                pass
         return None
 
     def src(self):
@@ -94,25 +97,32 @@ class SlotAsset(object):
 
     @classmethod
     def get_assets(cls):
-        assets = dict([(x.key, x) for x in Asset.objects.all()])
+        assets = {x.key: x for x in Asset.objects.all()}
         uniq_slots = {}
         uniq_slots.update(settings.COMBO_ASSET_SLOTS)
-        for cell in CellBase.get_cells(
-                cell_filter=lambda x: bool(x.get_asset_slots)):
+        cells = CellBase.get_cells(select_related={'__all__': ['page'], 'data_linkcell': ['link_page']})
+        for cell in cells:
             uniq_slots.update(cell.get_asset_slots())
+        for map_layer in MapLayer.objects.filter(kind='geojson'):
+            uniq_slots.update(map_layer.get_asset_slots())
         for key, value in uniq_slots.items():
-            yield cls(key,
-                      name=value.get('label'),
-                      asset_type=value.get('asset-type', 'image'),
-                      asset=assets.get(key))
+            yield cls(
+                key,
+                name=value.get('label') or '',
+                asset_type=value.get('asset-type', 'image'),
+                asset=assets.get(key),
+            )
 
 
 class Assets(ListView):
     template_name = 'combo/manager_assets.html'
     paginate_by = 10
 
+    def get_files(self):
+        return list(SlotAsset.get_assets()) + CkEditorAsset.get_assets(self.request)
+
     def get_queryset(self):
-        files = list(SlotAsset.get_assets()) + CkEditorAsset.get_assets(self.request)
+        files = self.get_files()
         q = self.request.GET.get('q')
         if q:
             files = [x for x in files if q.lower() in x.name.lower()]
@@ -120,7 +130,7 @@ class Assets(ListView):
         return files
 
     def get_context_data(self, **kwargs):
-        context = super(Assets, self).get_context_data(**kwargs)
+        context = super().get_context_data(**kwargs)
         context['query'] = self.request.GET.get('q') or ''
         return context
 
@@ -134,9 +144,21 @@ class Assets(ListView):
             return url + '?page=%s' % ((i // self.paginate_by) + 1)
         return url
 
+
 assets = Assets.as_view()
 
 
+class AssetsBrowse(Assets):
+    template_name = 'combo/manager_assets_browse.html'
+    paginate_by = 7
+
+    def get_files(self):
+        return CkEditorAsset.get_assets(self.request)
+
+
+browse = AssetsBrowse.as_view()
+
+
 class AssetUpload(FormView):
     form_class = AssetUploadForm
     template_name = 'combo/manager_asset_upload.html'
@@ -145,13 +167,14 @@ class AssetUpload(FormView):
         # use native ckeditor view so it's available from ckeditor file/image
         # dialogs.
         ckeditor_upload_view = ckeditor.views.ImageUploadView()
-        self.request.GET = {'CKEditorFuncNum': '-'} # hack
+        self.request.GET = {'CKEditorFuncNum': '-'}  # hack
         ckeditor_upload_view.post(self.request)
-        return super(AssetUpload, self).form_valid(form)
+        return super().form_valid(form)
 
     def get_success_url(self):
         return Assets(request=self.request).get_anchored_url(name=self.request.FILES['upload'].name)
 
+
 asset_upload = AssetUpload.as_view()
 
 
@@ -163,21 +186,39 @@ class AssetOverwrite(FormView):
     def form_valid(self, form):
         img_orig = self.request.GET['img']
         if '..' in img_orig:
-            raise PermissionDenied() # better safe than sorry
+            raise PermissionDenied()  # better safe than sorry
         base_path = settings.CKEDITOR_UPLOAD_PATH
         if getattr(settings, 'CKEDITOR_RESTRICT_BY_USER', False):
             base_path = os.path.join(base_path, self.request.user.username)
         if not img_orig.startswith(base_path):
             raise PermissionDenied()
+        try:
+            os.stat(default_storage.path(img_orig))
+        except ValueError:
+            raise PermissionDenied()
+        if '\x00' in img_orig:
+            # os.stat should have raised "embedded null byte" but double check
+            raise PermissionDenied()
 
         upload = self.request.FILES['upload']
+
+        # check that the new file and the original have the same extension
+        ext_orig = os.path.splitext(img_orig)[1].lower()
+        ext_upload = os.path.splitext(upload.name)[1].lower()
+        if ext_orig != ext_upload:
+            messages.error(
+                self.request,
+                _('You have to upload a file with the same extension (%(ext)s).') % {'ext': ext_orig},
+            )
+            return super().form_valid(form)
+
         default_storage.delete(img_orig)
         if getattr(settings, 'CKEDITOR_IMAGE_BACKEND', None):
             thumb = ckeditor.utils.get_thumb_filename(img_orig)
             default_storage.delete(thumb)
         saved_path = default_storage.save(img_orig, upload)
         backend = ckeditor.image_processing.get_backend()
-        upload.seek(0) # rewind file to be sure
+        upload.seek(0)  # rewind file to be sure
         try:
             backend.image_verify(upload)
         except ckeditor.utils.NotAnImageException:
@@ -185,12 +226,13 @@ class AssetOverwrite(FormView):
         else:
             if backend.should_create_thumbnail(saved_path):
                 backend.create_thumbnail(saved_path)
-        return super(AssetOverwrite, self).form_valid(form)
+        return super().form_valid(form)
 
     def get_success_url(self):
         img_orig = self.request.GET['img']
         return Assets(request=self.request).get_anchored_url(name=os.path.basename(img_orig))
 
+
 asset_overwrite = AssetOverwrite.as_view()
 
 
@@ -200,20 +242,59 @@ class AssetDelete(TemplateView):
     def post(self, request):
         img_orig = request.GET['img']
         if '..' in img_orig:
-            raise PermissionDenied() # better safe than sorry
+            raise PermissionDenied()  # better safe than sorry
         base_path = settings.CKEDITOR_UPLOAD_PATH
         if getattr(settings, 'CKEDITOR_RESTRICT_BY_USER', False):
             base_path = os.path.join(base_path, request.user.username)
         if not img_orig.startswith(base_path):
             raise PermissionDenied()
+        try:
+            os.stat(default_storage.path(img_orig))
+        except ValueError:
+            raise PermissionDenied()
+        if '\x00' in img_orig:
+            # os.stat should have raised "embedded null byte" but double check
+            raise PermissionDenied()
+
         default_storage.delete(img_orig)
-        return redirect(
-                Assets(request=self.request).get_anchored_url(
-                    name=os.path.basename(img_orig)))
+        return redirect(Assets(request=self.request).get_anchored_url(name=os.path.basename(img_orig)))
+
 
 asset_delete = AssetDelete.as_view()
 
 
+class SlotAssets(ListView):
+    template_name = 'combo/manager_slot_assets.html'
+
+    def get_assets(self, cell):
+        asset_slots = cell.get_asset_slots()
+        assets = {x.key: x for x in Asset.objects.filter(key__in=asset_slots.keys())}
+        for key, value in asset_slots.items():
+            yield SlotAsset(
+                key,
+                name=value.get('short_label'),
+                asset_type=value.get('asset-type', 'image'),
+                asset=assets.get(key),
+            )
+
+    def get_queryset(self):
+        cell_reference = self.kwargs['cell_reference']
+        try:
+            cell = CellBase.get_cell(cell_reference)
+        except ObjectDoesNotExist:
+            raise Http404()
+
+        return self.get_assets(cell)
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+        context['cell_reference'] = self.kwargs['cell_reference']
+        return context
+
+
+slot_assets = SlotAssets.as_view()
+
+
 class SlotAssetUpload(FormView):
     form_class = AssetUploadForm
     template_name = 'combo/manager_asset_upload.html'
@@ -226,11 +307,25 @@ class SlotAssetUpload(FormView):
             self.asset = Asset(key=self.kwargs['key'])
         self.asset.asset = self.request.FILES['upload']
         self.asset.save()
-        return super(SlotAssetUpload, self).form_valid(form)
+        return super().form_valid(form)
 
     def get_success_url(self):
+        if self.request.GET.get('cell_reference'):
+            cell_reference = self.request.GET['cell_reference']
+            try:
+                cell = CellBase.get_cell(cell_reference)
+            except ObjectDoesNotExist:
+                pass
+            else:
+                return (
+                    reverse('combo-manager-page-view', kwargs={'pk': cell.page_id})
+                    + '#cell-'
+                    + cell_reference
+                )
+
         return Assets(request=self.request).get_anchored_url(key=self.kwargs['key'])
 
+
 slot_asset_upload = SlotAssetUpload.as_view()
 
 
@@ -239,8 +334,21 @@ class SlotAssetDelete(TemplateView):
 
     def post(self, request, *args, **kwargs):
         Asset.objects.filter(key=kwargs['key']).delete()
+        if self.request.GET.get('cell_reference'):
+            cell_reference = self.request.GET['cell_reference']
+            try:
+                cell = CellBase.get_cell(cell_reference)
+            except ObjectDoesNotExist:
+                pass
+            else:
+                return redirect(
+                    reverse('combo-manager-page-view', kwargs={'pk': cell.page_id})
+                    + '#cell-'
+                    + cell_reference
+                )
         return redirect(Assets(request=self.request).get_anchored_url(key=kwargs['key']))
 
+
 slot_asset_delete = SlotAssetDelete.as_view()
 
 
@@ -252,38 +360,67 @@ class AssetsImport(FormView):
     def form_valid(self, form):
         overwrite = form.cleaned_data.get('overwrite')
         try:
-            assets = tarfile.open(fileobj=form.cleaned_data['assets_file'])
+            import_assets(form.cleaned_data['assets_file'], overwrite)
         except tarfile.TarError:
             messages.error(self.request, _('The assets file is not valid.'))
-            return super(AssetsImport, self).form_valid(form)
-        media_prefix = default_storage.path('')
-        for tarinfo in assets.getmembers():
-            filepath = default_storage.path(tarinfo.name)
-            if not overwrite and os.path.exists(filepath):
-                continue
-            assets.extract(tarinfo, path=media_prefix)
+            return super().form_valid(form)
         messages.success(self.request, _('The assets file has been imported.'))
-        return super(AssetsImport, self).form_valid(form)
+        return super().form_valid(form)
+
 
 assets_import = AssetsImport.as_view()
 
 
 def assets_export(request, *args, **kwargs):
     fd = BytesIO()
-    assets_file = tarfile.open('assets.tar', 'w', fileobj=fd)
-    media_prefix = default_storage.path('')
-    for basedir, dirnames, filenames in os.walk(media_prefix):
-        for filename in filenames:
-            assets_file.add(
-                    os.path.join(basedir, filename),
-                    os.path.join(basedir, filename)[len(media_prefix):])
-    assets_file.close()
+    export_assets(fd)
     return HttpResponse(fd.getvalue(), content_type='application/x-tar')
 
 
 def serve_asset(request, key):
-    try:
-        asset = Asset.objects.get(key=key)
-        return redirect(asset.asset.url)
-    except (Asset.DoesNotExist, AttributeError):
+    asset = get_object_or_404(Asset, key=key)
+
+    if not os.path.exists(asset.asset.path):
         raise Http404()
+
+    # get options for thumbnail
+    thumb_options = request.GET.dict()
+    width = thumb_options.pop('width', None)
+    height = thumb_options.pop('height', None)
+
+    geometry_string = ''
+    if width:
+        geometry_string += width
+    if height:
+        geometry_string += 'x%s' % height
+
+    # no thumbnail whithout geometry_string or for a svg file
+    if not geometry_string or asset.asset.name.endswith('svg'):
+        url = asset.asset.url
+    else:
+        # get or create thumbnail
+        url = get_thumbnail(asset.asset, geometry_string, **thumb_options).url
+
+    if settings.COMBO_X_ACCEL_ASSETS and url.startswith(settings.MEDIA_URL):
+        response = HttpResponse(content_type='')  # let nginx set it
+        response['X-Accel-Redirect'] = url
+        return response
+
+    return redirect(url)
+
+
+class AssetsExportSize(TemplateView):
+    template_name = 'combo/manager_assets_export_size.html'
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+        media_prefix = default_storage.path('')
+        computed_size = 0
+        for basedir, dummy, filenames in os.walk(media_prefix):
+            for filename in filenames:
+                computed_size += os.stat(os.path.join(basedir, filename)).st_size
+        context['size'] = computed_size
+        return context
+
+
+assets_export_size = AssetsExportSize.as_view()

combo/apps/calendar/forms.py

@@ -1,79 +0,0 @@
-# combo - content management system
-# Copyright (C) 2017  Entr'ouvert
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-from django import forms
-from django.utils.translation import ugettext_lazy as _
-from django.utils.dateparse import parse_datetime, parse_time
-
-from .models import BookingCalendar
-from .utils import get_agendas
-from combo.apps.wcs.utils import get_wcs_options
-
-
-class BookingCalendarForm(forms.ModelForm):
-
-    class Meta:
-        model = BookingCalendar
-        fields = (
-            'title', 'agenda_reference', 'formdef_reference',
-            'slot_duration', 'minimal_booking_duration', 'days_displayed')
-
-    def __init__(self, *args, **kwargs):
-        super(BookingCalendarForm, self).__init__(*args, **kwargs)
-        agenda_references = get_agendas()
-        formdef_references = get_wcs_options('/api/formdefs/')
-        self.fields['agenda_reference'].widget = forms.Select(choices=agenda_references)
-        self.fields['formdef_reference'].widget = forms.Select(choices=formdef_references)
-
-
-class BookingForm(forms.Form):
-
-    def __init__(self, *args, **kwargs):
-        self.cell = kwargs.pop('cell')
-        super(BookingForm, self).__init__(*args, **kwargs)
-        self.cleaned_data = {}
-
-    def is_valid(self):
-        slots = getattr(self.data, 'getlist', lambda x: [])('slots')
-        # check that at least one slot if selected
-        if not slots:
-            raise ValueError(_('Please select slots'))
-        offset = self.cell.slot_duration
-        start_dt = parse_datetime(slots[0])
-        end_dt = parse_datetime(slots[-1]) + offset
-        slots.append(end_dt.isoformat())
-
-        # check that all slots are part of the same day
-        for slot in slots:
-            if parse_datetime(slot).date() != start_dt.date():
-                raise ValueError(_('Please select slots of the same day'))
-
-        # check that slots datetime are contiguous
-        start = start_dt
-        while start <= end_dt:
-            if start.isoformat() not in slots:
-                raise ValueError(_('Please select contiguous slots'))
-            start = start + offset
-
-        # check that event booking duration >= minimal booking duration
-        min_duration = self.cell.minimal_booking_duration
-        if not (end_dt - start_dt) >= min_duration:
-            str_min_duration = parse_time(str(min_duration)).strftime('%H:%M')
-            message = _("Minimal booking duration is %s") % str_min_duration
-            raise ValueError(message)
-        self.cleaned_data['start'] = start_dt
-        self.cleaned_data['end'] = end_dt
-        return True

combo/apps/calendar/migrations/0001_initial.py

@@ -1,39 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-import datetime
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('data', '0027_page_picture'),
-        ('auth', '0006_require_contenttypes_0002'),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='BookingCalendar',
-            fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('placeholder', models.CharField(max_length=20)),
-                ('order', models.PositiveIntegerField()),
-                ('slug', models.SlugField(verbose_name='Slug', blank=True)),
-                ('extra_css_class', models.CharField(max_length=100, verbose_name='Extra classes for CSS styling', blank=True)),
-                ('public', models.BooleanField(default=True, verbose_name='Public')),
-                ('restricted_to_unlogged', models.BooleanField(default=False, verbose_name='Restrict to unlogged users')),
-                ('last_update_timestamp', models.DateTimeField(auto_now=True)),
-                ('title', models.CharField(max_length=128, null=True, verbose_name='Title', blank=True)),
-                ('agenda_reference', models.CharField(max_length=128, verbose_name='Agenda')),
-                ('formdef_reference', models.CharField(max_length=128, verbose_name='Form')),
-                ('slot_duration', models.DurationField(default=datetime.timedelta(0, 1800), help_text='Format is hours:minutes:seconds', verbose_name='Slot duration')),
-                ('minimal_booking_duration', models.DurationField(default=datetime.timedelta(0, 3600), help_text='Format is hours:minutes:seconds', verbose_name='Minimal booking duration')),
-                ('groups', models.ManyToManyField(to='auth.Group', verbose_name='Groups', blank=True)),
-                ('page', models.ForeignKey(to='data.Page')),
-            ],
-            options={
-                'verbose_name': 'Booking Calendar',
-            },
-        ),
-    ]

combo/apps/calendar/migrations/0002_bookingcalendar_days_displayed.py

@@ -1,19 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('calendar', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='bookingcalendar',
-            name='days_displayed',
-            field=models.PositiveSmallIntegerField(default=7, verbose_name='Number of days to display'),
-        ),
-    ]

combo/apps/calendar/models.py

@@ -1,68 +0,0 @@
-# combo - content management system
-# Copyright (C) 2017  Entr'ouvert
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import datetime
-
-from django.conf import settings
-from django.db import models
-from django.utils.translation import ugettext_lazy as _
-
-from combo.data.models import CellBase
-from combo.data.library import register_cell_class
-
-from .utils import (is_chrono_enabled, is_wcs_enabled,
-                    get_chrono_events, get_calendar_context_vars)
-
-
-@register_cell_class
-class BookingCalendar(CellBase):
-
-    title = models.CharField(_('Title'), max_length=128, blank=True, null=True)
-    agenda_reference = models.CharField(_('Agenda'), max_length=128)
-    formdef_reference = models.CharField(_('Form'), max_length=128)
-    slot_duration = models.DurationField(
-        _('Slot duration'), default=datetime.timedelta(minutes=30),
-        help_text=_('Format is hours:minutes:seconds'))
-    minimal_booking_duration = models.DurationField(
-        _('Minimal booking duration'), default=datetime.timedelta(hours=1),
-        help_text=_('Format is hours:minutes:seconds'))
-    days_displayed = models.PositiveSmallIntegerField(_('Number of days to display'), default=7)
-
-    template_name = 'calendar/booking_calendar_cell.html'
-
-    class Meta:
-        verbose_name = _('Booking Calendar')
-
-    def get_default_form_class(self):
-        from .forms import BookingCalendarForm
-        return BookingCalendarForm
-
-    @classmethod
-    def is_enabled(cls):
-        return settings.BOOKING_CALENDAR_CELL_ENABLED and is_chrono_enabled() and is_wcs_enabled()
-
-    def is_visible(self, user=None):
-        return self.agenda_reference and self.formdef_reference \
-            and super(BookingCalendar, self).is_visible(user=user)
-
-    def get_cell_extra_context(self, context):
-        if context.get('placeholder_search_mode'):
-            return {}
-        extra_context = super(BookingCalendar, self).get_cell_extra_context(context)
-        events_data = get_chrono_events(self.agenda_reference, not(context.get('synchronous')))
-        extra_context.update(get_calendar_context_vars(
-                             context['request'], extra_context['cell'], events_data))
-        return extra_context

combo/apps/calendar/templates/calendar/booking_calendar_cell.html

@@ -1,27 +0,0 @@
-{% load i18n calendar %}
-{% block cell-content %}
-
-{% if cell.title %}
-<h2>
-    <span>{{cell.title}}</span>
-    {% if calendar %}
-    <span class="calinfo">
-        {% with calendar.get_first_available_slot as slot %}
-        ({% if slot %}{% trans "Next available slot:" %} {{ slot.date_time|date:"DATETIME_FORMAT"}}{% else %}{% trans "No available slots." %}{% endif %})
-        {% endwith %}
-    </span>
-    {% endif %}
-</h2>
-{% endif %}
-
-{% if error %}
-    <div><p>{{ error }}</p></div>
-{% else %}
-    <div class="calcontent">
-    {% include 'calendar/booking_calendar_content.html' %}
-    </div>
-{% endif %}
-
-
-<style>.calinfo { font-style: italic; font-size: 80%; }</style>
-{% endblock %}

combo/apps/calendar/templates/calendar/booking_calendar_content.html

@@ -1,57 +0,0 @@
-{% load i18n calendar %}
-
-{% if calendar_days.has_other_pages %}
-<p class="paginator">
-
-{% if calendar_days.has_previous %}
-<a class="previous calchunk" href="?chunk_{{ cell.pk }}={{ calendar_days.previous_page_number }}" data-content-url="{% url 'ajax-calendar-content' pk=cell.pk %}?chunk_{{cell.pk}}={{ calendar_days.previous_page_number }}">{% trans "previous" %}</a>
-{% else %}
-<span class="previous">{% trans "previous" %}</span>
-{% endif %}
-<span class="current">
-    {{ calendar_days.number }} / {{ calendar_days.paginator.num_pages }}
-</span>
-{% if calendar_days.has_next %}
-<a class="next calchunk" href="?chunk_{{ cell.pk }}={{ calendar_days.next_page_number }}" data-content-url="{% url 'ajax-calendar-content' pk=cell.pk %}?chunk_{{cell.pk}}={{ calendar_days.next_page_number }}">{% trans "next" %}</a>
-{% else %}
-<span class="next">{% trans "next" %}</span>
-{% endif %}
-</p>
-{% endif %}
-
-{% if calendar_days %}
-<form method="POST" action="{% url 'calendar-booking' pk=cell.pk %}?chunk_{{cell.pk}}={{calendar_days.number}}">
-    {% csrf_token %}
-    <table id="cal-table-{{cell.pk}}">
-        <thead>
-            <tr>
-                <th></th>
-                {% for day in calendar_days %}
-                <th>{{day|date:"SHORT_DATE_FORMAT"}}</th>
-                {% endfor %}
-            </tr>
-        </thead>
-        <tbody>
-            {% for slot in calendar_slots %}
-            <tr>
-                <th>{{slot|date:"TIME_FORMAT"}}</th>
-                {% for day in calendar_days %}
-                {% get_day_slot calendar day=day slot=slot as value %}
-                {% if not value.exist %}
-                <td class="absent"></td>
-                {% elif value.available %}
-                <td class="available">
-                    <input type="checkbox" name="slots" value="{{value.label}}" id="slot-{{cell.pk}}-{{value.label}}"/>
-                    <label for="slot-{{cell.pk}}-{{value.label}}"></label>
-                </td>
-                {% else %}
-                <td class="unavailable"></td>
-                {% endif %}
-                {% endfor %}
-            </tr>
-            {% endfor %}
-        </tbody>
-    </table>
-    <button class="submit-button">{% trans "Book" context "booking" %}</button>
-</form>
-{% endif %}

combo/apps/calendar/urls.py

@@ -1,24 +0,0 @@
-# combo - content management system
-# Copyright (C) 2017  Entr'ouvert
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-from django.conf.urls import url
-
-from .views import BookingView, CalendarContentAjaxView
-
-urlpatterns = [
-    url(r'^calendar/book/(?P<pk>[\w,-]+)/$', BookingView.as_view(), name='calendar-booking'),
-    url(r'^ajax/calendar/content/(?P<pk>\w+)/$', CalendarContentAjaxView.as_view(), name='ajax-calendar-content'),
-]

combo/apps/calendar/utils.py

@@ -1,240 +0,0 @@
-# combo - content management system
-# Copyright (C) 2017 Entr'ouvert
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import datetime
-import math
-
-from django.conf import settings
-from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
-from django.utils.dateparse import parse_datetime
-from django.utils.http import urlencode
-from django.utils.timezone import localtime, make_aware
-from django.utils.translation import ugettext_lazy as _
-
-from combo.utils import requests
-
-
-def get_services(service_name):
-    if hasattr(settings, 'KNOWN_SERVICES') and settings.KNOWN_SERVICES.get(service_name):
-        return settings.KNOWN_SERVICES[service_name]
-    return {}
-
-
-def get_wcs_services():
-    return get_services('wcs')
-
-
-def get_chrono_service():
-    for chrono_key, chrono_site in get_services('chrono').items():
-        if not chrono_site.get('secondary', True):
-            chrono_site['slug'] = chrono_key
-            return chrono_site
-    return {}
-
-
-def is_chrono_enabled():
-    return bool(get_chrono_service())
-
-
-def is_wcs_enabled():
-    return bool(get_wcs_services())
-
-
-def get_agendas():
-    chrono = get_chrono_service()
-    references = []
-    response = requests.get('api/agenda/', remote_service=chrono, without_user=True)
-    try:
-        result = response.json()
-    except ValueError:
-        return references
-    for agenda in result.get('data'):
-        references.append((
-            '%s:%s' % (chrono['slug'], agenda['id']), agenda['text']))
-    return references
-
-
-def get_chrono_events(agenda_reference, synchronous):
-    chrono_key, chrono_slug = agenda_reference.split(':')
-    chrono = get_chrono_service()
-    response = requests.get('api/agenda/%s/datetimes/' % chrono_slug, remote_service=chrono,
-                            without_user=True, raise_if_not_cached=synchronous)
-    try:
-        if response.status_code != 200:
-            raise ValueError
-        result = response.json()
-    except ValueError:
-        return {'error': _('An error occurred while retrieving calendar\'s availabilities.')}
-    return result
-
-
-def get_calendar_context_vars(request, cell, events_data):
-    page = request.GET.get('chunk_%s' % cell.pk, 1)
-    if 'error' in events_data:
-        return events_data
-    events = events_data['data']
-    calendar = get_calendar(events, cell.slot_duration, cell.days_displayed,
-                            cell.minimal_booking_duration)
-    paginator = Paginator(calendar.get_computed_days(), cell.days_displayed)
-    try:
-        cal_page = paginator.page(page)
-    except PageNotAnInteger:
-        cal_page = paginator.page(1)
-    except (EmptyPage,):
-        cal_page = paginator.page(paginator.num_pages)
-    return {
-        'calendar': calendar,
-        'calendar_days': cal_page,
-        'calendar_slots': calendar.get_slots()
-    }
-
-
-def get_calendar(events, offset, days_displayed, min_duration):
-    calendar = Calendar(offset, days_displayed, min_duration)
-
-    for event in events:
-        event_datetime = parse_datetime(event['datetime'])
-        if not calendar.has_day(event_datetime.date()):
-            day = WeekDay(event_datetime.date())
-            calendar.days.append(day)
-        else:
-            day = calendar.get_day(event_datetime.date())
-        # add slots to day
-        day.add_slots(DaySlot(
-            event_datetime, True if not event.get('disabled', True) else False))
-
-    return calendar
-
-
-def get_form_url_with_params(cell, data):
-    session_vars = {
-        "session_var_booking_agenda_slug": cell.agenda_reference.split(':')[1],
-        "session_var_booking_start": data['start'].isoformat(),
-        "session_var_booking_end": data['end'].isoformat()
-    }
-    wcs_key, wcs_slug = cell.formdef_reference.split(':')
-    wcs = get_wcs_services().get(wcs_key)
-    url = '%s%s/?%s' % (wcs['url'], wcs_slug, urlencode(session_vars))
-    return url
-
-
-class DaySlot(object):
-
-    def __init__(self, date_time, available, exist=True):
-        self.date_time = localtime(make_aware(date_time))
-        self.available = available
-        self.exist = exist
-
-    def __repr__(self):
-        return '<DaySlot date_time=%s - available=%s>' % (self.date_time.isoformat(), self.available)
-
-    @property
-    def label(self):
-        return '%s' % self.date_time.isoformat()
-
-
-class WeekDay(object):
-
-    def __init__(self, date):
-        self.date = date
-        self.slots = []
-
-    def __repr__(self):
-        return '<WeekDay %s >' % self.date.isoformat()
-
-    def add_slots(self, slot):
-        if slot not in self.slots:
-            self.slots.append(slot)
-
-    def get_slot(self, slot_time):
-        for slot in self.slots:
-            if slot.date_time.time() == slot_time:
-                return slot
-        slot_datetime = datetime.datetime.combine(self.date, slot_time)
-        return DaySlot(slot_datetime, False, exist=False)
-
-    def get_minimum_slot(self):
-        return min(self.slots, key=lambda x: x.date_time.time())
-
-    def get_maximum_slot(self):
-        return max(self.slots, key=lambda x: x.date_time.time())
-
-
-class Calendar(object):
-
-    def __init__(self, offset, days_displayed, min_duration):
-        self.offset = offset
-        self.days_displayed = days_displayed
-        self.days = []
-        self.min_duration = min_duration
-
-    def __repr__(self):
-        return '<Calendar>'
-
-    def get_first_available_slot(self):
-        """return the first available slot that has enough
-        consecutive available slots to be allowed for booking
-        """
-        required_contiguous_slots = self.min_duration.seconds // self.offset.seconds
-        for day in self.days:
-            slots = day.slots
-            for idx in range(len(slots) - required_contiguous_slots):
-                if all([x.available for x in slots[idx:idx+required_contiguous_slots]]):
-                    return slots[idx]
-        return None
-
-    def get_slots(self):
-        start = self.get_minimum_slot()
-        end = self.get_maximum_slot()
-        while start <= end:
-            yield start
-            start = datetime.datetime.combine(
-                datetime.date.today(), start) + self.offset
-            start = start.time()
-
-    def get_computed_days(self):
-        if not self.days:
-            return []
-        computed_days = []
-        base_day = self.days[0].date
-        days_diff = (self.days[-1].date - self.days[0].date).days
-        # find a number which ensures calendar days are equally chunked
-        days_range = int(self.days_displayed * math.ceil(float(days_diff + 1) / self.days_displayed))
-        for index in range(days_range):
-            day = base_day + datetime.timedelta(days=index)
-            computed_days.append(day)
-        return computed_days
-
-    def get_day(self, date):
-        for day in self.days:
-            if day.date == date:
-                return day
-        return None
-
-    def has_day(self, date):
-        return bool(self.get_day(date))
-
-    def get_availability(self, slot):
-        if not self.has_day(slot.date()):
-            return DaySlot(slot, False, exist=False)
-        day = self.get_day(slot.date())
-        return day.get_slot(slot.time())
-
-    def get_minimum_slot(self):
-        return min([day.get_minimum_slot().date_time.time() for day in self.days])
-
-    def get_maximum_slot(self):
-        return max([day.get_maximum_slot().date_time.time() for day in self.days])

combo/apps/calendar/views.py

@@ -1,58 +0,0 @@
-# combo - content management system
-# Copyright (C) 2017  Entr'ouvert
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-from django.contrib import messages
-from django.http import HttpResponseRedirect
-from django.utils.encoding import force_text
-from django.views.generic import View, DetailView
-from django.views.generic.detail import SingleObjectMixin
-
-from .forms import BookingForm
-from .models import BookingCalendar
-from .utils import (get_form_url_with_params, get_chrono_events,
-                    get_calendar_context_vars)
-
-
-class BookingView(SingleObjectMixin, View):
-
-    http_method_names = ['post']
-    model = BookingCalendar
-
-    def post(self, request, *args, **kwargs):
-        cell = self.get_object()
-        form = BookingForm(request.POST, cell=cell)
-        try:
-            form.is_valid()
-        except ValueError as exc:
-            messages.error(request, force_text(exc))
-            redirect_url = '%s?%s' % (
-                cell.page.get_online_url(), request.GET.urlencode())
-            return HttpResponseRedirect(redirect_url)
-        data = form.cleaned_data
-        url = get_form_url_with_params(cell, data)
-        return HttpResponseRedirect(url)
-
-
-class CalendarContentAjaxView(DetailView):
-    model = BookingCalendar
-    template_name = 'calendar/booking_calendar_content.html'
-
-    def get_context_data(self, **kwargs):
-        context = super(CalendarContentAjaxView, self).get_context_data(**kwargs)
-        context['cell'] = self.object
-        events_data = get_chrono_events(self.object.agenda_reference, context.get('synchronous'))
-        context.update(get_calendar_context_vars(self.request, self.object, events_data))
-        return context

combo/apps/dashboard/__init__.py

@@ -1,35 +0,0 @@
-# combo - content management system
-# Copyright (C) 2014-2017  Entr'ouvert
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-import django.apps
-from django.utils.timezone import now, timedelta
-from django.utils.translation import ugettext_lazy as _
-
-class AppConfig(django.apps.AppConfig):
-    name = 'combo.apps.dashboard'
-    verbose_name = _('Dashboard')
-
-    def get_before_urls(self):
-        from . import urls
-        return urls.urlpatterns
-
-    def hourly(self):
-        self.clean_autotiles()
-
-    def clean_autotiles(self):
-        from combo.data.models import ConfigJsonCell
-        ConfigJsonCell.objects.filter(placeholder='_auto_tile',
-                last_update_timestamp__lte=now() - timedelta(days=2)).delete()
-
-
-default_app_config = 'combo.apps.dashboard.AppConfig'

combo/apps/dashboard/apps.py

@@ -0,0 +1,36 @@
+# combo - content management system
+# Copyright (C) 2014-2017  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+import django.apps
+from django.utils.timezone import now, timedelta
+from django.utils.translation import gettext_lazy as _
+
+
+class AppConfig(django.apps.AppConfig):
+    name = 'combo.apps.dashboard'
+    verbose_name = _('Dashboard')
+
+    def get_before_urls(self):
+        from . import urls
+
+        return urls.urlpatterns
+
+    def hourly(self):
+        self.clean_autotiles()
+
+    def clean_autotiles(self):
+        from combo.data.models import ConfigJsonCell
+
+        ConfigJsonCell.objects.filter(
+            placeholder='_auto_tile', last_update_timestamp__lte=now() - timedelta(days=2)
+        ).delete()

combo/apps/dashboard/migrations/0001_initial.py

@@ -1,12 +1,8 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import migrations, models
 from django.conf import settings
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('contenttypes', '0002_remove_content_type_name'),
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
@@ -18,16 +14,27 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='DashboardCell',
             fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                (
+                    'id',
+                    models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True),
+                ),
                 ('placeholder', models.CharField(max_length=20)),
                 ('order', models.PositiveIntegerField()),
                 ('slug', models.SlugField(verbose_name='Slug', blank=True)),
-                ('extra_css_class', models.CharField(max_length=100, verbose_name='Extra classes for CSS styling', blank=True)),
+                (
+                    'extra_css_class',
+                    models.CharField(
+                        max_length=100, verbose_name='Extra classes for CSS styling', blank=True
+                    ),
+                ),
                 ('public', models.BooleanField(default=True, verbose_name='Public')),
-                ('restricted_to_unlogged', models.BooleanField(default=False, verbose_name='Restrict to unlogged users')),
+                (
+                    'restricted_to_unlogged',
+                    models.BooleanField(default=False, verbose_name='Restrict to unlogged users'),
+                ),
                 ('last_update_timestamp', models.DateTimeField(auto_now=True)),
-                ('groups', models.ManyToManyField(to='auth.Group', verbose_name='Groups', blank=True)),
-                ('page', models.ForeignKey(to='data.Page')),
+                ('groups', models.ManyToManyField(to='auth.Group', verbose_name='Roles', blank=True)),
+                ('page', models.ForeignKey(to='data.Page', on_delete=models.CASCADE)),
             ],
             options={
                 'verbose_name': 'Dashboard',
@@ -36,12 +43,15 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='Tile',
             fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                (
+                    'id',
+                    models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True),
+                ),
                 ('cell_pk', models.PositiveIntegerField()),
                 ('order', models.PositiveIntegerField()),
-                ('cell_type', models.ForeignKey(to='contenttypes.ContentType')),
-                ('dashboard', models.ForeignKey(to='dashboard.DashboardCell')),
-                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
+                ('cell_type', models.ForeignKey(to='contenttypes.ContentType', on_delete=models.CASCADE)),
+                ('dashboard', models.ForeignKey(to='dashboard.DashboardCell', on_delete=models.CASCADE)),
+                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
             ],
             options={
                 'ordering': ('order',),

combo/apps/dashboard/migrations/0002_auto_20180105_0846.py

@@ -1,11 +1,7 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('dashboard', '0001_initial'),
     ]

combo/apps/dashboard/migrations/0003_dashboardcell_template_name.py

@@ -0,0 +1,17 @@
+# Generated by Django 2.2.21 on 2021-07-23 11:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dashboard', '0002_auto_20180105_0846'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='dashboardcell',
+            name='template_name',
+            field=models.CharField(blank=True, max_length=50, null=True, verbose_name='Cell Template'),
+        ),
+    ]

combo/apps/dashboard/migrations/0004_display_condition.py

@@ -0,0 +1,15 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dashboard', '0003_dashboardcell_template_name'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='dashboardcell',
+            name='condition',
+            field=models.CharField(blank=True, max_length=1000, null=True, verbose_name='Display condition'),
+        ),
+    ]

combo/apps/dashboard/migrations/0005_increase_extra_css_class.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.2.16 on 2024-01-09 09:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dashboard', '0004_display_condition'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='dashboardcell',
+            name='extra_css_class',
+            field=models.CharField(blank=True, max_length=500, verbose_name='Extra classes for CSS styling'),
+        ),
+    ]

combo/apps/dashboard/models.py

@@ -14,14 +14,17 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from django.conf import settings
-from django.contrib.contenttypes.models import ContentType
-from django.contrib.contenttypes import fields
-from django.db import models
-from django.utils.translation import ugettext_lazy as _
+import datetime
+
+from django.conf import settings
+from django.contrib.contenttypes import fields
+from django.contrib.contenttypes.models import ContentType
+from django.db import models
+from django.utils.timezone import now
+from django.utils.translation import gettext_lazy as _
 
-from combo.data.models import CellBase
 from combo.data.library import register_cell_class
+from combo.data.models import CellBase, ValidityInfo
 
 
 @register_cell_class
@@ -40,21 +43,26 @@ class DashboardCell(CellBase):
         return settings.COMBO_DASHBOARD_ENABLED
 
     def is_relevant(self, context):
-        if not (getattr(context['request'], 'user', None) and context['request'].user.is_authenticated()):
+        if not (getattr(context['request'], 'user', None) and context['request'].user.is_authenticated):
             return False
         return True
 
     def render(self, context):
-        context['tiles'] = Tile.objects.filter(dashboard=self, user=context['user'])
-        return super(DashboardCell, self).render(context)
+        tiles = Tile.objects.filter(dashboard=self, user=context['user'])
+        validity_info_dict = {
+            (x.content_type_id, x.object_id): True
+            for x in ValidityInfo.objects.filter(invalid_since__lt=now() - datetime.timedelta(days=2))
+        }
+        context['tiles'] = [x for x in tiles if (x.cell_type_id, x.cell_pk) not in validity_info_dict]
+        return super().render(context)
 
 
 class Tile(models.Model):
-    dashboard = models.ForeignKey(DashboardCell)
-    cell_type = models.ForeignKey(ContentType)
+    dashboard = models.ForeignKey(DashboardCell, on_delete=models.CASCADE)
+    cell_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
     cell_pk = models.PositiveIntegerField()
     cell = fields.GenericForeignKey('cell_type', 'cell_pk')
-    user = models.ForeignKey(settings.AUTH_USER_MODEL)
+    user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
     order = models.IntegerField()
 
     class Meta:

combo/apps/dashboard/templates/combo/dashboard_cell_icons.html

@@ -1,18 +1,18 @@
 {% load i18n dashboard %}
 {% block content %}
-{% if user.is_authenticated %}
-<span class="dashboard-cell-icons">
-{% if not in_dashboard %}
-  {% with tile=cell|as_dashboard_cell:request.user %}
-    {% if tile %}
-    <a class="remove-from-dashboard" href="{% url 'combo-dashboard-remove-tile' cell_reference=tile.cell.get_reference %}"></a>
-    {% else %}
-    <a class="add-to-dashboard" href="{% url 'combo-dashboard-add-tile' cell_reference=cell.get_reference %}"></a>
-    {% endif %}
-  {% endwith %}
-{% else %}
-  <a class="remove-from-dashboard" href="{% url 'combo-dashboard-remove-tile' cell_reference=cell.get_reference %}"></a>
-{% endif %}
-</span>
-{% endif %}
+  {% if user.is_authenticated %}
+    <span class="dashboard-cell-icons">
+      {% if not in_dashboard %}
+        {% with tile=cell|as_dashboard_cell:request.user %}
+          {% if tile %}
+            <a class="remove-from-dashboard" href="{% url 'combo-dashboard-remove-tile' cell_reference=tile.cell.get_reference %}"></a>
+          {% else %}
+            <a class="add-to-dashboard" href="{% url 'combo-dashboard-add-tile' cell_reference=cell.get_reference %}"></a>
+          {% endif %}
+        {% endwith %}
+      {% else %}
+        <a class="remove-from-dashboard" href="{% url 'combo-dashboard-remove-tile' cell_reference=cell.get_reference %}"></a>
+      {% endif %}
+    </span>
+  {% endif %}
 {% endblock %}

combo/apps/dashboard/templates/combo/dashboardcell.html

@@ -1,13 +1,14 @@
-{% load combo i18n %}
+{% load i18n %}
 {% block cell-content %}
-{% for tile in tiles %}
-{% with cell=tile.cell %}
-<div class="cell {{ cell.css_class_names }} {% if cell.slug %}{{cell.slug}}{% endif %}"
-     data-ajax-cell-url="{{ site_base }}{% url 'combo-public-ajax-page-cell' page_pk=cell.page.id cell_reference=cell.get_reference %}"
-     data-ajax-cell-loading-message="{{ cell.loading_message }}"
-     {% if cell.ajax_refresh %}
-     data-ajax-cell-refresh="{{ cell.ajax_refresh }}"
-     {% endif %}><div>{% render_cell cell %}</div></div>
-{% endwith %}
-{% endfor %}
+  {% for tile in tiles %}
+    {% with cell=tile.cell %}
+      <div class="cell {{ cell.css_class_names }} {% if cell.slug %}{{cell.slug}}{% endif %}"
+           data-ajax-cell-url="{{ site_base }}{{ cell.get_ajax_url }}"
+           data-ajax-cell-loading-message="{{ cell.loading_message }}"
+           data-ajax-cell-error-message="{% trans "Loading error" %}"
+           {% if cell.ajax_refresh %}
+             data-ajax-cell-refresh="{{ cell.ajax_refresh }}"
+           {% endif %}><div>{% render_cell cell %}</div></div>
+    {% endwith %}
+  {% endfor %}
 {% endblock %}

combo/apps/dashboard/templatetags/dashboard.py

@@ -21,17 +21,26 @@ from ..models import Tile
 
 register = template.Library()
 
+
 def get_cell_data(cell):
     # return a dictionary with cell parameters relevant for tile comparison
     if cell is None:
         return {}
     cell_data = serializers.serialize('python', [cell])[0]
     del cell_data['pk']
-    for key in ('restricted_to_unlogged', 'groups', 'last_update_timestamp',
-            'order', 'placeholder', 'public', 'page'):
+    for key in (
+        'restricted_to_unlogged',
+        'groups',
+        'last_update_timestamp',
+        'order',
+        'placeholder',
+        'public',
+        'page',
+    ):
         del cell_data['fields'][key]
     return cell_data
 
+
 @register.filter
 def as_dashboard_cell(cell, user):
     cell_data = get_cell_data(cell)

combo/apps/dashboard/urls.py

@@ -14,21 +14,34 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from django.conf.urls import url
+from django.urls import path, re_path
 
 from . import views
 
 urlpatterns = [
-    url(r'^api/dashboard/add/(?P<cell_reference>[\w_-]+)/$',
+    re_path(
+        r'^api/dashboard/add/(?P<cell_reference>[\w_-]+)/$',
         views.dashboard_add_tile,
-        name='combo-dashboard-add-tile'),
-    url(r'^api/dashboard/remove/(?P<cell_reference>[\w_-]+)/$',
+        name='combo-dashboard-add-tile',
+    ),
+    re_path(
+        r'^api/dashboard/remove/(?P<cell_reference>[\w_-]+)/$',
         views.dashboard_remove_tile,
-        name='combo-dashboard-remove-tile'),
-    url(r'^api/dashboard/auto-tile/(?P<key>[\w_-]+)/$',
+        name='combo-dashboard-remove-tile',
+    ),
+    re_path(
+        r'^api/dashboard/auto-tile/(?P<key>[\w_-]+)/$',
         views.dashboard_auto_tile,
-        name='combo-dashboard-auto-tile'),
-    url(r'^api/dashboard/reorder/(?P<dashboard_id>[\w]+)/$',
+        name='combo-dashboard-auto-tile',
+    ),
+    re_path(
+        r'^api/dashboard/reorder/(?P<dashboard_id>[\w]+)/$',
         views.dashboard_reorder_tiles,
-        name='combo-dashboard-reorder-tiles'),
+        name='combo-dashboard-reorder-tiles',
+    ),
+    path(
+        'api/dashboard/tile-stats/',
+        views.dashboard_tile_stats,
+        name='combo-dashboard-tile-stats',
+    ),
 ]

combo/apps/dashboard/views.py

@@ -17,17 +17,27 @@
 import json
 
 from django.conf import settings
-from django.contrib.contenttypes.models import ContentType
+from django.contrib.auth.models import User
 from django.core.exceptions import PermissionDenied
-from django.core.urlresolvers import reverse
 from django.db.models import Max, Min
-from django.http import Http404, HttpResponse, HttpResponseRedirect
+from django.http import (
+    Http404,
+    HttpResponse,
+    HttpResponseBadRequest,
+    HttpResponseNotAllowed,
+    HttpResponseRedirect,
+)
+from django.urls import reverse
+from django.utils.encoding import force_str
 from django.views.decorators.csrf import csrf_exempt
 from django.views.generic import View
+from rest_framework import permissions
+from rest_framework.generics import GenericAPIView
+from rest_framework.response import Response
 
 from combo.data.models import CellBase, ConfigJsonCell
-from combo.data.library import get_cell_class
 from combo.public.views import render_cell
+from combo.utils import is_ajax
 
 from .models import DashboardCell, Tile
 from .templatetags.dashboard import get_cell_data
@@ -36,52 +46,58 @@ from .templatetags.dashboard import get_cell_data
 def dashboard_success(request, dashboard, cell_data):
     dashboard_url = dashboard.page.get_online_url()
 
-    if request.is_ajax():
+    if is_ajax(request):
         return HttpResponse(
-                json.dumps({
-                    'err': 0,
-                    'url': request.build_absolute_uri(dashboard_url),
-                    'cell_data': cell_data}),
-                content_type='application/json')
+            json.dumps({'err': 0, 'url': request.build_absolute_uri(dashboard_url), 'cell_data': cell_data}),
+            content_type='application/json',
+        )
 
     return HttpResponseRedirect(dashboard_url)
 
 
 class DashboardAddTileView(View):
     def get(self, request, *args, **kwargs):
-        if not request.user.is_authenticated():
+        if not request.user.is_authenticated:
             raise PermissionDenied()
 
-        dashboard = DashboardCell.objects.all()[0]
+        dashboard = DashboardCell.objects.filter(page__snapshot__isnull=True).first()
+        if dashboard is None:
+            raise Http404()
         cell = CellBase.get_cell(kwargs['cell_reference'])
         if not cell.page.is_visible(request.user):
             raise PermissionDenied()
-        if not cell.is_visible(request.user):
+        if not cell.is_visible(request):
             raise PermissionDenied()
         cell.pk = None
         cell.page = dashboard.page
         cell.placeholder = '_dashboard'
         cell.save()
 
-        tile = Tile(dashboard=dashboard,
-                cell=cell,
-                user=request.user,
-                order=0)
+        tile = Tile(dashboard=dashboard, cell=cell, user=request.user, order=0)
         if settings.COMBO_DASHBOARD_NEW_TILE_POSITION == 'first':
-            order = Tile.objects.filter(dashboard=dashboard, user=request.user).aggregate(Min('order')).get('order__min')
+            order = (
+                Tile.objects.filter(dashboard=dashboard, user=request.user)
+                .aggregate(Min('order'))
+                .get('order__min')
+            )
             tile.order = order - 1 if order is not None else 0
         elif settings.COMBO_DASHBOARD_NEW_TILE_POSITION == 'last':
-            order = Tile.objects.filter(dashboard=dashboard, user=request.user).aggregate(Max('order')).get('order_max')
+            order = (
+                Tile.objects.filter(dashboard=dashboard, user=request.user)
+                .aggregate(Max('order'))
+                .get('order__max')
+            )
             tile.order = order + 1 if order is not None else 0
         tile.save()
 
         cell_data = get_cell_data(cell)
         cell_data['remove_url'] = reverse(
-                'combo-dashboard-remove-tile',
-                kwargs={'cell_reference': cell.get_reference()})
+            'combo-dashboard-remove-tile', kwargs={'cell_reference': cell.get_reference()}
+        )
 
         return dashboard_success(request, dashboard, cell_data)
 
+
 dashboard_add_tile = DashboardAddTileView.as_view()
 
 
@@ -100,38 +116,55 @@ class DashboardRemoveTileView(View):
 
         # do not remove cell so it can directly be added back
         cell_data['add_url'] = reverse(
-                'combo-dashboard-add-tile',
-                kwargs={'cell_reference': cell.get_reference()})
+            'combo-dashboard-add-tile', kwargs={'cell_reference': cell.get_reference()}
+        )
 
         return dashboard_success(request, dashboard, cell_data)
 
+
 dashboard_remove_tile = DashboardRemoveTileView.as_view()
 
 
 @csrf_exempt
 def dashboard_auto_tile(request, *args, **kwargs):
-    dashboard = DashboardCell.objects.all()[0]
-    cell = ConfigJsonCell(key=kwargs.get('key'), order=1,
-            page_id=dashboard.page_id, placeholder='_auto_tile')
+    if request.method != 'POST':
+        return HttpResponseNotAllowed(['post'])
+
+    try:
+        request_body = json.loads(force_str(request.body))
+    except json.JSONDecodeError:
+        return HttpResponseBadRequest('bad json request: "%s"' % request.body)
+
+    dashboard = DashboardCell.objects.filter(page__snapshot__isnull=True).first()
+    if dashboard is None:
+        raise Http404()
+    cell = ConfigJsonCell(key=kwargs.get('key'), order=1, page_id=dashboard.page_id, placeholder='_auto_tile')
+
+    if cell.key not in settings.JSON_CELL_TYPES:
+        return HttpResponseBadRequest('bad request, invalid cell type: "%s"' % cell.key)
 
     # only keep parameters that are actually defined for this cell type.
-    cell_form_keys = [x['varname'] for x in settings.JSON_CELL_TYPES[cell.key].get('form') or {}]
     cell.parameters = {}
-    request_body = json.loads(request.body)
-    for key in cell_form_keys:
+    for field in settings.JSON_CELL_TYPES[cell.key].get('form') or []:
+        key = field['varname']
         cell.parameters[key] = request_body.get(key)
+        if cell.parameters[key] is None and field.get('required', True):
+            return HttpResponseBadRequest('missing key: %s' % key)
 
     # save cell so it can be reused later, for example to be added to
     # dashboard, or to be used as reference in another page, etc.
     cell.save()
 
-    return render_cell(request, cell=cell)
+    response = render_cell(request, cell=cell)
+    response['x-add-to-dashboard-url'] = reverse(
+        'combo-dashboard-add-tile', kwargs={'cell_reference': cell.get_reference()}
+    )
+    return response
 
 
 def dashboard_reorder_tiles(request, *args, **kwargs):
-    dashboard = DashboardCell.objects.all()[0]
     new_order = request.GET['order'].split(',')
-    tiles = dict((str(x.id), x) for x in Tile.objects.filter(id__in=new_order))
+    tiles = {str(x.id): x for x in Tile.objects.filter(id__in=new_order)}
     for i, tile_id in enumerate(new_order):
         tile = tiles.get(tile_id)
         if tile.user != request.user:
@@ -141,3 +174,63 @@ def dashboard_reorder_tiles(request, *args, **kwargs):
     for tile in tiles.values():
         tile.save()
     return HttpResponse(status=204)
+
+
+class TileStats(GenericAPIView):
+    permission_classes = (permissions.IsAuthenticated,)
+
+    def get(self, request, *args, **kwargs):
+        data = {}
+        data['users'] = {}
+        data['users']['count'] = User.objects.all().count()
+        data['tiles'] = {}
+        tiles_by_user = {}
+        manual_tiles_by_user = {}
+
+        # preload
+        cells = {}
+        for cell in ConfigJsonCell.objects.filter(placeholder__in=['_dashboard', '_suggested_tile']):
+            cells[cell.id] = cell
+
+        for tile in Tile.objects.filter(dashboard__isnull=False):
+            try:
+                cell = cells[tile.cell_pk]  # no db access
+            except KeyError:
+                # likely added after the preload request
+                continue
+
+            if cell.key not in settings.JSON_CELL_TYPES:
+                continue
+
+            if tile.user_id not in tiles_by_user:
+                tiles_by_user[tile.user_id] = []
+            tiles_by_user[tile.user_id].append(cell.key)
+
+            if cell.key not in data['tiles']:
+                data['tiles'][cell.key] = {
+                    'name': settings.JSON_CELL_TYPES[cell.key]['name'],
+                    'count': 0,
+                    'manual': 0,
+                }
+
+            data['tiles'][cell.key]['count'] += 1
+
+            if cell.placeholder != '_suggested_tile':
+                data['tiles'][cell.key]['manual'] += 1
+                if tile.user_id not in manual_tiles_by_user:
+                    manual_tiles_by_user[tile.user_id] = []
+                manual_tiles_by_user[tile.user_id].append(cell.key)
+
+        data['users']['have-tiles'] = len(tiles_by_user.keys())
+        data['users']['have-more-than-suggested-tiles'] = len(manual_tiles_by_user.keys())
+        data['users']['have-no-tiles'] = data['users']['count'] - data['users']['have-tiles']
+        dashboard_lengths = [len(x) for x in tiles_by_user.values()]
+        if dashboard_lengths:
+            dashboard_lengths.sort()
+            data['users']['max-tiles'] = max(dashboard_lengths)
+            data['users']['median-tiles'] = dashboard_lengths[len(dashboard_lengths) // 2]
+
+        return Response(data)
+
+
+dashboard_tile_stats = TileStats.as_view()

combo/apps/dataviz/__init__.py

@@ -1,33 +0,0 @@
-# combo - content management system
-# Copyright (C) 2015  Entr'ouvert
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import re
-
-import django.apps
-from django.core import checks
-from django.conf import settings
-from django.utils.translation import ugettext_lazy as _
-
-
-class AppConfig(django.apps.AppConfig):
-    name = 'combo.apps.dataviz'
-    verbose_name = _('Data Visualisation')
-
-    def get_before_urls(self):
-        from . import urls
-        return urls.urlpatterns
-
-default_app_config = 'combo.apps.dataviz.AppConfig'

combo/apps/dataviz/apps.py

@@ -0,0 +1,36 @@
+# combo - content management system
+# Copyright (C) 2015  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import django.apps
+from django.utils.translation import gettext_lazy as _
+
+
+class AppConfig(django.apps.AppConfig):
+    name = 'combo.apps.dataviz'
+    verbose_name = _('Data Visualisation')
+
+    def get_before_urls(self):
+        from . import urls
+
+        return urls.urlpatterns
+
+    def hourly(self):
+        from .models import ChartNgCell
+        from .utils import update_available_statistics
+
+        update_available_statistics()
+        for cell in ChartNgCell.objects.all():
+            cell.check_validity()

combo/apps/dataviz/fields.py

@@ -0,0 +1,8 @@
+from django import forms
+
+
+class StaticField(forms.Field):
+    widget = forms.HiddenInput
+
+    def bound_data(self, data, initial):
+        return initial

combo/apps/dataviz/forms.py

@@ -14,12 +14,39 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import dataclasses
+import datetime
+from collections import OrderedDict, defaultdict
+
 from django import forms
 from django.conf import settings
+from django.core.cache import cache
+from django.db import transaction
+from django.db.models import Q
+from django.db.models.fields import BLANK_CHOICE_DASH
+from django.template import Context, Template, TemplateSyntaxError, VariableDoesNotExist
+from django.utils.translation import gettext_lazy as _
 
-from combo.utils import requests
+from combo.utils import cache_during_request, requests, spooler
 
-from .models import ChartCell
+from .fields import StaticField
+from .models import ChartCell, ChartFiltersCell, ChartNgCell
+from .widgets import MultipleSelect2Widget, Select2Widget
+
+
+@dataclasses.dataclass
+class Choice:
+    id: str
+    label: str
+    group: str = None
+
+    @staticmethod
+    def get_field_choices(choices):
+        choices_by_group = defaultdict(list)
+        for choice in choices:
+            choices_by_group[choice.group].append((choice.id, choice.label))
+
+        return list(choices_by_group.items())
 
 
 class ChartForm(forms.ModelForm):
@@ -28,12 +55,456 @@ class ChartForm(forms.ModelForm):
         fields = ('title', 'url')
 
     def __init__(self, *args, **kwargs):
-        super(ChartForm, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         available_charts = []
-        for site_key, site_dict in settings.KNOWN_SERVICES.get('bijoe').items():
-            result = requests.get('/visualization/json/',
-                    remote_service=site_dict, without_user=True,
-                    headers={'accept': 'application/json'}).json()
+        for site_dict in (settings.KNOWN_SERVICES.get('bijoe') or {}).values():
+            result = requests.get(
+                '/visualization/json/',
+                remote_service=site_dict,
+                without_user=True,
+                headers={'accept': 'application/json'},
+            ).json()
             available_charts.extend([(x['path'], x['name']) for x in result])
         available_charts.sort(key=lambda x: x[1])
         self.fields['url'].widget = forms.Select(choices=available_charts)
+
+
+@cache_during_request
+def trigger_statistics_list_refresh():
+    transaction.on_commit(spooler.refresh_statistics_list)
+
+
+class ChartFiltersMixin:
+    ajax_choices = True
+    time_intervals = (
+        Choice('week', _('Week')),
+        Choice('month', _('Month')),
+        Choice('year', _('Year')),
+        Choice('weekday', _('Week day')),
+    )
+
+    def get_filter_fields(self, cell):
+        fields = OrderedDict()
+        for filter_ in cell.available_filters:
+            filter_id = filter_['id']
+            if filter_.get('deprecated') and (
+                filter_id not in cell.filter_params
+                or cell.filter_params.get(filter_id) == filter_.get('default')
+            ):
+                continue
+
+            initial = cell.filter_params.get(filter_id, filter_.get('default'))
+            required = filter_.get('required', False)
+
+            if required and {x['id'] for x in filter_['options'] if isinstance(x, dict)} == {'true', 'false'}:
+                field = self.build_boolean_field(cell, filter_, initial)
+            else:
+                field = self.build_choice_field(cell, filter_, initial)
+
+            fields[filter_id] = field
+            if filter_.get('deprecated'):
+                fields[filter_id].label += ' (%s)' % _('deprecated')
+                fields[filter_id].help_text = filter_.get('deprecation_hint')
+            fields[filter_id].is_filter_field = True
+
+        return fields
+
+    @classmethod
+    def get_filter_options(cls, cell, filter_, initial):
+        filter_id = filter_['id']
+
+        filter_options = filter_['options']
+        if not isinstance(filter_options[0], list):
+            # no option groups, add empty one for consistency
+            filter_options = [(None, filter_options)]
+        choices = [
+            Choice(id=opt['id'], label=opt['label'], group=group)
+            for group, options in filter_options
+            for opt in options
+        ]
+
+        if filter_id == 'time_interval':
+            cls.extend_time_interval_choices(choices)
+
+        required = filter_.get('required', False)
+        multiple = filter_.get('multiple')
+        if not required and not multiple:
+            choices.insert(0, Choice(*BLANK_CHOICE_DASH[0]))
+
+        extra_variables = cell.page.get_extra_variables_keys()
+        variable_choices = [
+            Choice(id='variable:' + key, label=key, group=_('Page variables')) for key in extra_variables
+        ]
+
+        for choice in initial if isinstance(initial, list) else [initial]:
+            if not choice:
+                continue
+            if choice.startswith('variable:'):
+                variable = choice.replace('variable:', '')
+                if not variable in extra_variables:
+                    variable_choices.append(
+                        Choice(id=choice, label=_('%s (unavailable)') % variable, group=_('Page variables'))
+                    )
+            elif not any(x.id == choice for x in choices):
+                choices.append(Choice(id=choice, label=_('%s (unavailable)') % choice))
+
+        if variable_choices and not multiple and filter_id != 'time_interval':
+            choices.extend(variable_choices)
+
+        return choices
+
+    def build_choice_field(self, cell, filter_, initial):
+        multiple = filter_.get('multiple')
+        required = filter_.get('required', False)
+        choices = self.get_filter_options(cell, filter_, initial)
+
+        widget_class = MultipleSelect2Widget if multiple else Select2Widget
+        widget = widget_class(cell, filter_['id'], choices, initial, self.ajax_choices)
+
+        field_class = forms.MultipleChoiceField if multiple else forms.ChoiceField
+        field = field_class(
+            label=filter_['label'],
+            choices=Choice.get_field_choices(choices),
+            required=required,
+            initial=initial,
+        )
+        field.widget = widget
+        field.dataviz_choices = choices
+        return field
+
+    def build_boolean_field(self, cell, filter_, initial):
+        return forms.BooleanField(
+            label=filter_['label'],
+            required=False,
+            initial=bool(initial == 'true'),
+        )
+
+    @classmethod
+    def extend_time_interval_choices(cls, choices):
+        if any(choice.id == 'day' for choice in choices):
+            for choice in cls.time_intervals:
+                if choice not in choices:
+                    choices.append(choice)
+
+    def update_time_range_choices(self, statistic, exclude_template_choice=False):
+        choices = self.fields['time_range'].choices
+
+        if not statistic.has_future_data:
+            choices = [choice for choice in choices if not choice[0].startswith('next')]
+
+        if exclude_template_choice:
+            choices = [choice for choice in choices if choice[0] != 'range-template']
+
+        self.fields['time_range'].choices = choices
+
+    def clean(self):
+        for field, value in self.cleaned_data.items():
+            if hasattr(self.fields[field], 'is_filter_field') and isinstance(value, bool):
+                self.cleaned_data[field] = 'true' if value is True else 'false'
+
+
+class ChartNgForm(ChartFiltersMixin, forms.ModelForm):
+    class Meta:
+        model = ChartNgCell
+        fields = (
+            'statistic',
+            'time_range',
+            'time_range_start',
+            'time_range_end',
+            'time_range_start_template',
+            'time_range_end_template',
+            'chart_type',
+            'display_total',
+            'height',
+            'sort_order',
+            'hide_null_values',
+            'print_values',
+        )
+        widgets = {
+            'time_range_start': forms.DateInput(attrs={'type': 'date'}, format='%Y-%m-%d'),
+            'time_range_end': forms.DateInput(attrs={'type': 'date'}, format='%Y-%m-%d'),
+        }
+
+    def __init__(self, *args, **kwargs):
+        trigger_statistics_list_refresh()
+        super().__init__(*args, **kwargs)
+
+        stat_field = self.fields['statistic']
+        if not self.instance.statistic:
+            stat_field.queryset = stat_field.queryset.filter(available=True)
+        else:
+            # display current statistic in choices even if unavailable
+            stat_field.queryset = stat_field.queryset.filter(
+                Q(available=True) | Q(pk=self.instance.statistic.pk)
+            )
+            self.add_filter_fields()
+            self.update_time_range_choices(self.instance.statistic)
+
+        if not self.instance.statistic or self.instance.statistic.service_slug == 'bijoe':
+            for field in (
+                'time_range',
+                'time_range_start',
+                'time_range_end',
+                'time_range_start_template',
+                'time_range_end_template',
+                'display_total',
+            ):
+                del self.fields[field]
+        else:
+            if self.instance.time_range != 'range':
+                del self.fields['time_range_start']
+                del self.fields['time_range_end']
+
+            if self.instance.time_range != 'range-template':
+                del self.fields['time_range_start_template']
+                del self.fields['time_range_end_template']
+
+            if not self.instance.is_table_chart() or self.instance.statistic.data_type:
+                del self.fields['display_total']
+
+    def add_filter_fields(self):
+        new_fields = OrderedDict()
+        for field_name, field in self.fields.items():
+            new_fields[field_name] = field
+            if field_name == 'statistic':
+                # insert filter fields after statistic field
+                new_fields.update(self.get_filter_fields(self.instance))
+        self.fields = new_fields
+
+    def save(self, *args, **kwargs):
+        if 'statistic' in self.changed_data:
+            self.instance.filter_params.clear()
+            self.instance.time_range = ''
+            self.instance.subfilters.clear()
+            for filter_ in self.instance.available_filters:
+                if 'default' in filter_:
+                    self.instance.filter_params[filter_['id']] = filter_['default']
+                elif filter_.get('required'):
+                    options = (
+                        filter_['options'][0][1]
+                        if isinstance(filter_['options'][0], list)
+                        else filter_['options']
+                    )
+                    self.instance.filter_params[filter_['id']] = options[0]['id']
+        else:
+            for filter_ in self.instance.available_filters:
+                if filter_['id'] in self.cleaned_data:
+                    self.instance.filter_params[filter_['id']] = self.cleaned_data[filter_['id']]
+
+        cell = super().save(*args, **kwargs)
+
+        for filter_ in cell.available_filters:
+            if filter_.get('has_subfilters') and filter_['id'] in self.changed_data:
+                cell.update_subfilters()
+                self.fields = OrderedDict(
+                    (name, field)
+                    for name, field in self.fields.items()
+                    if not hasattr(field, 'is_filter_field')
+                )
+                self.add_filter_fields()
+                break
+
+        return cell
+
+    def clean(self):
+        super().clean()
+
+        for template_field in ('time_range_start_template', 'time_range_end_template'):
+            if not self.cleaned_data.get(template_field):
+                continue
+            context = {'now': datetime.datetime.now, 'today': datetime.datetime.now}
+            try:
+                Template('{{ %s|date:"Y-m-d" }}' % self.cleaned_data[template_field]).render(Context(context))
+            except (VariableDoesNotExist, TemplateSyntaxError) as e:
+                self.add_error(template_field, e)
+
+
+class ChartNgPartialForm(ChartFiltersMixin, forms.ModelForm):
+    overridden_filters = forms.CharField()
+
+    prefix = 'filter'
+
+    class Meta:
+        model = ChartNgCell
+        fields = (
+            'time_range',
+            'time_range_start',
+            'time_range_end',
+        )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.fields.update(self.get_filter_fields(self.instance))
+        for field in self.fields.values():
+            field.required = False
+
+    def clean(self):
+        super().clean()
+
+        for field in self._meta.fields:
+            if field not in self.cleaned_data['overridden_filters']:
+                self.cleaned_data[field] = self.initial[field]
+
+        for filter_ in self.instance.available_filters:
+            if filter_['id'] in self.cleaned_data['overridden_filters']:
+                self.instance.filter_params[filter_['id']] = self.cleaned_data.get(filter_['id'])
+
+    def clean_overridden_filters(self):
+        if not self.cleaned_data['overridden_filters']:
+            return []
+
+        return self.cleaned_data['overridden_filters'].split(',')
+
+
+class ChartFiltersForm(ChartFiltersMixin, forms.ModelForm):
+    ajax_choices = False
+    overridden_filters = StaticField()
+
+    prefix = 'filter'
+
+    class Meta:
+        model = ChartNgCell
+        fields = (
+            'time_range',
+            'time_range_start',
+            'time_range_end',
+        )
+        widgets = {
+            'time_range_start': forms.DateInput(attrs={'type': 'date'}, format='%Y-%m-%d'),
+            'time_range_end': forms.DateInput(attrs={'type': 'date'}, format='%Y-%m-%d'),
+        }
+
+    def __init__(self, *args, **kwargs):
+        page = kwargs.pop('page')
+        filters_cell = kwargs.pop('filters_cell')
+        filters_cell_id = kwargs.pop('filters_cell_id', None)
+        super().__init__(*args, **kwargs)
+
+        chart_cells = []
+        for cell in ChartNgCell.objects.filter(page=page, statistic__isnull=False).order_by('order'):
+            cell.page = page  # use cached placeholders
+            if cell.is_placeholder_active(traverse_cells=False):
+                chart_cells.append(cell)
+
+        if not chart_cells:
+            self.fields.clear()
+            return
+
+        if filters_cell_id:
+            for cell in chart_cells:
+                cell.subfilters = cache.get(cell.get_cache_key(filters_cell_id), cell.subfilters)
+
+        first_cell = chart_cells[0]
+        for field in self._meta.fields:
+            self.fields[field].initial = getattr(first_cell, field)
+        dynamic_fields = self.get_filter_fields(first_cell)
+        dynamic_fields_values = {k: v for k, v in first_cell.filter_params.items()}
+
+        if first_cell.time_range == 'range-template':
+            for field in self._meta.fields:
+                self.fields.pop(field, None)
+        else:
+            self.update_time_range_choices(first_cell.statistic, exclude_template_choice=True)
+            if not self.is_bound and first_cell.time_range != 'range':
+                del self.fields['time_range_start']
+                del self.fields['time_range_end']
+
+        for cell in chart_cells[1:]:
+            cell_filter_fields = self.get_filter_fields(cell)
+
+            # keep only common fields
+            dynamic_fields = {k: v for k, v in dynamic_fields.items() if k in cell_filter_fields}
+
+            # keep only same value fields
+            for field, value in cell.filter_params.items():
+                if field in dynamic_fields and value != dynamic_fields_values.get(field, ''):
+                    del dynamic_fields[field]
+
+            if cell.time_range != first_cell.time_range:
+                for field in self._meta.fields:
+                    self.fields.pop(field, None)
+
+            # ensure compatible choices lists
+            for field_name, field in cell_filter_fields.items():
+                if field_name not in dynamic_fields or isinstance(field, forms.BooleanField):
+                    continue
+
+                dynamic_fields[field_name].dataviz_choices = [
+                    x for x in dynamic_fields[field_name].dataviz_choices if x in field.dataviz_choices
+                ]
+                dynamic_fields[field_name].choices = Choice.get_field_choices(
+                    dynamic_fields[field_name].dataviz_choices
+                )
+
+                if dynamic_fields[field_name].choices == []:
+                    del dynamic_fields[field_name]
+
+            if 'time_range' in self.fields:
+                self.update_time_range_choices(cell.statistic)
+
+        self.update_backoffice_filter_choices(filters_cell, dynamic_fields)
+        dynamic_fields = {
+            name: field for name, field in dynamic_fields.items() if filters_cell.filters[name]['enabled']
+        }
+        self.fields.update(dynamic_fields)
+        self.fields['overridden_filters'].initial = ','.join(self.fields)
+
+    @staticmethod
+    def update_backoffice_filter_choices(filters_cell, dynamic_fields):
+        # remove absent filters from cell configuration, except if it was disabled
+        filters_cell.filters = {
+            k: v for k, v in filters_cell.filters.items() if k in dynamic_fields or not v['enabled']
+        }
+
+        # add filters to cell configuration
+        for field_name, field in dynamic_fields.items():
+            if not field_name in filters_cell.filters:
+                filters_cell.filters[field_name] = {'label': str(field.label), 'enabled': True}
+                continue
+
+        filters_cell.save()
+
+    def clean(self):
+        super().clean()
+        if self.cleaned_data.get('time_range') != 'range':
+            self.fields.pop('time_range_start', None)
+            self.fields.pop('time_range_end', None)
+
+
+class ChartFiltersConfigForm(forms.ModelForm):
+    filters = forms.MultipleChoiceField(
+        label=_('Filters'), widget=forms.CheckboxSelectMultiple, required=False
+    )
+
+    class Meta:
+        model = ChartFiltersCell
+        fields = []
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if not self.instance.filters:
+            del self.fields['filters']
+            return
+
+        self.initial['filters'] = []
+        self.fields['filters'].choices = []
+        for filter_id, config in self.instance.filters.items():
+            self.fields['filters'].choices.append((filter_id, config['label']))
+
+            if config['enabled']:
+                self.initial['filters'].append(filter_id)
+
+    def save(self, *args, **kwargs):
+        for filter_id in self.instance.filters:
+            self.instance.filters[filter_id]['enabled'] = bool(filter_id in self.cleaned_data['filters'])
+        return super().save(*args, **kwargs)
+
+
+class ChartNgExportForm(forms.Form):
+    export_format = forms.ChoiceField(
+        label=_('Format'),
+        choices=(
+            ('svg', _('Picture (SVG)')),
+            ('ods', _('Table (ODS)')),
+        ),
+    )

combo/apps/dataviz/migrations/0001_initial.py

@@ -1,11 +1,7 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import models, migrations
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('auth', '0001_initial'),
         ('data', '0012_auto_20151029_1535'),
@@ -15,18 +11,27 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='Gauge',
             fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                (
+                    'id',
+                    models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True),
+                ),
                 ('placeholder', models.CharField(max_length=20)),
                 ('order', models.PositiveIntegerField()),
                 ('slug', models.SlugField(verbose_name='Slug', blank=True)),
                 ('public', models.BooleanField(default=True, verbose_name='Public')),
-                ('restricted_to_unlogged', models.BooleanField(default=False, verbose_name='Restrict to unlogged users')),
+                (
+                    'restricted_to_unlogged',
+                    models.BooleanField(default=False, verbose_name='Restrict to unlogged users'),
+                ),
                 ('title', models.CharField(max_length=150, null=True, verbose_name='Title', blank=True)),
                 ('url', models.CharField(max_length=150, null=True, verbose_name='URL', blank=True)),
-                ('data_source', models.CharField(max_length=150, null=True, verbose_name='Data Source', blank=True)),
+                (
+                    'data_source',
+                    models.CharField(max_length=150, null=True, verbose_name='Data Source', blank=True),
+                ),
                 ('max_value', models.PositiveIntegerField(null=True, verbose_name='Max Value', blank=True)),
-                ('groups', models.ManyToManyField(to='auth.Group', verbose_name='Groups', blank=True)),
-                ('page', models.ForeignKey(to='data.Page')),
+                ('groups', models.ManyToManyField(to='auth.Group', verbose_name='Roles', blank=True)),
+                ('page', models.ForeignKey(to='data.Page', on_delete=models.CASCADE)),
             ],
             options={
                 'verbose_name': 'Gauge',

combo/apps/dataviz/migrations/0002_gauge_jsonp_data_source.py

@@ -1,11 +1,7 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import models, migrations
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('dataviz', '0001_initial'),
     ]

combo/apps/dataviz/migrations/0003_cubesbarchart_cubestable.py

@@ -1,11 +1,7 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import models, migrations
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('auth', '0001_initial'),
         ('data', '0012_auto_20151029_1535'),
@@ -16,21 +12,39 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='CubesBarChart',
             fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                (
+                    'id',
+                    models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True),
+                ),
                 ('placeholder', models.CharField(max_length=20)),
                 ('order', models.PositiveIntegerField()),
                 ('slug', models.SlugField(verbose_name='Slug', blank=True)),
                 ('public', models.BooleanField(default=True, verbose_name='Public')),
-                ('restricted_to_unlogged', models.BooleanField(default=False, verbose_name='Restrict to unlogged users')),
+                (
+                    'restricted_to_unlogged',
+                    models.BooleanField(default=False, verbose_name='Restrict to unlogged users'),
+                ),
                 ('title', models.CharField(max_length=150, null=True, verbose_name='Title', blank=True)),
                 ('url', models.URLField(max_length=150, null=True, verbose_name='URL', blank=True)),
                 ('cube', models.CharField(max_length=256, null=True, verbose_name='Cube', blank=True)),
-                ('aggregate1', models.CharField(max_length=64, null=True, verbose_name='Aggregate', blank=True)),
-                ('drilldown1', models.CharField(max_length=64, null=True, verbose_name='Drilldown 1', blank=True)),
-                ('drilldown2', models.CharField(max_length=64, null=True, verbose_name='Drilldown 2', blank=True)),
-                ('other_parameters', models.TextField(null=True, verbose_name='Other parameters', blank=True)),
-                ('groups', models.ManyToManyField(to='auth.Group', verbose_name='Groups', blank=True)),
-                ('page', models.ForeignKey(to='data.Page')),
+                (
+                    'aggregate1',
+                    models.CharField(max_length=64, null=True, verbose_name='Aggregate', blank=True),
+                ),
+                (
+                    'drilldown1',
+                    models.CharField(max_length=64, null=True, verbose_name='Drilldown 1', blank=True),
+                ),
+                (
+                    'drilldown2',
+                    models.CharField(max_length=64, null=True, verbose_name='Drilldown 2', blank=True),
+                ),
+                (
+                    'other_parameters',
+                    models.TextField(null=True, verbose_name='Other parameters', blank=True),
+                ),
+                ('groups', models.ManyToManyField(to='auth.Group', verbose_name='Roles', blank=True)),
+                ('page', models.ForeignKey(to='data.Page', on_delete=models.CASCADE)),
             ],
             options={
                 'verbose_name': 'Cubes Barchart',
@@ -40,21 +54,39 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='CubesTable',
             fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                (
+                    'id',
+                    models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True),
+                ),
                 ('placeholder', models.CharField(max_length=20)),
                 ('order', models.PositiveIntegerField()),
                 ('slug', models.SlugField(verbose_name='Slug', blank=True)),
                 ('public', models.BooleanField(default=True, verbose_name='Public')),
-                ('restricted_to_unlogged', models.BooleanField(default=False, verbose_name='Restrict to unlogged users')),
+                (
+                    'restricted_to_unlogged',
+                    models.BooleanField(default=False, verbose_name='Restrict to unlogged users'),
+                ),
                 ('title', models.CharField(max_length=150, null=True, verbose_name='Title', blank=True)),
                 ('url', models.URLField(max_length=150, null=True, verbose_name='URL', blank=True)),
                 ('cube', models.CharField(max_length=256, null=True, verbose_name='Cube', blank=True)),
-                ('aggregate1', models.CharField(max_length=64, null=True, verbose_name='Aggregate', blank=True)),
-                ('drilldown1', models.CharField(max_length=64, null=True, verbose_name='Drilldown 1', blank=True)),
-                ('drilldown2', models.CharField(max_length=64, null=True, verbose_name='Drilldown 2', blank=True)),
-                ('other_parameters', models.TextField(null=True, verbose_name='Other parameters', blank=True)),
-                ('groups', models.ManyToManyField(to='auth.Group', verbose_name='Groups', blank=True)),
-                ('page', models.ForeignKey(to='data.Page')),
+                (
+                    'aggregate1',
+                    models.CharField(max_length=64, null=True, verbose_name='Aggregate', blank=True),
+                ),
+                (
+                    'drilldown1',
+                    models.CharField(max_length=64, null=True, verbose_name='Drilldown 1', blank=True),
+                ),
+                (
+                    'drilldown2',
+                    models.CharField(max_length=64, null=True, verbose_name='Drilldown 2', blank=True),
+                ),
+                (
+                    'other_parameters',
+                    models.TextField(null=True, verbose_name='Other parameters', blank=True),
+                ),
+                ('groups', models.ManyToManyField(to='auth.Group', verbose_name='Roles', blank=True)),
+                ('page', models.ForeignKey(to='data.Page', on_delete=models.CASCADE)),
             ],
             options={
                 'verbose_name': 'Cubes Table',

combo/apps/dataviz/migrations/0004_auto_20160108_1048.py

@@ -1,11 +1,7 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import models, migrations
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('dataviz', '0003_cubesbarchart_cubestable'),
     ]

combo/apps/dataviz/migrations/0005_auto_20160928_1152.py

@@ -1,11 +1,7 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('dataviz', '0004_auto_20160108_1048'),
     ]

combo/apps/dataviz/migrations/0006_chartcell.py

@@ -1,11 +1,7 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('data', '0020_auto_20160928_1152'),
         ('dataviz', '0005_auto_20160928_1152'),
@@ -15,17 +11,28 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='ChartCell',
             fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                (
+                    'id',
+                    models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True),
+                ),
                 ('placeholder', models.CharField(max_length=20)),
                 ('order', models.PositiveIntegerField()),
                 ('slug', models.SlugField(verbose_name='Slug', blank=True)),
-                ('extra_css_class', models.CharField(max_length=100, verbose_name='Extra classes for CSS styling', blank=True)),
+                (
+                    'extra_css_class',
+                    models.CharField(
+                        max_length=100, verbose_name='Extra classes for CSS styling', blank=True
+                    ),
+                ),
                 ('public', models.BooleanField(default=True, verbose_name='Public')),
-                ('restricted_to_unlogged', models.BooleanField(default=False, verbose_name='Restrict to unlogged users')),
+                (
+                    'restricted_to_unlogged',
+                    models.BooleanField(default=False, verbose_name='Restrict to unlogged users'),
+                ),
                 ('title', models.CharField(max_length=150, null=True, verbose_name='Title', blank=True)),
                 ('url', models.URLField(max_length=150, null=True, verbose_name='URL', blank=True)),
-                ('groups', models.ManyToManyField(to='auth.Group', verbose_name='Groups', blank=True)),
-                ('page', models.ForeignKey(to='data.Page')),
+                ('groups', models.ManyToManyField(to='auth.Group', verbose_name='Roles', blank=True)),
+                ('page', models.ForeignKey(to='data.Page', on_delete=models.CASCADE)),
             ],
             options={
                 'verbose_name': 'Chart',

combo/apps/dataviz/migrations/0007_auto_20170214_2006.py

@@ -1,13 +1,10 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
+import datetime
 
 from django.db import migrations, models
-import datetime
 from django.utils.timezone import utc
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('dataviz', '0006_chartcell'),
     ]

combo/apps/dataviz/migrations/0008_auto_20190328_0857.py

@@ -1,12 +1,9 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.11.12 on 2019-03-28 07:57
-from __future__ import unicode_literals
 
 from django.db import migrations
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('dataviz', '0007_auto_20170214_2006'),
     ]

combo/apps/dataviz/migrations/0009_auto_20190617_1214.py

@@ -0,0 +1,17 @@
+# Generated by Django 1.11.12 on 2019-06-17 10:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0008_auto_20190328_0857'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='chartcell',
+            name='url',
+            field=models.URLField(blank=True, max_length=250, null=True, verbose_name='URL'),
+        ),
+    ]

combo/apps/dataviz/migrations/0010_auto_20190328_1111.py

@@ -0,0 +1,81 @@
+# Generated by Django 1.11.12 on 2019-03-28 10:11
+
+import django.db.models.deletion
+from django.db import migrations, models
+from django.db.models import JSONField
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('data', '0036_page_sub_slug'),
+        ('dataviz', '0009_auto_20190617_1214'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ChartNgCell',
+            fields=[
+                (
+                    'id',
+                    models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+                ),
+                ('placeholder', models.CharField(max_length=20)),
+                ('order', models.PositiveIntegerField()),
+                ('slug', models.SlugField(blank=True, verbose_name='Slug')),
+                (
+                    'extra_css_class',
+                    models.CharField(
+                        blank=True, max_length=100, verbose_name='Extra classes for CSS styling'
+                    ),
+                ),
+                ('public', models.BooleanField(default=True, verbose_name='Public')),
+                (
+                    'restricted_to_unlogged',
+                    models.BooleanField(default=False, verbose_name='Restrict to unlogged users'),
+                ),
+                ('last_update_timestamp', models.DateTimeField(auto_now=True)),
+                ('data_reference', models.CharField(max_length=150, verbose_name='Data')),
+                ('title', models.CharField(blank=True, max_length=150, verbose_name='Title')),
+                ('cached_json', JSONField(blank=True, default=dict)),
+                (
+                    'chart_type',
+                    models.CharField(
+                        choices=[
+                            (b'bar', 'Bar'),
+                            (b'horizontal-bar', 'Horizontal Bar'),
+                            (b'stacked-bar', 'Stacked Bar'),
+                            (b'line', 'Line'),
+                            (b'pie', 'Pie'),
+                            (b'dot', 'Dot'),
+                            (b'table', 'Table'),
+                        ],
+                        default=b'bar',
+                        max_length=20,
+                        verbose_name='Chart Type',
+                    ),
+                ),
+                (
+                    'height',
+                    models.CharField(
+                        choices=[
+                            (b'150', 'Short (150px)'),
+                            (b'250', 'Average (250px)'),
+                            (b'350', 'Tall (350px)'),
+                        ],
+                        default=b'250',
+                        max_length=20,
+                        verbose_name='Height',
+                    ),
+                ),
+                ('groups', models.ManyToManyField(blank=True, to='auth.Group', verbose_name='Roles')),
+                ('page', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='data.Page')),
+            ],
+            options={
+                'verbose_name': 'Chart',
+            },
+        ),
+        migrations.AlterModelOptions(
+            name='chartcell',
+            options={'verbose_name': 'Chart (legacy)'},
+        ),
+    ]

combo/apps/dataviz/migrations/0011_auto_20200813_1100.py

@@ -0,0 +1,68 @@
+# Generated by Django 1.11.18 on 2020-08-13 09:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0010_auto_20190328_1111'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='chartngcell',
+            name='hide_null_values',
+            field=models.BooleanField(
+                default=False,
+                help_text='This setting only applies for one-dimensional charts.',
+                verbose_name='Hide null values',
+            ),
+        ),
+        migrations.AddField(
+            model_name='chartngcell',
+            name='sort_order',
+            field=models.CharField(
+                choices=[
+                    ('none', 'None'),
+                    ('alpha', 'Alphabetically'),
+                    ('asc', 'Increasing values'),
+                    ('desc', 'Decreasing values'),
+                ],
+                default='none',
+                help_text='This setting only applies for one-dimensional charts.',
+                max_length=5,
+                verbose_name='Sort data',
+            ),
+        ),
+        migrations.AlterField(
+            model_name='chartngcell',
+            name='chart_type',
+            field=models.CharField(
+                choices=[
+                    ('bar', 'Bar'),
+                    ('horizontal-bar', 'Horizontal Bar'),
+                    ('stacked-bar', 'Stacked Bar'),
+                    ('stacked-bar-percent', 'Stacked Bar (%)'),
+                    ('line', 'Line'),
+                    ('pie', 'Pie'),
+                    ('pie-percent', 'Pie (%)'),
+                    ('dot', 'Dot'),
+                    ('table', 'Table'),
+                    ('table-inverted', 'Table (inverted)'),
+                ],
+                default='bar',
+                max_length=20,
+                verbose_name='Chart Type',
+            ),
+        ),
+        migrations.AlterField(
+            model_name='chartngcell',
+            name='height',
+            field=models.CharField(
+                choices=[('150', 'Short (150px)'), ('250', 'Average (250px)'), ('350', 'Tall (350px)')],
+                default='250',
+                max_length=20,
+                verbose_name='Height',
+            ),
+        ),
+    ]

combo/apps/dataviz/migrations/0012_auto_20201126_1557.py

@@ -0,0 +1,49 @@
+# Generated by Django 1.11.29 on 2020-11-26 14:57
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0011_auto_20200813_1100'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Statistic',
+            fields=[
+                (
+                    'id',
+                    models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+                ),
+                ('slug', models.SlugField(max_length=256, verbose_name='Slug')),
+                ('label', models.CharField(max_length=256, verbose_name='Label')),
+                ('site_slug', models.SlugField(max_length=256, verbose_name='Site slug')),
+                ('service_slug', models.SlugField(max_length=256, verbose_name='Service slug')),
+                ('site_title', models.CharField(max_length=256, verbose_name='Site title')),
+                ('url', models.URLField(verbose_name='Data URL')),
+                ('last_update', models.DateTimeField(auto_now=True, null=True, verbose_name='Last update')),
+                ('available', models.BooleanField(default=True, verbose_name='Available data')),
+            ],
+            options={
+                'ordering': ['-available', 'site_title', 'label'],
+            },
+        ),
+        migrations.AlterUniqueTogether(
+            name='statistic',
+            unique_together={('slug', 'site_slug', 'service_slug')},
+        ),
+        migrations.AddField(
+            model_name='chartngcell',
+            name='statistic',
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name='cells',
+                to='dataviz.Statistic',
+                verbose_name='Data',
+                help_text='This list may take a few seconds to be updated, please refresh the page if an item is missing.',
+            ),
+        ),
+    ]

combo/apps/dataviz/migrations/0013_update_chartng_cells.py

@@ -0,0 +1,37 @@
+# Generated by Django 1.11.29 on 2020-11-30 14:26
+
+from django.conf import settings
+from django.db import migrations
+
+
+def update_cells(apps, schema_editor):
+    Statistic = apps.get_model('dataviz', 'Statistic')
+    ChartNgCell = apps.get_model('dataviz', 'ChartNgCell')
+    bijoe_sites = settings.KNOWN_SERVICES.get('bijoe', {})
+
+    for cell in ChartNgCell.objects.filter(statistic__isnull=True):
+        if not cell.data_reference or not cell.cached_json:
+            continue
+        site_slug, slug = cell.data_reference.split(':')
+        statistic, _ = Statistic.objects.get_or_create(
+            slug=slug,
+            site_slug=site_slug,
+            service_slug='bijoe',
+            defaults={
+                'label': cell.cached_json['name'],
+                'url': cell.cached_json['data-url'],
+                'site_title': bijoe_sites.get(site_slug, {}).get('title'),
+            },
+        )
+        cell.statistic = statistic
+        cell.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0012_auto_20201126_1557'),
+    ]
+
+    operations = [
+        migrations.RunPython(update_cells, migrations.RunPython.noop),
+    ]

combo/apps/dataviz/migrations/0014_auto_20201130_1534.py

@@ -0,0 +1,20 @@
+# Generated by Django 1.11.29 on 2020-11-30 14:34
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0013_update_chartng_cells'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='chartngcell',
+            name='cached_json',
+        ),
+        migrations.RemoveField(
+            model_name='chartngcell',
+            name='data_reference',
+        ),
+    ]

combo/apps/dataviz/migrations/0015_auto_20201202_1424.py

@@ -0,0 +1,23 @@
+# Generated by Django 1.11.29 on 2020-12-02 13:24
+
+from django.db import migrations
+from django.db.models import JSONField
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0014_auto_20201130_1534'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='chartngcell',
+            name='filter_params',
+            field=JSONField(default=dict),
+        ),
+        migrations.AddField(
+            model_name='statistic',
+            name='filters',
+            field=JSONField(default=list),
+        ),
+    ]

combo/apps/dataviz/migrations/0016_auto_20201215_1624.py

@@ -0,0 +1,34 @@
+# Generated by Django 1.11.29 on 2020-12-15 15:24
+
+from django.db import migrations, models
+
+from combo.apps.dataviz.models import TIME_FILTERS
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0015_auto_20201202_1424'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='chartngcell',
+            name='time_range',
+            field=models.CharField(
+                blank=True,
+                choices=TIME_FILTERS,
+                max_length=20,
+                verbose_name='Shown period',
+            ),
+        ),
+        migrations.AddField(
+            model_name='chartngcell',
+            name='time_range_end',
+            field=models.DateField(blank=True, null=True, verbose_name='To'),
+        ),
+        migrations.AddField(
+            model_name='chartngcell',
+            name='time_range_start',
+            field=models.DateField(blank=True, null=True, verbose_name='From'),
+        ),
+    ]

combo/apps/dataviz/migrations/0017_text_to_jsonb.py

@@ -0,0 +1,11 @@
+# Generated by Django 2.2.12 on 2021-04-11 15:54
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0016_auto_20201215_1624'),
+    ]
+
+    operations = []

combo/apps/dataviz/migrations/0018_auto_20210723_1318.py

@@ -0,0 +1,27 @@
+# Generated by Django 2.2.21 on 2021-07-23 11:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0017_text_to_jsonb'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='chartcell',
+            name='template_name',
+            field=models.CharField(blank=True, max_length=50, null=True, verbose_name='Cell Template'),
+        ),
+        migrations.AddField(
+            model_name='chartngcell',
+            name='template_name',
+            field=models.CharField(blank=True, max_length=50, null=True, verbose_name='Cell Template'),
+        ),
+        migrations.AddField(
+            model_name='gauge',
+            name='template_name',
+            field=models.CharField(blank=True, max_length=50, null=True, verbose_name='Cell Template'),
+        ),
+    ]

combo/apps/dataviz/migrations/0019_auto_20211006_1525.py

@@ -0,0 +1,38 @@
+# Generated by Django 2.2.19 on 2021-10-06 13:25
+
+from django.db import migrations, models
+
+import combo.data.models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0018_auto_20210723_1318'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='chartngcell',
+            name='time_range_end_template',
+            field=models.CharField(
+                blank=True,
+                max_length=200,
+                validators=[combo.data.models.django_template_validator],
+                verbose_name='To',
+            ),
+        ),
+        migrations.AddField(
+            model_name='chartngcell',
+            name='time_range_start_template',
+            field=models.CharField(
+                blank=True,
+                max_length=200,
+                validators=[combo.data.models.django_template_validator],
+                verbose_name='From',
+                help_text=(
+                    'Template code returning a date. For example, Monday in two weeks would be '
+                    'today|add_days:"14"|adjust_to_week_monday. Page variables are also accessible.'
+                ),
+            ),
+        ),
+    ]

combo/apps/dataviz/migrations/0020_auto_20220118_1103.py

@@ -0,0 +1,26 @@
+# Generated by Django 2.2.19 on 2022-01-18 10:03
+
+from django.db import migrations
+
+
+def update_time_intervals(apps, schema_editor):
+    ChartNgCell = apps.get_model('dataviz', 'ChartNgCell')
+
+    for cell in ChartNgCell.objects.all():
+        save = False
+        for param in cell.filter_params:
+            if param == 'time_interval' and cell.filter_params[param].startswith('_'):
+                cell.filter_params[param] = cell.filter_params[param][1:]
+                save = True
+        if save:
+            cell.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0019_auto_20211006_1525'),
+    ]
+
+    operations = [
+        migrations.RunPython(update_time_intervals, migrations.RunPython.noop),
+    ]

combo/apps/dataviz/migrations/0021_chartfilterscell.py

@@ -0,0 +1,48 @@
+# Generated by Django 2.2.19 on 2022-01-18 10:17
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('data', '0051_link_cell_max_length'),
+        ('auth', '0011_update_proxy_permissions'),
+        ('dataviz', '0020_auto_20220118_1103'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ChartFiltersCell',
+            fields=[
+                (
+                    'id',
+                    models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+                ),
+                ('placeholder', models.CharField(max_length=20)),
+                ('order', models.PositiveIntegerField()),
+                ('slug', models.SlugField(blank=True, verbose_name='Slug')),
+                (
+                    'extra_css_class',
+                    models.CharField(
+                        blank=True, max_length=100, verbose_name='Extra classes for CSS styling'
+                    ),
+                ),
+                (
+                    'template_name',
+                    models.CharField(blank=True, max_length=50, null=True, verbose_name='Cell Template'),
+                ),
+                ('public', models.BooleanField(default=True, verbose_name='Public')),
+                (
+                    'restricted_to_unlogged',
+                    models.BooleanField(default=False, verbose_name='Restrict to unlogged users'),
+                ),
+                ('last_update_timestamp', models.DateTimeField(auto_now=True)),
+                ('groups', models.ManyToManyField(blank=True, to='auth.Group', verbose_name='Roles')),
+                ('page', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='data.Page')),
+            ],
+            options={
+                'verbose_name': 'Filters',
+            },
+        ),
+    ]

combo/apps/dataviz/migrations/0022_chartngcell_subfilters.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.19 on 2022-01-25 16:21
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0021_chartfilterscell'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='chartngcell',
+            name='subfilters',
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=list),
+        ),
+    ]

combo/apps/dataviz/migrations/0023_statistic_has_future_data.py

@@ -0,0 +1,17 @@
+# Generated by Django 2.2.19 on 2022-03-17 10:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0022_chartngcell_subfilters'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='statistic',
+            name='has_future_data',
+            field=models.BooleanField(default=False),
+        ),
+    ]

combo/apps/dataviz/migrations/0024_display_condition.py

@@ -0,0 +1,30 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0023_statistic_has_future_data'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='chartcell',
+            name='condition',
+            field=models.CharField(blank=True, max_length=1000, null=True, verbose_name='Display condition'),
+        ),
+        migrations.AddField(
+            model_name='chartfilterscell',
+            name='condition',
+            field=models.CharField(blank=True, max_length=1000, null=True, verbose_name='Display condition'),
+        ),
+        migrations.AddField(
+            model_name='chartngcell',
+            name='condition',
+            field=models.CharField(blank=True, max_length=1000, null=True, verbose_name='Display condition'),
+        ),
+        migrations.AddField(
+            model_name='gauge',
+            name='condition',
+            field=models.CharField(blank=True, max_length=1000, null=True, verbose_name='Display condition'),
+        ),
+    ]

combo/apps/dataviz/migrations/0025_statistic_data_type.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.26 on 2022-12-06 17:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0024_display_condition'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='statistic',
+            name='data_type',
+            field=models.CharField(default='', max_length=32),
+            preserve_default=False,
+        ),
+    ]

combo/apps/dataviz/migrations/0026_chartfilterscell_filters.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.26 on 2022-12-12 13:28
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0025_statistic_data_type'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='chartfilterscell',
+            name='filters',
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict, verbose_name='Filters'),
+        ),
+    ]

combo/apps/dataviz/migrations/0027_auto_20230222_1001.py

@@ -0,0 +1,21 @@
+# Generated by Django 2.2.26 on 2023-02-22 09:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0026_chartfilterscell_filters'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='statistic',
+            options={'ordering': ['-available', 'deprecated', 'site_title', 'label']},
+        ),
+        migrations.AddField(
+            model_name='statistic',
+            name='deprecated',
+            field=models.BooleanField(default=False),
+        ),
+    ]

combo/apps/dataviz/migrations/0028_increase_extra_css_class.py

@@ -0,0 +1,42 @@
+# Generated by Django 3.2.16 on 2024-01-09 09:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0027_auto_20230222_1001'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='chartcell',
+            name='extra_css_class',
+            field=models.CharField(blank=True, max_length=500, verbose_name='Extra classes for CSS styling'),
+        ),
+        migrations.AlterField(
+            model_name='chartfilterscell',
+            name='extra_css_class',
+            field=models.CharField(blank=True, max_length=500, verbose_name='Extra classes for CSS styling'),
+        ),
+        migrations.AlterField(
+            model_name='chartfilterscell',
+            name='filters',
+            field=models.JSONField(default=dict, verbose_name='Filters'),
+        ),
+        migrations.AlterField(
+            model_name='chartngcell',
+            name='extra_css_class',
+            field=models.CharField(blank=True, max_length=500, verbose_name='Extra classes for CSS styling'),
+        ),
+        migrations.AlterField(
+            model_name='chartngcell',
+            name='subfilters',
+            field=models.JSONField(default=list),
+        ),
+        migrations.AlterField(
+            model_name='gauge',
+            name='extra_css_class',
+            field=models.CharField(blank=True, max_length=500, verbose_name='Extra classes for CSS styling'),
+        ),
+    ]

combo/apps/dataviz/migrations/0029_chartngcell_display_total.py

@@ -0,0 +1,27 @@
+# Generated by Django 3.2.18 on 2024-02-14 11:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0028_increase_extra_css_class'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='chartngcell',
+            name='display_total',
+            field=models.CharField(
+                choices=[
+                    ('none', 'None'),
+                    ('line-and-column', 'Total line and total column'),
+                    ('line', 'Total line'),
+                    ('column', 'Total column'),
+                ],
+                default='line-and-column',
+                max_length=20,
+                verbose_name='Display of total',
+            ),
+        ),
+    ]

combo/apps/dataviz/migrations/0030_remove_all_forms_option.py

@@ -0,0 +1,29 @@
+# Generated by Django 3.2.18 on 2024-04-29 12:05
+
+from django.db import migrations
+
+
+def remove_all_forms_option(apps, schema_editor):
+    ChartNgCell = apps.get_model('dataviz', 'ChartNgCell')
+
+    for cell in ChartNgCell.objects.filter(statistic__slug='forms_counts'):
+        form_filter = cell.filter_params.get('form')
+        if not form_filter or not isinstance(form_filter, list):
+            continue
+
+        try:
+            form_filter.remove('_all')
+        except ValueError:
+            continue
+
+        cell.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0029_chartngcell_display_total'),
+    ]
+
+    operations = [
+        migrations.RunPython(remove_all_forms_option, migrations.RunPython.noop),
+    ]

combo/apps/dataviz/migrations/0031_chart_add_print_values_option.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.2.18 on 2024-04-30 09:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('dataviz', '0030_remove_all_forms_option'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='chartngcell',
+            name='print_values',
+            field=models.BooleanField(
+                default=False,
+                help_text='When not checked values are printed on hover.',
+                verbose_name='Print values on chart',
+            ),
+        ),
+    ]

combo/apps/dataviz/models.py

@@ -14,14 +14,47 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from django.core.urlresolvers import reverse
-from django.db import models
-from django.utils.translation import ugettext_lazy as _
-from django.conf import settings
+import copy
+import os
+import re
+import urllib.parse
+from collections import OrderedDict
+from datetime import date, datetime, timedelta
+
+import pygal
+import pygal.util
+from dateutil.relativedelta import MO, relativedelta
+from django.conf import settings
+from django.db import models, transaction
+from django.db.models import JSONField
+from django.template import RequestContext, Template, TemplateSyntaxError, VariableDoesNotExist
+from django.template.defaultfilters import date as format_date
+from django.urls import reverse
+from django.utils import timezone
+from django.utils.dates import WEEKDAYS
+from django.utils.functional import cached_property
+from django.utils.text import slugify
+from django.utils.timesince import timesince
+from django.utils.translation import gettext
+from django.utils.translation import gettext_lazy as _
+from requests.exceptions import RequestException
 
-from combo.data.models import CellBase
 from combo.data.library import register_cell_class
-from combo.utils import get_templated_url
+from combo.data.models import CellBase, django_template_validator
+from combo.middleware import get_request
+from combo.utils import get_templated_url, requests, spooler
+
+
+class UnsupportedDataSet(Exception):
+    pass
+
+
+class MissingRequest(Exception):
+    pass
+
+
+class MissingVariable(Exception):
+    pass
 
 
 @register_cell_class
@@ -32,7 +65,7 @@ class Gauge(CellBase):
     jsonp_data_source = models.BooleanField(_('Use JSONP to get data'), default=True)
     max_value = models.PositiveIntegerField(_('Max Value'), blank=True, null=True)
 
-    template_name = 'combo/gauge-cell.html'
+    default_template_name = 'combo/gauge-cell.html'
 
     class Media:
         js = ('js/gauge.min.js', 'js/combo.gauge.js')
@@ -51,31 +84,37 @@ class Gauge(CellBase):
             data_source_url = get_templated_url(self.data_source)
         else:
             data_source_url = reverse('combo-ajax-gauge-count', kwargs={'cell': self.id})
-        return {'cell': self,
-                'title': self.title,
-                'url': get_templated_url(self.url) if self.url else None,
-                'max_value': self.max_value,
-                'data_source_url': data_source_url,
-                'jsonp': self.jsonp_data_source,
-                }
+        return {
+            'cell': self,
+            'title': self.title,
+            'url': get_templated_url(self.url) if self.url else None,
+            'max_value': self.max_value,
+            'data_source_url': data_source_url,
+            'jsonp': self.jsonp_data_source,
+        }
 
 
 @register_cell_class
 class ChartCell(CellBase):
-    template_name = 'combo/dataviz-chart.html'
+    default_template_name = 'combo/dataviz-chart.html'
 
     title = models.CharField(_('Title'), max_length=150, blank=True, null=True)
-    url = models.URLField(_('URL'), max_length=150, blank=True, null=True)
+    url = models.URLField(_('URL'), max_length=250, blank=True, null=True)
 
     class Meta:
-        verbose_name = _('Chart')
+        verbose_name = _('Chart (legacy)')
 
     @classmethod
-    def is_enabled(self):
-        return hasattr(settings, 'KNOWN_SERVICES') and settings.KNOWN_SERVICES.get('bijoe')
+    def is_enabled(cls):
+        return (
+            settings.LEGACY_CHART_CELL_ENABLED
+            and hasattr(settings, 'KNOWN_SERVICES')
+            and settings.KNOWN_SERVICES.get('bijoe')
+        )
 
     def get_default_form_class(self):
         from .forms import ChartForm
+
         return ChartForm
 
     def get_additional_label(self):
@@ -84,7 +123,773 @@ class ChartCell(CellBase):
         return ''
 
     def get_cell_extra_context(self, context):
-        context = super(ChartCell, self).get_cell_extra_context(context)
+        context = super().get_cell_extra_context(context)
         context['title'] = self.title
         context['url'] = self.url
         return context
+
+
+class StatisticManager(models.Manager):
+    def get_by_natural_key(self, slug, site_slug, service_slug):
+        return self.get_or_create(slug=slug, site_slug=site_slug, service_slug=service_slug)[0]
+
+
+class Statistic(models.Model):
+    slug = models.SlugField(_('Slug'), max_length=256)
+    label = models.CharField(_('Label'), max_length=256)
+    site_slug = models.SlugField(_('Site slug'), max_length=256)
+    service_slug = models.SlugField(_('Service slug'), max_length=256)
+    site_title = models.CharField(_('Site title'), max_length=256)
+    url = models.URLField(_('Data URL'))
+    filters = JSONField(default=list)
+    has_future_data = models.BooleanField(default=False)
+    data_type = models.CharField(max_length=32)
+    deprecated = models.BooleanField(default=False)
+    available = models.BooleanField(_('Available data'), default=True)
+    last_update = models.DateTimeField(_('Last update'), null=True, auto_now=True)
+
+    objects = StatisticManager()
+
+    class Meta:
+        ordering = ['-available', 'deprecated', 'site_title', 'label']
+        unique_together = ['slug', 'site_slug', 'service_slug']
+
+    def __str__(self):
+        name = _('%(title)s: %(label)s') % {'title': self.site_title or self.site_slug, 'label': self.label}
+        if not self.available:
+            name = _('%s (unavailable)') % name
+        elif self.deprecated:
+            name = _('%s (deprecated)') % name
+        return name
+
+    def natural_key(self):
+        return (self.slug, self.site_slug, self.service_slug)
+
+    def has_native_support_for_interval(self, time_interval):
+        # pylint: disable=not-an-iterable
+        return any(
+            time_interval == x['id']
+            for filter_ in self.filters
+            for x in filter_['options']
+            if filter_['id'] == 'time_interval'
+        )
+
+
+TIME_FILTERS = [
+    ('previous-year', _('Previous year')),
+    ('current-year', _('Current year')),
+    ('next-year', _('Next year')),
+    ('previous-month', _('Previous month')),
+    ('current-month', _('Current month')),
+    ('next-month', _('Next month')),
+    ('previous-week', _('Previous week')),
+    ('current-week', _('Current week')),
+    ('next-week', _('Next week')),
+    ('range', _('Free range (date)')),
+    ('range-template', _('Free range (template)')),
+]
+
+
+@register_cell_class
+class ChartNgCell(CellBase):
+    statistic = models.ForeignKey(
+        verbose_name=_('Data'),
+        to=Statistic,
+        blank=False,
+        null=True,
+        on_delete=models.SET_NULL,
+        related_name='cells',
+        help_text=_(
+            'This list may take a few seconds to be updated, please refresh the page if an item is missing.'
+        ),
+    )
+    subfilters = JSONField(default=list)
+    filter_params = JSONField(default=dict)
+    title = models.CharField(_('Title'), max_length=150, blank=True)
+    time_range = models.CharField(
+        _('Shown period'),
+        max_length=20,
+        blank=True,
+        choices=TIME_FILTERS,
+    )
+    time_range_start = models.DateField(_('From'), null=True, blank=True)
+    time_range_end = models.DateField(_('To'), null=True, blank=True)
+    time_range_start_template = models.CharField(
+        _('From'),
+        max_length=200,
+        blank=True,
+        validators=[django_template_validator],
+        help_text=_(
+            'Template code returning a date. For example, Monday in two weeks would be '
+            'today|add_days:"14"|adjust_to_week_monday. Page variables are also accessible.'
+        ),
+    )
+    time_range_end_template = models.CharField(
+        _('To'),
+        max_length=200,
+        blank=True,
+        validators=[django_template_validator],
+    )
+    chart_type = models.CharField(
+        _('Chart Type'),
+        max_length=20,
+        default='bar',
+        choices=(
+            ('bar', _('Bar')),
+            ('horizontal-bar', _('Horizontal Bar')),
+            ('stacked-bar', _('Stacked Bar')),
+            ('stacked-bar-percent', _('Stacked Bar (%)')),
+            ('line', _('Line')),
+            ('pie', _('Pie')),
+            ('pie-percent', _('Pie (%)')),
+            ('dot', _('Dot')),
+            ('table', _('Table')),
+            ('table-inverted', _('Table (inverted)')),
+        ),
+    )
+    display_total = models.CharField(
+        _('Display of total'),
+        max_length=20,
+        default='line-and-column',
+        choices=(
+            ('none', _('None')),
+            ('line-and-column', _('Total line and total column')),
+            ('line', _('Total line')),
+            ('column', _('Total column')),
+        ),
+    )
+
+    height = models.CharField(
+        _('Height'),
+        max_length=20,
+        default='250',
+        choices=(
+            ('150', _('Short (150px)')),
+            ('250', _('Average (250px)')),
+            ('350', _('Tall (350px)')),
+        ),
+    )
+
+    sort_order = models.CharField(
+        _('Sort data'),
+        max_length=5,
+        default='none',
+        help_text=_('This setting only applies for one-dimensional charts.'),
+        choices=(
+            ('none', _('None')),
+            ('alpha', _('Alphabetically')),
+            ('asc', _('Increasing values')),
+            ('desc', _('Decreasing values')),
+        ),
+    )
+
+    hide_null_values = models.BooleanField(
+        default=False,
+        verbose_name=_('Hide null values'),
+        help_text=_('This setting only applies for one-dimensional charts.'),
+    )
+
+    print_values = models.BooleanField(
+        default=False,
+        verbose_name=_('Print values on chart'),
+        help_text=_('When not checked values are printed on hover.'),
+    )
+
+    manager_form_template = 'combo/chartngcell_form.html'
+
+    invalid_reason_codes = {
+        'missing_statistic_url': _('No statistic URL set'),
+        'statistic_data_not_found': _('Statistic URL seems to unexist'),
+        'statistic_url_invalid': _('Statistic URL seems to be invalid'),
+    }
+
+    class Meta:
+        verbose_name = _('Chart')
+
+    class Media:
+        js = ('js/chartngcell.js',)
+        css = {'all': ('css/combo.chartngcell.css',)}
+
+    @classmethod
+    def is_enabled(cls):
+        return settings.KNOWN_SERVICES.get('bijoe') or settings.STATISTICS_PROVIDERS
+
+    def get_default_form_class(self):
+        from .forms import ChartNgForm
+
+        return ChartNgForm
+
+    def get_additional_label(self):
+        return self.title
+
+    def get_download_filename(self):
+        label = slugify(self.title or self.statistic.label)
+        return 'export-%s-%s' % (label, date.today().strftime('%Y%m%d'))
+
+    def is_relevant(self, context):
+        return bool(self.statistic)
+
+    def is_table_chart(self):
+        return bool(self.chart_type in ('table', 'table-inverted'))
+
+    def check_validity(self):
+        if not self.statistic:
+            return
+
+        if not self.statistic.url:
+            self.mark_as_invalid('missing_statistic_url')
+            return
+
+        resp = None
+        try:
+            resp = self.get_statistic_data()
+        except (RequestException, MissingRequest, MissingVariable):
+            pass
+
+        self.set_validity_from_url(
+            resp, not_found_code='statistic_data_not_found', invalid_code='statistic_url_invalid'
+        )
+
+    def get_statistic_data(self, filter_params=None, raise_if_not_cached=False, invalidate_cache=False):
+        headers = {
+            'X-Statistics-Page-URL': urllib.parse.urljoin(
+                settings.SITE_BASE_URL,
+                reverse('combo-manager-page-view', kwargs={'pk': self.page_id})
+                + '#cell-%s' % self.get_reference(),
+            )
+        }
+        return requests.get(
+            self.statistic.url,
+            params=filter_params or self.get_filter_params(),
+            headers=headers,
+            cache_duration=300,
+            remote_service='auto',
+            without_user=True,
+            raise_if_not_cached=raise_if_not_cached,
+            log_errors=False,
+            invalidate_cache=invalidate_cache,
+        )
+
+    def get_chart(self, width=None, height=None, raise_if_not_cached=False):
+        filter_params = self.get_filter_params()
+        transaction.on_commit(
+            lambda: spooler.refresh_statistics_data(cell_pk=self.pk, filter_params=filter_params)
+        )
+        response = self.get_statistic_data(filter_params, raise_if_not_cached)
+        response.raise_for_status()
+        response = response.json()
+
+        style = pygal.style.DefaultStyle(font_family='"Open Sans", sans-serif', background='transparent')
+
+        chart = {
+            'bar': pygal.Bar,
+            'horizontal-bar': pygal.HorizontalBar,
+            'stacked-bar': pygal.StackedBar,
+            'stacked-bar-percent': pygal.StackedBar,
+            'line': pygal.Line,
+            'pie': pygal.Pie,
+            'pie-percent': pygal.Pie,
+            'dot': pygal.Dot,
+            'table': pygal.Bar,
+            'table-inverted': pygal.Bar,
+        }[self.chart_type](
+            config=pygal.Config(
+                style=copy.copy(style), order_min=0.1, max_scale=5, print_values=self.print_values
+            )
+        )
+
+        if self.statistic.service_slug == 'bijoe':
+            x_labels, y_labels, data = self.parse_response(response, chart)
+            chart.x_labels = x_labels
+
+            if chart.axis_count == 1:
+                data = self.process_one_dimensional_data(chart, data)
+                if getattr(chart, 'compute_sum', True) and self.is_table_chart():
+                    data = self.add_total_to_line_table(chart, data)
+            self.add_data_to_chart(chart, data, y_labels)
+        else:
+            data = response['data']
+
+            interval = self.filter_params.get('time_interval', '')
+            if data['x_labels'] and interval:
+                if interval == 'day' or not self.statistic.has_native_support_for_interval(interval):
+                    self.aggregate_data(data, interval)
+                elif interval == 'month':
+                    data['x_labels'] = [
+                        format_date(datetime.strptime(x, '%Y-%m'), 'M Y') for x in data['x_labels']
+                    ]
+
+            chart.x_labels = data['x_labels']
+            chart.axis_count = min(len(data['series']), 2)
+            chart.compute_sum = False
+
+            if self.statistic.data_type:
+                chart.config.value_formatter = self.get_value_formatter(self.statistic.data_type)
+
+            if chart.axis_count == 1:
+                data['series'][0]['data'] = self.process_one_dimensional_data(
+                    chart, data['series'][0]['data']
+                )
+                if self.chart_type in ('pie', 'pie-percent'):
+                    data['series'] = [
+                        {'label': chart.config.x_value_formatter(label), 'data': [data]}
+                        for label, data in zip(chart.x_labels, data['series'][0]['data'])
+                        if data
+                    ]
+                elif self.chart_type == 'dot':
+                    data['series'][0]['label'] = ''
+
+            if self.chart_type == 'stacked-bar-percent':
+                self.make_percent([serie['data'] for serie in data['series']])
+            elif self.chart_type == 'pie-percent':
+                self.make_global_percent([serie['data'] for serie in data['series']])
+                chart.config.value_formatter = self.get_value_formatter(measure='percent')
+
+            for serie in data['series']:
+                chart.add(serie['label'], serie['data'])
+
+            if self.is_table_chart() and not self.statistic.data_type:
+                self.add_total_to_table(chart, [serie['data'] for serie in data['series']])
+
+        self.configure_chart(chart, width, height)
+
+        return chart
+
+    def get_filter_params(self):
+        params = {k: self.evaluate_filter_value(v) for k, v in self.filter_params.items() if v}
+
+        now = timezone.now().date()
+        if self.time_range == 'current-year':
+            params['start'] = date(year=now.year, month=1, day=1)
+            params['end'] = date(year=now.year + 1, month=1, day=1)
+        elif self.time_range == 'previous-year':
+            params['start'] = date(year=now.year - 1, month=1, day=1)
+            params['end'] = date(year=now.year, month=1, day=1)
+        elif self.time_range == 'next-year':
+            params['start'] = date(year=now.year + 1, month=1, day=1)
+            params['end'] = date(year=now.year + 2, month=1, day=1)
+        elif self.time_range == 'current-month':
+            params['start'] = now.replace(day=1)
+            params['end'] = now + relativedelta(day=1, months=1)
+        elif self.time_range == 'previous-month':
+            params['start'] = now + relativedelta(day=1, months=-1)
+            params['end'] = now.replace(day=1)
+        elif self.time_range == 'next-month':
+            params['start'] = now + relativedelta(day=1, months=1)
+            params['end'] = now + relativedelta(day=1, months=2)
+        elif self.time_range == 'current-week':
+            params['start'] = now + relativedelta(weekday=MO(-1))
+            params['end'] = now + relativedelta(weekday=MO(+1), days=+1)
+        elif self.time_range == 'previous-week':
+            params['start'] = now + relativedelta(weekday=MO(-2))
+            params['end'] = now + relativedelta(weekday=MO(-1))
+        elif self.time_range == 'next-week':
+            params['start'] = now + relativedelta(weekday=MO(+1), days=+1)
+            params['end'] = now + relativedelta(weekday=MO(+2), days=+1)
+        elif self.time_range == 'range':
+            if self.time_range_start:
+                params['start'] = self.time_range_start
+            if self.time_range_end:
+                params['end'] = self.time_range_end
+        elif self.time_range == 'range-template':
+            if self.time_range_start_template:
+                start = self.evaluate_range_template(self.time_range_start_template)
+                if start:
+                    params['start'] = start
+            if self.time_range_end_template:
+                end = self.evaluate_range_template(self.time_range_end_template)
+                if end:
+                    params['end'] = end
+        if 'time_interval' in params and not self.statistic.has_native_support_for_interval(
+            params['time_interval']
+        ):
+            params['time_interval'] = 'day'
+        return params
+
+    def evaluate_range_template(self, value):
+        if value in self.page.extra_variables:
+            value = self.page.extra_variables[value].strip('{ }')
+
+        context = self.request_context
+        context.update({'now': datetime.now, 'today': datetime.now})
+        try:
+            return Template('{{ %s|date:"Y-m-d" }}' % value).render(context)
+        except (VariableDoesNotExist, TemplateSyntaxError):
+            return None
+
+    def evaluate_filter_value(self, value):
+        if isinstance(value, list) or not value.startswith('variable:'):
+            return value
+
+        try:
+            variable = self.page.extra_variables[value.replace('variable:', '')]
+        except KeyError:
+            raise MissingVariable
+
+        return Template(variable).render(self.request_context)
+
+    @cached_property
+    def request_context(self):
+        if not getattr(self, '_request', None):
+            raise MissingRequest
+
+        ctx = RequestContext(self._request, getattr(self._request, 'extra_context', {}))
+        ctx['request'] = self._request
+        return ctx
+
+    def parse_response(self, response, chart):
+        # normalize axis to have a fake axis when there are no dimensions and
+        # always a x axis when there is a single dimension.
+        data = response['data']
+        loop_labels = response['axis'].get('loop') or []
+        x_labels = response['axis'].get('x_labels') or []
+        y_labels = response['axis'].get('y_labels') or []
+        if loop_labels:
+            if 'x_labels' in response['axis'] and 'y_labels' in response['axis']:
+                # no support for three dimensions
+                raise UnsupportedDataSet()
+            if not y_labels:
+                y_labels = loop_labels
+            else:
+                x_labels, y_labels = y_labels, loop_labels
+            if (
+                x_labels
+                and y_labels
+                and (len(y_labels) != len(data) or not all([len(x) == len(x_labels) for x in data]))
+            ):
+                # varying dimensions
+                raise UnsupportedDataSet()
+        if not x_labels and not y_labels:  # unidata
+            x_labels = ['']
+            y_labels = ['']
+            data = [data]
+            chart.axis_count = 0
+        elif not x_labels:
+            x_labels = y_labels
+            y_labels = ['']
+            chart.axis_count = 1
+        elif not y_labels:
+            y_labels = ['']
+            chart.axis_count = 1
+        else:
+            chart.axis_count = 2
+
+        chart.compute_sum = bool(response.get('measure') == 'integer' and chart.axis_count > 0)
+
+        formatter = self.get_value_formatter(response.get('unit'), response.get('measure'))
+        if formatter is not None:
+            chart.config.value_formatter = formatter
+
+        return x_labels, y_labels, data
+
+    def configure_dot_chart(self, chart, width, height):
+        chart.show_legend = False
+        # use a single colour for dots
+        chart.config.style.colors = ('#1f77b4',) * max(len(chart.x_labels), 1)
+
+    def configure_horizontal_bar_chart(self, chart, width, height):
+        if width and width < 500:
+            # truncate labels
+            chart.x_labels = [pygal.util.truncate(x, 15) for x in chart.x_labels]
+
+    def configure_pie_chart(self, chart, width, height):
+        chart.show_legend = True
+        if width and height:
+            # pies are as tall as wide, reserve the appropriate space and distribute
+            # the rest for the legend.
+            chart.truncate_legend = (width - height) // 10
+        elif width:
+            chart.truncate_legend = width // 20
+
+    def configure_pie_percent_chart(self, *args, **kwargs):
+        self.configure_pie_chart(*args, **kwargs)
+
+    def configure_chart(self, chart, width, height):
+        auto_height_scale = pygal.style.DefaultStyle.legend_font_size * 1.75
+        chart.config.margin = 0
+        if width:
+            chart.config.width = width
+        height = height or int(self.height)
+        # adapt chart's height to legend length
+        chart.config.height = max(height, auto_height_scale * len(chart.raw_series))
+
+        if width or height:
+            chart.config.explicit_size = True
+
+        chart.config.js = [os.path.join(settings.STATIC_URL, 'js/pygal-tooltips.js')]
+
+        chart.show_legend = bool(chart.axis_count > 1)
+        chart.truncate_legend = 30
+        # matplotlib tab10 palette
+        chart.config.style.colors = (
+            '#1f77b4',
+            '#ff7f0e',
+            '#2ca02c',
+            '#d62728',
+            '#9467bd',
+            '#8c564b',
+            '#e377c2',
+            '#7f7f7f',
+            '#bcbd22',
+            '#17becf',
+        )
+
+        custom_configure_method_name = 'configure_%s_chart' % self.chart_type.replace('-', '_')
+        if hasattr(self, custom_configure_method_name):
+            getattr(self, custom_configure_method_name)(chart, width, height)
+
+        if self.chart_type not in ('pie', 'pie-percent'):
+            if width and width < 500:
+                chart.legend_at_bottom = True
+                # restore demanded chart's height
+                chart.config.height = height
+
+    def process_one_dimensional_data(self, chart, data):
+        if self.hide_null_values:
+            data = self.hide_values(chart, data)
+        if data and self.sort_order != 'none':
+            data = self.sort_values(chart, data)
+        return data
+
+    @staticmethod
+    def hide_values(chart, data):
+        x_labels, new_data = [], []
+        for label, value in zip(chart.x_labels, data):
+            if value:
+                x_labels.append(label)
+                new_data.append(value)
+        chart.x_labels = x_labels
+        return new_data
+
+    def sort_values(self, chart, data):
+        if self.sort_order == 'alpha':
+            digit_re = re.compile('([0-9]+)')
+
+            def natural_sort_key(item):
+                return [
+                    int(text) if text.isdigit() else text.lower() for text in digit_re.split(str(item[0]))
+                ]
+
+            tmp_items = sorted(zip(chart.x_labels, data), key=natural_sort_key)
+
+        elif self.sort_order == 'asc':
+            tmp_items = sorted(zip(chart.x_labels, data), key=lambda x: (x[1] or 0))
+        elif self.sort_order == 'desc':
+            tmp_items = sorted(zip(chart.x_labels, data), key=lambda x: (x[1] or 0), reverse=True)
+        x_labels, sorted_data = zip(*((label, value) for label, value in tmp_items))
+        chart.x_labels = list(x_labels)
+        return list(sorted_data)
+
+    @staticmethod
+    def add_total_to_line_table(chart, data):
+        # workaround pygal
+        chart.compute_sum = False
+
+        # do not add total for single point
+        if len(data) <= 1:
+            return data
+
+        data.append(sum(x for x in data if x is not None))
+        chart.x_labels.append(gettext('Total'))
+        return data
+
+    def add_total_to_table(self, chart, series_data):
+        if chart.axis_count == 0:
+            return
+
+        # do not add total for single point
+        if len(series_data) == 1 and len(series_data[0]) == 1:
+            return
+
+        if self.display_total in ('line', 'line-and-column'):
+            chart.x_labels.append(gettext('Total'))
+            for serie in series_data:
+                serie.append(sum(x for x in serie if x is not None))
+
+        if chart.axis_count == 1:
+            return
+
+        if self.display_total in ('column', 'line-and-column'):
+            line_totals = []
+            for line in zip(*series_data):
+                line_totals.append(sum(x for x in line if x is not None))
+            chart.add(gettext('Total'), line_totals)
+
+    def add_data_to_chart(self, chart, data, y_labels):
+        if self.chart_type not in ('pie', 'pie-percent'):
+            series_data = []
+            for i, serie_label in enumerate(y_labels):
+                if chart.axis_count < 2:
+                    values = data
+                else:
+                    values = [data[i][j] for j in range(len(chart.x_labels))]
+                series_data.append(values)
+                chart.add(serie_label, values)
+            if self.chart_type == 'stacked-bar-percent':
+                self.make_percent(series_data)
+        else:
+            # pie, create a serie by data, to get different colours
+            values = data
+            for label, value in zip(chart.x_labels, values):
+                if not value:
+                    continue
+                chart.add(label, value)
+
+    @staticmethod
+    def get_value_formatter(unit=None, measure='duration'):
+        if unit == 'seconds' or measure == 'duration':
+
+            def format_duration(value):
+                if value is None:
+                    return '-'
+                base_date = datetime(1871, 3, 18)
+                return timesince(base_date, base_date + timedelta(seconds=value))
+
+            return format_duration
+        elif measure == 'percent':
+            percent_formatter = lambda x: f'{x:.1f}%'
+            return percent_formatter
+
+    def make_percent(self, series_data):
+        for i, values in enumerate(zip(*series_data)):
+            sum_values = sum(v for v in values if v is not None)
+            if sum_values == 0:
+                continue
+
+            factor = 100 / sum_values
+            for values in series_data:
+                if values[i] is not None:
+                    values[i] = round(values[i] * factor, 1)
+
+    def make_global_percent(self, series_data):
+        sum_values = sum(v for values in series_data for v in values if v is not None)
+        if sum_values == 0:
+            return
+
+        factor = 100 / sum_values
+        for serie in series_data:
+            for i, value in enumerate(serie):
+                if value is not None:
+                    serie[i] = round(value * factor, 1)
+
+    @staticmethod
+    def aggregate_data(data, interval):
+        series_data = [serie['data'] for serie in data['series']]
+        dates = [datetime.strptime(label, '%Y-%m-%d') for label in data['x_labels']]
+        min_date, max_date = min(dates), max(dates)
+
+        date_formats = {
+            'day': 'd M Y',
+            # Translators: This indicates week number followed by year, for example it can yield W2-2021.
+            # First "W" is the first letter of the word "week" and should be translated accordingly, second
+            # "W" and "o" are interpreted by Django's date filter and should be left as is. First W is
+            # backslash escaped to prevent it from being interpreted, translators should refer to Django's
+            # documentation in order to know if the new letter resulting of translation should be escaped or not.
+            'week': gettext(r'\WW-o'),
+            'month': 'M Y',
+            'year': 'Y',
+            'weekday': 'l',
+        }
+        if interval == 'day':
+            x_labels = [
+                format_date(min_date + timedelta(days=i), date_formats['day'])
+                for i in range((max_date - min_date).days + 1)
+            ]
+        elif interval == 'month':
+            month_difference = max_date.month - min_date.month + (max_date.year - min_date.year) * 12
+            x_labels = [
+                format_date(min_date + relativedelta(months=i), date_formats['month'])
+                for i in range(month_difference + 1)
+            ]
+        elif interval == 'year':
+            x_labels = [str(year) for year in range(min_date.year, max_date.year + 1)]
+        elif interval == 'weekday':
+            x_labels = [str(label) for label in WEEKDAYS.values()]
+        elif interval == 'week':
+            x_labels = []
+            date, last_date = min_date, max_date
+            if min_date.weekday() > max_date.weekday():
+                last_date += relativedelta(weeks=1)
+            while date <= last_date:
+                x_labels.append(format_date(date, date_formats['week']))
+                date += relativedelta(weeks=1)
+
+        aggregates = OrderedDict((label, [0] * len(series_data)) for label in x_labels)
+        for i, date in enumerate(dates):
+            key = format_date(date, date_formats[interval])
+            for j, dummy in enumerate(series_data):
+                aggregates[key][j] += series_data[j][i] or 0
+
+        data['x_labels'] = x_labels
+        for i, serie in enumerate(data['series']):
+            serie['data'] = [values[i] for values in aggregates.values()]
+
+    @property
+    def available_filters(self):
+        return self.statistic.filters + self.subfilters
+
+    def update_subfilters(self, filter_params=None):
+        self._request = get_request()
+        try:
+            response = self.get_statistic_data(filter_params=filter_params)
+        except (TemplateSyntaxError, VariableDoesNotExist):
+            return
+
+        try:
+            response.raise_for_status()
+            data = response.json()['data']
+        except Exception:
+            return
+
+        new_subfilters = data.get('subfilters', [])
+        if self.subfilters != new_subfilters:
+            self.subfilters = new_subfilters
+            subfilter_ids = {filter_['id'] for filter_ in self.available_filters}
+            self.filter_params = {k: v for k, v in self.filter_params.items() if k in subfilter_ids}
+            self.save()
+
+    def get_cache_key(self, filters_cell_id):
+        return 'dataviz:%s:%s' % (filters_cell_id, self.pk)
+
+
+@register_cell_class
+class ChartFiltersCell(CellBase):
+    filters = JSONField(_('Filters'), default=dict)
+
+    title = _('Filters')
+    default_template_name = 'combo/chart-filters.html'
+    manager_form_template = 'combo/chartfilterscell_form.html'
+    max_one_by_page = True
+
+    class Meta:
+        verbose_name = _('Filters')
+
+    class Media:
+        js = ('xstatic/select2.min.js', 'xstatic/i18n/fr.js')
+        css = {'all': ('xstatic/select2.min.css',)}
+
+    @classmethod
+    def is_enabled(cls):
+        return settings.CHART_FILTERS_CELL_ENABLED and settings.STATISTICS_PROVIDERS
+
+    def get_cell_extra_context(self, context):
+        from .forms import ChartFiltersForm
+
+        ctx = super().get_cell_extra_context(context)
+        if 'filters_cell_id' in context['request'].GET:  # detect refresh on submit
+            ctx['form'] = ChartFiltersForm(
+                data=context['request'].GET,
+                page=self.page,
+                filters_cell=self,
+                filters_cell_id=context['request'].GET['filters_cell_id'],
+            )
+            ctx['form'].errors.clear()
+        else:
+            ctx['form'] = ChartFiltersForm(page=self.page, filters_cell=self)
+
+        return ctx
+
+    def get_default_form_class(self):
+        from .forms import ChartFiltersConfigForm
+
+        return ChartFiltersConfigForm

combo/apps/dataviz/static/css/combo.chartngcell.css

@@ -0,0 +1,21 @@
+.cell.chart-ng-cell {
+      position: relative;
+}
+
+.chart-ng-cell .download-button {
+      position: absolute;
+      bottom: 0px;
+      right: 0px;
+      line-height: unset;
+}
+
+.chart-ng-cell .download-button:after {
+      font-family: FontAwesome;
+      content: "\f019"; /* download */
+}
+
+.dataviz-table.total-line tr:last-child,
+.dataviz-table.total-line-and-column tr:last-child {
+      font-weight: 600;
+      background: #f7f7f7;
+}

combo/apps/dataviz/static/js/chartngcell.js

@@ -0,0 +1,14 @@
+function get_graph_querystring(extra_context, width=undefined) {
+  qs = [];
+  if ($('#chart-filters').length) {
+    qs.push($('#chart-filters').serialize());
+    qs.push('filters_cell_id=' + $('body').data('filters-cell-id'));
+  }
+  if (extra_context)
+      qs.push('ctx=' + extra_context);
+  if (window.location.search)
+      qs.push(window.location.search.slice(1));
+  if (width)
+    qs.push('width=' + width);
+  return '?' + qs.join('&');
+};

combo/apps/dataviz/static/js/pygal-tooltips.js

@@ -0,0 +1,353 @@
+(function() {
+  var $, get_translation, init, init_svg, matches, padding, r_translation, sibl, svg_ns, tooltip_timeout, xlink_ns;
+
+  svg_ns = 'http://www.w3.org/2000/svg';
+
+  xlink_ns = 'http://www.w3.org/1999/xlink';
+
+  $ = function(sel, ctx) {
+    if (ctx == null) {
+      ctx = null;
+    }
+    ctx = ctx || document;
+    return Array.prototype.slice.call(ctx.querySelectorAll(sel), 0).filter(function(e) {
+      return e !== ctx;
+    });
+  };
+
+  matches = function(el, selector) {
+    return (el.matches || el.matchesSelector || el.msMatchesSelector || el.mozMatchesSelector || el.webkitMatchesSelector || el.oMatchesSelector).call(el, selector);
+  };
+
+  sibl = function(el, match) {
+    if (match == null) {
+      match = null;
+    }
+    return Array.prototype.filter.call(el.parentElement.children, function(child) {
+      return child !== el && (!match || matches(child, match));
+    });
+  };
+
+  Array.prototype.one = function() {
+    return this.length > 0 && this[0] || {};
+  };
+
+  padding = 5;
+
+  tooltip_timeout = null;
+
+  r_translation = /translate\((\d+)[ ,]+(\d+)\)/;
+
+  get_translation = function(el) {
+    return (r_translation.exec(el.getAttribute('transform')) || []).slice(1).map(function(x) {
+      return +x;
+    });
+  };
+
+  init = function(ctx) {
+    var bbox, box, config, el, graph, inner_svg, num, parent, tooltip, tooltip_el, tt, uid, untooltip, xconvert, yconvert, _i, _j, _k, _len, _len1, _len2, _ref, _ref1, _ref2, _ref3;
+    if ($('svg', ctx).length) {
+      inner_svg = $('svg', ctx).one();
+      parent = inner_svg.parentElement;
+      box = inner_svg.viewBox.baseVal;
+      bbox = parent.getBBox();
+      xconvert = function(x) {
+        return ((x - box.x) / box.width) * bbox.width;
+      };
+      yconvert = function(y) {
+        return ((y - box.y) / box.height) * bbox.height;
+      };
+    } else {
+      xconvert = yconvert = function(x) {
+        return x;
+      };
+    }
+    if (((_ref = window.pygal) != null ? _ref.config : void 0) != null) {
+      if (window.pygal.config.no_prefix != null) {
+        config = window.pygal.config;
+      } else {
+        uid = ctx.id.replace('chart-', '');
+        config = window.pygal.config[uid];
+      }
+    } else {
+      config = window.config;
+    }
+    tooltip_el = null;
+    graph = $('.graph').one();
+    tt = $('.tooltip', ctx).one();
+    _ref1 = $('.reactive', ctx);
+    for (_i = 0, _len = _ref1.length; _i < _len; _i++) {
+      el = _ref1[_i];
+      el.addEventListener('mouseenter', (function(el) {
+        return function() {
+          return el.classList.add('active');
+        };
+      })(el));
+      el.addEventListener('mouseleave', (function(el) {
+        return function() {
+          return el.classList.remove('active');
+        };
+      })(el));
+    }
+    _ref2 = $('.activate-serie', ctx);
+    for (_j = 0, _len1 = _ref2.length; _j < _len1; _j++) {
+      el = _ref2[_j];
+      num = el.id.replace('activate-serie-', '');
+      el.addEventListener('mouseenter', (function(num) {
+        return function() {
+          var re, _k, _l, _len2, _len3, _ref3, _ref4, _results;
+          _ref3 = $('.serie-' + num + ' .reactive', ctx);
+          for (_k = 0, _len2 = _ref3.length; _k < _len2; _k++) {
+            re = _ref3[_k];
+            re.classList.add('active');
+          }
+          _ref4 = $('.serie-' + num + ' .showable', ctx);
+          _results = [];
+          for (_l = 0, _len3 = _ref4.length; _l < _len3; _l++) {
+            re = _ref4[_l];
+            _results.push(re.classList.add('shown'));
+          }
+          return _results;
+        };
+      })(num));
+      el.addEventListener('mouseleave', (function(num) {
+        return function() {
+          var re, _k, _l, _len2, _len3, _ref3, _ref4, _results;
+          _ref3 = $('.serie-' + num + ' .reactive', ctx);
+          for (_k = 0, _len2 = _ref3.length; _k < _len2; _k++) {
+            re = _ref3[_k];
+            re.classList.remove('active');
+          }
+          _ref4 = $('.serie-' + num + ' .showable', ctx);
+          _results = [];
+          for (_l = 0, _len3 = _ref4.length; _l < _len3; _l++) {
+            re = _ref4[_l];
+            _results.push(re.classList.remove('shown'));
+          }
+          return _results;
+        };
+      })(num));
+      el.addEventListener('click', (function(el, num) {
+        return function() {
+          var ov, re, rect, show, _k, _l, _len2, _len3, _ref3, _ref4, _results;
+          rect = $('rect', el).one();
+          show = rect.style.fill !== '';
+          rect.style.fill = show ? '' : 'transparent';
+          _ref3 = $('.serie-' + num + ' .reactive', ctx);
+          for (_k = 0, _len2 = _ref3.length; _k < _len2; _k++) {
+            re = _ref3[_k];
+            re.style.display = show ? '' : 'none';
+          }
+          _ref4 = $('.text-overlay .serie-' + num, ctx);
+          _results = [];
+          for (_l = 0, _len3 = _ref4.length; _l < _len3; _l++) {
+            ov = _ref4[_l];
+            _results.push(ov.style.display = show ? '' : 'none');
+          }
+          return _results;
+        };
+      })(el, num));
+    }
+    _ref3 = $('.tooltip-trigger', ctx);
+    for (_k = 0, _len2 = _ref3.length; _k < _len2; _k++) {
+      el = _ref3[_k];
+      el.addEventListener('mouseenter', (function(el) {
+        return function() {
+          return tooltip_el = tooltip(el);
+        };
+      })(el));
+    }
+    tt.addEventListener('mouseenter', function() {
+      return tooltip_el != null ? tooltip_el.classList.add('active') : void 0;
+    });
+    tt.addEventListener('mouseleave', function() {
+      return tooltip_el != null ? tooltip_el.classList.remove('active') : void 0;
+    });
+    ctx.addEventListener('mouseleave', function() {
+      if (tooltip_timeout) {
+        clearTimeout(tooltip_timeout);
+      }
+      return untooltip(0);
+    });
+    graph.addEventListener('mousemove', function(el) {
+      if (tooltip_timeout) {
+        return;
+      }
+      if (!matches(el.target, '.background')) {
+        return;
+      }
+      return untooltip(1000);
+    });
+    tooltip = function(el) {
+      var a, baseline, cls, current_x, current_y, dy, h, i, key, keys, label, legend, name, plot_x, plot_y, rect, serie_index, subval, text, text_group, texts, traversal, value, w, x, x_elt, x_label, xlink, y, y_elt, _l, _len3, _len4, _len5, _m, _n, _ref4, _ref5, _ref6, _ref7, _ref8;
+      clearTimeout(tooltip_timeout);
+      tooltip_timeout = null;
+      tt.style.opacity = 1;
+      tt.style.display = '';
+      text_group = $('g.text', tt).one();
+      rect = $('rect', tt).one();
+      text_group.innerHTML = '';
+      label = sibl(el, '.label').one().textContent;
+      x_label = sibl(el, '.x_label').one().textContent;
+      value = sibl(el, '.value').one().textContent;
+      xlink = sibl(el, '.xlink').one().textContent;
+      serie_index = null;
+      parent = el;
+      traversal = [];
+      while (parent) {
+        traversal.push(parent);
+        if (parent.classList.contains('series')) {
+          break;
+        }
+        parent = parent.parentElement;
+      }
+      if (parent) {
+        _ref4 = parent.classList;
+        for (_l = 0, _len3 = _ref4.length; _l < _len3; _l++) {
+          cls = _ref4[_l];
+          if (cls.indexOf('serie-') === 0) {
+            serie_index = +cls.replace('serie-', '');
+            break;
+          }
+        }
+      }
+      legend = null;
+      if (serie_index !== null) {
+        legend = config.legends[serie_index];
+      }
+      dy = 0;
+      keys = [[label, 'label']];
+      _ref5 = value.split('\n');
+      for (i = _m = 0, _len4 = _ref5.length; _m < _len4; i = ++_m) {
+        subval = _ref5[i];
+        keys.push([subval, 'value-' + i]);
+      }
+      if (config.tooltip_fancy_mode) {
+        keys.push([xlink, 'xlink']);
+        keys.unshift([x_label, 'x_label']);
+        keys.unshift([legend, 'legend']);
+      }
+      texts = {};
+      for (_n = 0, _len5 = keys.length; _n < _len5; _n++) {
+        _ref6 = keys[_n], key = _ref6[0], name = _ref6[1];
+        if (key) {
+          text = document.createElementNS(svg_ns, 'text');
+          text.textContent = key;
+          text.setAttribute('x', padding);
+          text.setAttribute('dy', dy);
+          text.classList.add(name.indexOf('value') === 0 ? 'value' : name);
+          if (name.indexOf('value') === 0 && config.tooltip_fancy_mode) {
+            text.classList.add('color-' + serie_index);
+          }
+          if (name === 'xlink') {
+            a = document.createElementNS(svg_ns, 'a');
+            a.setAttributeNS(xlink_ns, 'href', key);
+            a.textContent = void 0;
+            a.appendChild(text);
+            text.textContent = 'Link >';
+            text_group.appendChild(a);
+          } else {
+            text_group.appendChild(text);
+          }
+          dy += text.getBBox().height + padding / 2;
+          baseline = padding;
+          if (text.style.dominantBaseline !== void 0) {
+            text.style.dominantBaseline = 'text-before-edge';
+          } else {
+            baseline += text.getBBox().height * .8;
+          }
+          text.setAttribute('y', baseline);
+          texts[name] = text;
+        }
+      }
+      w = text_group.getBBox().width + 2 * padding;
+      h = text_group.getBBox().height + 2 * padding;
+      rect.setAttribute('width', w);
+      rect.setAttribute('height', h);
+      if (texts.value) {
+        texts.value.setAttribute('dx', (w - texts.value.getBBox().width) / 2 - padding);
+      }
+      if (texts.x_label) {
+        texts.x_label.setAttribute('dx', w - texts.x_label.getBBox().width - 2 * padding);
+      }
+      if (texts.xlink) {
+        texts.xlink.setAttribute('dx', w - texts.xlink.getBBox().width - 2 * padding);
+      }
+      x_elt = sibl(el, '.x').one();
+      y_elt = sibl(el, '.y').one();
+      x = parseInt(x_elt.textContent);
+      if (x_elt.classList.contains('centered')) {
+        x -= w / 2;
+      } else if (x_elt.classList.contains('left')) {
+        x -= w;
+      } else if (x_elt.classList.contains('auto')) {
+        x = xconvert(el.getBBox().x + el.getBBox().width / 2) - w / 2;
+      }
+      y = parseInt(y_elt.textContent);
+      if (y_elt.classList.contains('centered')) {
+        y -= h / 2;
+      } else if (y_elt.classList.contains('top')) {
+        y -= h;
+      } else if (y_elt.classList.contains('auto')) {
+        y = yconvert(el.getBBox().y + el.getBBox().height / 2) - h / 2;
+      }
+      _ref7 = get_translation(tt.parentElement), plot_x = _ref7[0], plot_y = _ref7[1];
+      if (x + w + plot_x > config.width) {
+        x = config.width - w - plot_x;
+      }
+      if (y + h + plot_y > config.height) {
+        y = config.height - h - plot_y;
+      }
+      if (x + plot_x < 0) {
+        x = -plot_x;
+      }
+      if (y + plot_y < 0) {
+        y = -plot_y;
+      }
+      _ref8 = get_translation(tt), current_x = _ref8[0], current_y = _ref8[1];
+      if (current_x === x && current_y === y) {
+        return el;
+      }
+      tt.setAttribute('transform', "translate(" + x + " " + y + ")");
+      return el;
+    };
+    return untooltip = function(ms) {
+      return tooltip_timeout = setTimeout(function() {
+        tt.style.display = 'none';
+        tt.style.opacity = 0;
+        if (tooltip_el != null) {
+          tooltip_el.classList.remove('active');
+        }
+        return tooltip_timeout = null;
+      }, ms);
+    };
+  };
+
+  init_svg = function() {
+    var chart, charts, _i, _len, _results;
+    charts = $('.pygal-chart');
+    if (charts.length) {
+      _results = [];
+      for (_i = 0, _len = charts.length; _i < _len; _i++) {
+        chart = charts[_i];
+        _results.push(init(chart));
+      }
+      return _results;
+    }
+  };
+
+  if (document.readyState !== 'loading') {
+    init_svg();
+  } else {
+    document.addEventListener('DOMContentLoaded', function() {
+      return init_svg();
+    });
+  }
+
+  window.pygal = window.pygal || {};
+
+  window.pygal.init = init;
+
+  window.pygal.init_svg = init_svg;
+
+}).call(this);

combo/apps/dataviz/templates/combo/chart-filters.html

@@ -0,0 +1,63 @@
+{% load i18n gadjo %}
+
+{% block cell-content %}
+  <h2>{{ cell.title }}</h2>
+
+  <div>
+    {% if form.fields|length > 1 %}
+      <form method='get' enctype='multipart/form-data' id='chart-filters'>
+        {{ form|with_template }}
+        <div class='buttons'>
+          <button class='submit-button'>{% trans 'Refresh' %}</button>
+        </div>
+      </form>
+    {% else %}
+      <p>
+        {% blocktrans trimmed %}
+          No filters are available. Note that only filters that are shared between all chart cells will appear. Furthermore, in case they have a value, it must be the same accross all cells.
+        {% endblocktrans %}
+      </p>
+    {% endif %}
+  </div>
+
+
+  <script>
+    $(function () {
+      const form = $('#chart-filters');
+      var loaded_cell_count = 0;
+
+      if (!$('body').data('filters-cell-id')) {
+        $('body').data('filters-cell-id', Math.random().toString(36).slice(2, 7));
+
+        function load_filters_cell_last() {
+          if (++loaded_cell_count == $('div.chartngcell').length) {
+            combo_load_cell($('.chart-filters-cell'));
+            loaded_cell_count = 0;
+          }
+        }
+
+        document.querySelectorAll('div.dataviz-table').forEach(graph => {
+          graph.addEventListener('DOMSubtreeModified', load_filters_cell_last);
+        });
+        document.querySelectorAll('div.chartngcell embed').forEach(graph => {
+          graph.addEventListener('load', load_filters_cell_last)
+        });
+      }
+
+      form.submit(function(e) {
+        e.preventDefault();
+        $(window).trigger('combo:refresh-graphs');
+        chart_cell = $(this).parents('.cell');
+        ajax_cell_url = "{{ site_base }}{{ cell.get_ajax_url }}";
+        new_url = ajax_cell_url + '?filters_cell_id=' + $('body').data('filters-cell-id') + '&' + $(this).serialize();
+        chart_cell.data('ajax-cell-url', new_url);
+      });
+
+      form.change(function() {
+        if(loaded_cell_count == 0) {
+          form.submit();
+        }
+      });
+    });
+  </script>
+{% endblock %}

combo/apps/dataviz/templates/combo/chartfilterscell_form.html

@@ -0,0 +1,6 @@
+{% load gadjo %}
+
+{% block cell-form %}
+  {{form|with_template}}
+{% endblock %}
+

combo/apps/dataviz/templates/combo/chartngcell.html

@@ -0,0 +1,71 @@
+{% load i18n %}
+{% if cell.title %}<h2>{{cell.title}}</h2>{% endif %}
+{% if cell.is_table_chart %}
+  <div id="chart-{{cell.id}}" class="dataviz-table total-{{ cell.display_total }}"></div>
+  <script>
+    $(function() {
+      var extra_context = $('#chart-{{cell.id}}').parents('.cell').data('extra-context');
+      $(window).on('combo:refresh-graphs', function() {
+        var url = "{% url 'combo-dataviz-graph' cell=cell.id %}" + get_graph_querystring(extra_context);
+        $('#chart-{{cell.id}}-download').attr('href', url + '&export-format=ods');
+        $.ajax({
+          url : url,
+          type: 'GET',
+          success: function(data) {
+            $('#chart-{{cell.id}}').html(data);
+          }
+        });
+      }).trigger('combo:refresh-graphs');
+    });
+  </script>
+{% else %}
+  <div style="min-height: {{cell.height}}px">
+    <embed id="chart-{{cell.id}}" type="image/svg+xml" style="width: 100%"/>
+  </div>
+  <script>
+    $(function() {
+      var last_width = 1;
+      var extra_context = $('#chart-{{cell.id}}').parents('.cell').data('extra-context');
+      $(window).on('load resize gadjo:sidepage-toggled combo:resize-graphs', function() {
+        var chart_cell = $('#chart-{{cell.id}}').parent();
+        var new_width = Math.floor($(chart_cell).width());
+        var ratio = new_width / last_width;
+        if (ratio > 1.2 || ratio < 0.8) {
+          var querystring = get_graph_querystring(extra_context, new_width);
+          $('#chart-{{cell.id}}').attr('src', "{% url 'combo-dataviz-graph' cell=cell.id %}" + querystring);
+          $('#chart-{{cell.id}}-download').attr('href', "{% url 'combo-dataviz-graph-export' cell=cell.id %}" + querystring);
+          last_width = new_width;
+        }
+      }).trigger('combo:resize-graphs');
+      $(window).on('combo:refresh-graphs', function() {
+        var querystring = get_graph_querystring(extra_context, last_width);
+        $('#chart-{{cell.id}}').attr('src', "{% url 'combo-dataviz-graph' cell=cell.id %}" + querystring);
+        $('#chart-{{cell.id}}-download').attr('href', "{% url 'combo-dataviz-graph-export' cell=cell.id %}" + querystring);
+      });
+    });
+  </script>
+{% endif %}
+
+<a
+  class="button download-button"
+  id="chart-{{ cell.id }}-download"
+  title="{% trans "Download" %}"
+  href="{% url 'combo-dataviz-graph-export' cell=cell.id %}"
+  {% if cell.is_table_chart %}
+    download
+  {% else %}
+    rel="popup"
+    data-autoclose-dialog="true"
+  {% endif %}
+>
+  <span class="sr-only">{% trans "Download" %}</span>
+</a>
+<script>
+  $(function() {
+    $('#chart-{{cell.id}}').parents('.cell').on('mouseenter', function() {
+      $('#chart-{{ cell.id }}-download').show();
+    }).on('mouseleave', function() {
+      $('#chart-{{ cell.id }}-download').hide();
+    }).trigger('mouseleave');
+  });
+</script>

combo/apps/dataviz/templates/combo/chartngcell_export_form.html

@@ -0,0 +1,17 @@
+{% extends "combo/manager_base.html" %}
+{% load i18n %}
+
+{% block appbar %}
+  <h2>{% trans "Export data" %}</h2>
+{% endblock %}
+
+{% block content %}
+  <form method="post">
+    {% csrf_token %}
+    {{ form.as_p }}
+    <div class="buttons">
+      <button class="submit-button">{% trans "Download" %}</button>
+      <a class="cancel" href="{% url 'combo-manager-page-view' pk=object.pk %}">{% trans 'Cancel' %}</a>
+    </div>
+  </form>
+{% endblock %}

combo/apps/dataviz/templates/combo/chartngcell_form.html

@@ -0,0 +1,18 @@
+{% load gadjo %}
+
+<div style="position: relative">
+  {{ form|with_template }}
+  {% if cell.statistic and not cell.is_table_chart %}
+    <div style="position: absolute; right: 0; top: 0; width: 300px; height: 150px">
+      <embed type="image/svg+xml" src="{% url 'combo-dataviz-graph' cell=cell.id %}?width=300&height=150"/>
+    </div>
+  {% endif %}
+</div>
+
+<script>
+  $(function () {
+    $('div#panel-dataviz_chartngcell-{{ cell.pk }}-general div.content').change(function() {
+      $('div#cell-dataviz_chartngcell-{{ cell.pk }} button.save').click();
+    });
+  });
+</script>

combo/apps/dataviz/templates/combo/dataviz-error.svg

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg xmlns="http://www.w3.org/2000/svg" version="1.1"
+  viewBox="0 0 {{ width }} 30" width="{{ width }}" height="30">
+  <text
+       y="20"
+       x="10"
+       style="font-family: sans-serif; font-size: 16px; fill:#000000;">{{ text }}</text>
+</svg>

combo/apps/dataviz/templates/combo/gauge-cell.html

@@ -1,16 +1,16 @@
 {% block cell-content %}
-<div
-     data-combo-gauge="true"
-     {% if jsonp %}
-        data-gauge-count-jsonp-url="{{data_source_url}}"
-     {% else %}
-        data-gauge-count-url="{% url 'combo-ajax-gauge-count' cell=cell.id %}"
-     {% endif %}
-     data-gauge-max-value="{{max_value|default_if_none:100}}" class="bo-block">
-  <canvas style="width: 100%;">
-  </canvas>
-  {% if title %}
-    {% if url %}<a href="{{url}}">{% endif %}{{title}}{% if url %}</a>{% endif %}
-  {% endif %}
-</div>
+  <div
+    data-combo-gauge="true"
+    {% if jsonp %}
+      data-gauge-count-jsonp-url="{{data_source_url}}"
+    {% else %}
+      data-gauge-count-url="{% url 'combo-ajax-gauge-count' cell=cell.id %}"
+    {% endif %}
+    data-gauge-max-value="{{max_value|default_if_none:100}}" class="bo-block">
+    <canvas style="width: 100%;">
+    </canvas>
+    {% if title %}
+      {% if url %}<a href="{{url}}">{% endif %}{{title}}{% if url %}</a>{% endif %}
+    {% endif %}
+  </div>
 {% endblock %}

combo/apps/dataviz/urls.py

@@ -14,11 +14,21 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from django.conf.urls import url
+from django.urls import path, re_path
 
-from .views import ajax_gauge_count
+from combo.urls_utils import manager_required
+
+from .views import ajax_gauge_count, dataviz_choices, dataviz_graph, dataviz_graph_export
 
 urlpatterns = [
-    url(r'^ajax/gauge-count/(?P<cell>[\w_-]+)/$',
-        ajax_gauge_count, name='combo-ajax-gauge-count'),
+    re_path(r'^ajax/gauge-count/(?P<cell>[\w_-]+)/$', ajax_gauge_count, name='combo-ajax-gauge-count'),
+    re_path(r'^api/dataviz/graph/(?P<cell>[\w_-]+)/$', dataviz_graph, name='combo-dataviz-graph'),
+    re_path(
+        r'^dataviz/graph/(?P<cell>[\w_-]+)/export/$', dataviz_graph_export, name='combo-dataviz-graph-export'
+    ),
+    path(
+        'api/dataviz/graph/<int:cell_id>/<filter_id>/ajax-choices',
+        manager_required(dataviz_choices),
+        name='combo-dataviz-choices',
+    ),
 ]

combo/apps/dataviz/utils.py

@@ -0,0 +1,122 @@
+import datetime
+import json
+import logging
+
+from django.conf import settings
+from django.utils.timezone import now
+from requests.exceptions import RequestException
+
+from combo.utils import requests
+
+from .models import Statistic
+
+logger = logging.getLogger('combo.apps.dataviz')
+
+
+def update_available_statistics():
+    if not settings.KNOWN_SERVICES:
+        return
+
+    results = []
+    temporary_unavailable_sites = []
+    for provider in settings.STATISTICS_PROVIDERS:
+        if isinstance(provider, dict):
+            url = provider['url']
+            sites = {provider['id']: {'title': provider['name']}}
+            provider = provider['id']
+        else:
+            sites = settings.KNOWN_SERVICES.get(provider, {})
+            url = '/visualization/json/' if provider == 'bijoe' else '/api/statistics/'
+
+        for site_key, site_dict in sites.items():
+            try:
+                response = requests.get(
+                    url,
+                    allow_redirects=False,
+                    timeout=5,
+                    remote_service=site_dict if provider in settings.KNOWN_SERVICES else {},
+                    without_user=True,
+                    headers={'accept': 'application/json'},
+                    log_errors='warn',
+                )
+                response.raise_for_status()
+            except RequestException:
+                temporary_unavailable_sites.append((provider, site_key))
+                continue
+
+            try:
+                result = response.json()
+            except json.JSONDecodeError:
+                continue
+            if isinstance(result, dict):
+                result = result['data']  # detect new api
+
+            for stat in result:
+                results.append(
+                    Statistic(
+                        slug=stat.get('slug') or stat['id'],
+                        site_slug=site_key,
+                        service_slug=provider,
+                        label=stat['name'],
+                        url=stat.get('data-url') or stat['url'],
+                        site_title=site_dict.get('title', ''),
+                        filters=stat.get('filters', []),
+                        has_future_data=stat.get('future_data', False),
+                        data_type=stat.get('data_type', ''),
+                        deprecated=stat.get('deprecated', False),
+                        available=True,
+                    )
+                )
+
+    update_fields = (
+        'label',
+        'url',
+        'site_title',
+        'filters',
+        'available',
+        'has_future_data',
+        'data_type',
+        'deprecated',
+    )
+    all_statistics = {stat.natural_key(): stat for stat in Statistic.objects.all()}
+    statistics_to_create = []
+    statistics_to_update = {}
+    for stat in results:
+        existing_stat = all_statistics.get(stat.natural_key())
+        if existing_stat:
+            for field in update_fields:
+                new_value = getattr(stat, field)
+                if getattr(existing_stat, field) != new_value:
+                    setattr(existing_stat, field, new_value)
+                    statistics_to_update[existing_stat.pk] = existing_stat
+        else:
+            statistics_to_create.append(stat)
+
+    Statistic.objects.bulk_create(statistics_to_create)
+    Statistic.objects.bulk_update(statistics_to_update.values(), update_fields)
+
+    available_stats = Statistic.objects.filter(available=True)
+    for stat in results:
+        available_stats = available_stats.exclude(
+            slug=stat.slug, site_slug=stat.site_slug, service_slug=stat.service_slug
+        )
+
+    # set last_update on all seen statistics
+    Statistic.objects.exclude(pk__in=available_stats).update(last_update=now())
+
+    for service_slug, site_slug in temporary_unavailable_sites:
+        available_stats = available_stats.exclude(site_slug=site_slug, service_slug=service_slug)
+    available_stats.update(available=False)
+
+    # log errors for outdated statistics
+    sites_with_outdated_statistics = set()
+    outdated_hours = 48
+    for available_stat in Statistic.objects.filter(available=True):
+        time_since_last_update = now() - available_stat.last_update
+        if time_since_last_update > datetime.timedelta(hours=outdated_hours):
+            sites_with_outdated_statistics.add(available_stat.site_title)
+
+    for title in sites_with_outdated_statistics:
+        logger.error(
+            f'statistics from "{title}" have not been available for more than %s hours.', outdated_hours
+        )

combo/apps/dataviz/views.py

@@ -1,5 +1,5 @@
 # combo - content management system
-# Copyright (C) 2015  Entr'ouvert
+# Copyright (C) 2015-2019  Entr'ouvert
 #
 # This program is free software: you can redistribute it and/or modify it
 # under the terms of the GNU Affero General Public License as published
@@ -14,13 +14,234 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from django.http import HttpResponse
+import io
+import unicodedata
 
-from combo.utils import get_templated_url, requests
-from .models import Gauge
+import pyexcel_ods
+from django.core import signing
+from django.core.cache import cache
+from django.core.exceptions import PermissionDenied
+from django.http import FileResponse, Http404, HttpResponse, HttpResponseBadRequest, JsonResponse
+from django.shortcuts import render, reverse
+from django.template import TemplateSyntaxError, VariableDoesNotExist
+from django.utils.translation import gettext_lazy as _
+from django.views.decorators.clickjacking import xframe_options_sameorigin
+from django.views.generic import DetailView, FormView
+from django.views.generic.detail import SingleObjectMixin
+from requests.exceptions import HTTPError
+
+from combo.utils import NothingInCacheException, get_templated_url, requests
+
+from .forms import ChartFiltersMixin, ChartNgExportForm, ChartNgPartialForm, Choice
+from .models import ChartNgCell, Gauge, MissingVariable, UnsupportedDataSet
 
 
 def ajax_gauge_count(request, *args, **kwargs):
     gauge = Gauge.objects.get(id=kwargs['cell'])
     response = requests.get(get_templated_url(gauge.data_source))
     return HttpResponse(response.content, content_type='text/json')
+
+
+class DatavizGraphView(DetailView):
+    model = ChartNgCell
+    pk_url_kwarg = 'cell'
+
+    def dispatch(self, request, *args, **kwargs):
+        self.cell = self.get_object()
+        self.filters_cell_id = request.GET.get('filters_cell_id')
+
+        if not self.cell.page.is_visible(request.user):
+            raise PermissionDenied()
+        if not self.cell.is_visible(request):
+            raise PermissionDenied()
+        if not self.cell.statistic or not self.cell.statistic.url:
+            raise Http404('misconfigured cell')
+
+        if self.filters_cell_id:
+            self.cell.subfilters = cache.get(
+                self.cell.get_cache_key(self.filters_cell_id), self.cell.subfilters
+            )
+        return super().dispatch(request, *args, **kwargs)
+
+    def get(self, request, *args, **kwargs):
+        form = ChartNgPartialForm(request.GET, instance=self.cell)
+        if not form.is_valid():
+            return self.error(_('Wrong parameters.'))
+
+        if request.GET.get('ctx'):
+            try:
+                request.extra_context = signing.loads(request.GET['ctx'])
+            except signing.BadSignature:
+                return HttpResponseBadRequest('bad signature')
+
+        form.instance._request = request
+        try:
+            chart = form.instance.get_chart(
+                width=int(request.GET['width']) if request.GET.get('width') else None,
+                height=int(request.GET['height']) if request.GET.get('height') else None,
+            )
+        except UnsupportedDataSet:
+            return self.error(_('Unsupported dataset.'))
+        except MissingVariable:
+            return self.error(_('Page variable not found.'))
+        except TemplateSyntaxError:
+            return self.error(_('Syntax error in page variable.'))
+        except VariableDoesNotExist:
+            return self.error(_('Backoffice preview unavailable.'))
+        except HTTPError as e:
+            if e.response.status_code == 404:
+                return self.error(_('Visualization not found.'))
+            else:
+                return self.error(_('Unknown HTTP error: %s' % e))
+
+        if self.filters_cell_id and self.cell.statistic.service_slug != 'bijoe':
+            self.update_subfilters_cache(form.instance)
+
+        export_format = request.GET.get('export-format')
+        if export_format == 'svg':
+            return self.export_to_svg(chart)
+        elif export_format == 'ods':
+            return self.export_to_ods(chart)
+
+        if self.cell.is_table_chart():
+            if not chart.raw_series:
+                return self.error(_('No data'))
+
+            # force x_value_formatter on x_labels (avoid bug https://github.com/Kozea/pygal/issues/372)
+            chart.x_labels = [chart.config.x_value_formatter(x) for x in chart.x_labels]
+
+            rendered = chart.render_table(
+                transpose=bool(self.cell.chart_type == 'table-inverted'),
+                total=bool(self.cell.statistic.service_slug == 'bijoe' and chart.compute_sum),
+            )
+            rendered = rendered.replace('<table>', '<table class="main">')
+            return HttpResponse(rendered)
+
+        return HttpResponse(chart.render(), content_type='image/svg+xml')
+
+    def error(self, error_text):
+        if self.cell.is_table_chart():
+            return HttpResponse('<p>%s</p>' % error_text)
+
+        context = {
+            'width': self.request.GET.get('width', 200),
+            'text': error_text,
+        }
+        return render(self.request, 'combo/dataviz-error.svg', context=context, content_type='image/svg+xml')
+
+    def update_subfilters_cache(self, cell):
+        try:
+            data = cell.get_statistic_data(raise_if_not_cached=True)
+        except NothingInCacheException:
+            pass  # should not happen
+        else:
+            cache.set(
+                cell.get_cache_key(self.filters_cell_id), data.json()['data'].get('subfilters', []), 300
+            )
+
+    def export_to_svg(self, chart):
+        response = HttpResponse(chart.render(), content_type='image/svg+xml')
+        response['Content-Disposition'] = 'attachment; filename="%s.svg"' % self.cell.get_download_filename()
+        return response
+
+    def export_to_ods(self, chart):
+        data = [[''] + chart.x_labels] if any(chart.x_labels) else []
+        for serie in chart.raw_series:
+            line = [serie[1]['title']] + serie[0]
+            line = [x or 0 for x in line]
+            data.append(line)
+
+        data = [list(line) for line in zip(*data)]
+
+        output = io.BytesIO()
+        pyexcel_ods.save_data(output, {self.cell.title or self.cell.statistic.label: data})
+        output.seek(0)
+        return FileResponse(
+            output,
+            as_attachment=True,
+            content_type='application/vnd.oasis.opendocument.spreadsheet',
+            filename='%s.ods' % self.cell.get_download_filename(),
+        )
+
+
+dataviz_graph = xframe_options_sameorigin(DatavizGraphView.as_view())
+
+
+class DatavizGraphExportView(SingleObjectMixin, FormView):
+    model = ChartNgCell
+    pk_url_kwarg = 'cell'
+    form_class = ChartNgExportForm
+    template_name = 'combo/chartngcell_export_form.html'
+
+    def dispatch(self, *args, **kwargs):
+        self.object = self.get_object()
+        return super().dispatch(*args, **kwargs)
+
+    def form_valid(self, form):
+        self.querystring = self.request.GET.copy()
+        self.querystring['export-format'] = form.cleaned_data['export_format']
+        return super().form_valid(form)
+
+    def get_success_url(self):
+        return '%s?%s' % (
+            reverse('combo-dataviz-graph', kwargs={'cell': self.object.pk}),
+            self.querystring.urlencode(),
+        )
+
+
+dataviz_graph_export = DatavizGraphExportView.as_view()
+
+
+class DatavizChoicesView(DetailView):
+    model = ChartNgCell
+    pk_url_kwarg = 'cell_id'
+
+    def dispatch(self, *args, **kwargs):
+        self.cell = self.get_object()
+
+        filter_id = self.kwargs.get('filter_id')
+        for filter_ in self.cell.available_filters:
+            if filter_['id'] == filter_id:
+                self.filter = filter_
+                break
+        else:
+            raise Http404()
+
+        return super().dispatch(*args, **kwargs)
+
+    def get(self, request, *args, **kwargs):
+        search_term = request.GET.get('term', '')
+        search_term = unicodedata.normalize('NFKC', search_term).casefold()
+
+        try:
+            page_number = int(request.GET.get('page', 1))
+        except ValueError:
+            page_number = 1
+
+        initial = self.cell.filter_params.get(self.filter['id'], self.filter.get('default'))
+        objects = ChartFiltersMixin.get_filter_options(self.cell, self.filter, initial)
+        objects = [x for x in objects if search_term in unicodedata.normalize('NFKC', str(x)).casefold()]
+
+        return JsonResponse(
+            {
+                'results': self.format_results(objects, (page_number - 1) * 10, page_number * 10),
+                'pagination': {'more': bool(len(objects) >= page_number * 10)},
+            }
+        )
+
+    def format_results(self, objects, start_index, end_index):
+        page_objects = objects[start_index:end_index]
+
+        if start_index > 0:
+            last_displayed_group = objects[start_index - 1].group
+            for option in page_objects:
+                if option.group == last_displayed_group:
+                    option.group = None
+
+        return [
+            {'text': group, 'children': [{'id': k, 'text': v} for k, v in choices]}
+            for group, choices in Choice.get_field_choices(page_objects)
+        ]
+
+
+dataviz_choices = DatavizChoicesView.as_view()

combo/apps/dataviz/widgets.py

@@ -0,0 +1,64 @@
+# combo - content management system
+# Copyright (C) 2014-2022  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.conf import settings
+from django.forms.widgets import Select, SelectMultiple
+from django.urls import reverse
+
+
+class Select2WidgetMixin:
+    class Media:
+        js = 'xstatic/select2.min.js'
+        css = {'all': ('xstatic/select2.min.css',)}
+
+    def __init__(self, cell, filter_id, choices, initial, ajax_choices):
+        from .forms import Choice
+
+        attrs = {}
+        if self.enable_select2(choices):
+            attrs['data-combo-autocomplete'] = 'true'
+            attrs['lang'] = settings.LANGUAGE_CODE
+            if ajax_choices:
+                attrs['data-select2-url'] = reverse(
+                    'combo-dataviz-choices', kwargs={'cell_id': cell.pk, 'filter_id': filter_id}
+                )
+                choices = self.filter_choices(choices, initial)
+
+        super().__init__(choices=Choice.get_field_choices(choices), attrs=attrs)
+
+    def enable_select2(self, choices):
+        return True
+
+
+class Select2Widget(Select2WidgetMixin, Select):
+    min_choices = 20
+
+    @staticmethod
+    def filter_choices(choices, initial):
+        return [x for x in choices if x.id == initial]
+
+    def enable_select2(self, choices):
+        return bool(len(choices) > self.min_choices)
+
+
+class MultipleSelect2Widget(Select2WidgetMixin, SelectMultiple):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.attrs['multiple'] = 'multiple'
+
+    @staticmethod
+    def filter_choices(choices, initial):
+        return [x for x in choices if x.id in (initial or [])]