dataviz: allow setting time range using template (#57617)

combo/apps/dataviz/forms.py

@@ -14,12 +14,14 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import datetime
 from collections import OrderedDict
 
 from django import forms
 from django.conf import settings
 from django.db import transaction
 from django.db.models import Q
+from django.template import Context, Template, TemplateSyntaxError, VariableDoesNotExist
 from django.utils.translation import ugettext_lazy as _
 
 from combo.utils import cache_during_request, requests, spooler
@@ -69,6 +71,8 @@ class ChartNgForm(forms.ModelForm):
             'time_range',
             'time_range_start',
             'time_range_end',
+            'time_range_start_template',
+            'time_range_end_template',
             'chart_type',
             'height',
             'sort_order',
@@ -85,9 +89,14 @@ class ChartNgForm(forms.ModelForm):
 
         field_ids = list(self._meta.fields)
         if not self.instance.statistic or self.instance.statistic.service_slug == 'bijoe':
-            field_ids = [
+            exclude = (
-                x for x in field_ids if x not in ('time_range', 'time_range_start', 'time_range_end')
+                'time_range',
-            ]
+                'time_range_start',
+                'time_range_end',
+                'time_range_start_template',
+                'time_range_end_template',
+            )
+            field_ids = [x for x in field_ids if x not in exclude]
 
         stat_field = self.fields['statistic']
         if not self.instance.statistic:
@@ -142,3 +151,18 @@ class ChartNgForm(forms.ModelForm):
         for choice in self.time_intervals:
             if choice[0].strip('_') not in choice_ids:
                 self.fields['time_interval'].choices.append(choice)
+
+    def clean(self):
+        for template_field in ('time_range_start_template', 'time_range_end_template'):
+            if not self.cleaned_data.get(template_field):
+                continue
+            context = {'now': datetime.datetime.now, 'today': datetime.datetime.now}
+            try:
+                date = Template('{{ %s|date:"Y-m-d" }}' % self.cleaned_data[template_field]).render(
+                    Context(context)
+                )
+            except (VariableDoesNotExist, TemplateSyntaxError) as e:
+                self.add_error(template_field, e)
+            else:
+                if not date:
+                    self.add_error(template_field, _('Template does not evaluate to a valid date.'))

combo/apps/dataviz/migrations/0019_auto_20211006_1525.py

@@ -0,0 +1,36 @@
+# Generated by Django 2.2.19 on 2021-10-06 13:25
+
+from django.db import migrations, models
+
+import combo.data.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dataviz', '0018_auto_20210723_1318'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='chartngcell',
+            name='time_range_end_template',
+            field=models.CharField(
+                blank=True,
+                max_length=200,
+                validators=[combo.data.models.django_template_validator],
+                verbose_name='To',
+            ),
+        ),
+        migrations.AddField(
+            model_name='chartngcell',
+            name='time_range_start_template',
+            field=models.CharField(
+                blank=True,
+                max_length=200,
+                validators=[combo.data.models.django_template_validator],
+                verbose_name='From',
+                help_text='Template code returning a date. For example, Monday in two weeks would be today|add_days:"14"|adjust_to_week_monday.',
+            ),
+        ),
+    ]

combo/apps/dataviz/models.py

@@ -25,6 +25,7 @@ from dateutil.relativedelta import MO, relativedelta
 from django.conf import settings
 from django.contrib.postgres.fields import JSONField
 from django.db import models, transaction
+from django.template import Context, Template
 from django.template.defaultfilters import date as format_date
 from django.urls import reverse
 from django.utils import timezone
@@ -36,7 +37,7 @@ from django.utils.translation import ungettext
 from requests.exceptions import HTTPError, RequestException
 
 from combo.data.library import register_cell_class
-from combo.data.models import CellBase
+from combo.data.models import CellBase, django_template_validator
 from combo.utils import get_templated_url, requests, spooler
 
 
@@ -158,7 +159,8 @@ TIME_FILTERS = (
     ('previous-week', _('Previous week')),
     ('current-week', _('Current week')),
     ('next-week', _('Next week')),
-    ('range', _('Free range')),
+    ('range', _('Free range (date)')),
+    ('range-template', _('Free range (template)')),
 )
 
 
@@ -185,6 +187,21 @@ class ChartNgCell(CellBase):
     )
     time_range_start = models.DateField(_('From'), null=True, blank=True)
     time_range_end = models.DateField(_('To'), null=True, blank=True)
+    time_range_start_template = models.CharField(
+        _('From'),
+        max_length=200,
+        blank=True,
+        validators=[django_template_validator],
+        help_text=_(
+            'Template code returning a date. For example, Monday in two weeks would be today|add_days:"14"|adjust_to_week_monday.'
+        ),
+    )
+    time_range_end_template = models.CharField(
+        _('To'),
+        max_length=200,
+        blank=True,
+        validators=[django_template_validator],
+    )
     chart_type = models.CharField(
         _('Chart Type'),
         max_length=20,
@@ -394,6 +411,16 @@ class ChartNgCell(CellBase):
                 params['start'] = self.time_range_start
             if self.time_range_end:
                 params['end'] = self.time_range_end
+        elif self.time_range == 'range-template':
+            context = {'now': datetime.now, 'today': datetime.now}
+            if self.time_range_start_template:
+                params['start'] = Template('{{ %s|date:"Y-m-d" }}' % self.time_range_start_template).render(
+                    Context(context)
+                )
+            if self.time_range_end_template:
+                params['end'] = Template('{{ %s|date:"Y-m-d" }}' % self.time_range_end_template).render(
+                    Context(context)
+                )
         if 'time_interval' in params and params['time_interval'].startswith('_'):
             params['time_interval'] = 'day'
         return params

combo/apps/dataviz/templates/combo/chartngcell_form.html

@@ -11,6 +11,8 @@
 	$(function () {
 		start_field = $('#id_cdataviz_chartngcell-{{ cell.pk }}-time_range_start');
 		end_field = $('#id_cdataviz_chartngcell-{{ cell.pk }}-time_range_end');
+		start_field_template = $('#id_cdataviz_chartngcell-{{ cell.pk }}-time_range_start_template');
+		end_field_template = $('#id_cdataviz_chartngcell-{{ cell.pk }}-time_range_end_template');
 		$('#id_cdataviz_chartngcell-{{ cell.pk }}-time_range').change(function() {
 			if(this.value == 'range') {
 				start_field.parent().show();
@@ -19,6 +21,13 @@
 				start_field.parent().hide();
 				end_field.parent().hide();
 			}
+			if(this.value == 'range_template') {
+				start_field_template.parent().show();
+				end_field_template.parent().show();
+			} else {
+				start_field_template.parent().hide();
+				end_field_template.parent().hide();
+			}
 		}).change();
 	});
 </script>

combo/context_processors.py

@@ -14,7 +14,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
 from django.conf import settings
 
 from combo.apps.pwa.models import PwaSettings

tests/test_dataviz.py

@@ -1098,6 +1098,8 @@ def test_chartng_cell_manager(app, admin_user, statistics):
     assert 'time_range' not in resp.form.fields
     assert 'time_range_start' not in resp.form.fields
     assert 'time_range_end' not in resp.form.fields
+    assert 'time_range_start_template' not in resp.form.fields
+    assert 'time_range_end_template_end' not in resp.form.fields
 
     cell.statistic = Statistic.objects.get(slug='example')
     cell.save()
@@ -1111,6 +1113,8 @@ def test_chartng_cell_manager(app, admin_user, statistics):
     assert 'time_range' not in resp.form.fields
     assert 'time_range_start' not in resp.form.fields
     assert 'time_range_end' not in resp.form.fields
+    assert 'time_range_start_template' not in resp.form.fields
+    assert 'time_range_end_template_end' not in resp.form.fields
 
     cell.statistic = Statistic.objects.get(slug='unavailable-stat')
     cell.save()
@@ -1200,6 +1204,49 @@ def test_chartng_cell_manager_new_api(app, admin_user, new_api_statistics):
     assert cell.time_range == ''
 
 
+@with_httmock(new_api_mock)
+@pytest.mark.freeze_time('2021-10-06')
+def test_chartng_cell_manager_new_api_time_range_templates(app, admin_user, new_api_statistics):
+    page = Page.objects.create(title='One', slug='index')
+    cell = ChartNgCell(page=page, order=1, placeholder='content')
+    cell.statistic = Statistic.objects.get(slug='one-serie')
+    cell.save()
+
+    app = login(app)
+    resp = app.get('/manage/pages/%s/' % page.id)
+    field_prefix = 'cdataviz_chartngcell-%s-' % cell.id
+
+    resp.form[field_prefix + 'time_range'] = 'range-template'
+    resp.form[field_prefix + 'time_range_start_template'] = 'today|add_days:"7"|adjust_to_week_monday'
+    resp.form[field_prefix + 'time_range_end_template'] = 'now|add_days:"14"|adjust_to_week_monday'
+    resp = resp.form.submit().follow()
+    cell.refresh_from_db()
+    assert cell.time_range == 'range-template'
+    assert cell.time_range_start_template == 'today|add_days:"7"|adjust_to_week_monday'
+    assert cell.time_range_end_template == 'now|add_days:"14"|adjust_to_week_monday'
+
+    resp.form[field_prefix + 'time_range_start_template'] = ''
+    resp.form[field_prefix + 'time_range_end_template'] = ''
+    resp = resp.form.submit().follow()
+    cell.refresh_from_db()
+    assert cell.time_range_start_template == ''
+    assert cell.time_range_end_template == ''
+
+    resp.form[field_prefix + 'time_range_start_template'] = 'xxx'
+    resp = resp.form.submit()
+    assert 'Template does not evaluate to a valid date.' in resp.text
+
+    resp = app.get('/manage/pages/%s/' % page.id)
+    resp.form[field_prefix + 'time_range_start_template'] = 'today|xxx'
+    resp = resp.form.submit()
+    assert 'Invalid filter' in resp.text
+
+    resp = app.get('/manage/pages/%s/' % page.id)
+    resp.form[field_prefix + 'time_range_start_template'] = 'today|date:xxx'
+    resp = resp.form.submit()
+    assert 'Failed lookup for key [xxx]' in resp.text
+
+
 @with_httmock(new_api_mock)
 def test_chartng_cell_manager_new_api_dynamic_fields(app, admin_user, new_api_statistics):
     page = Page.objects.create(title='One', slug='index')
@@ -1435,6 +1482,26 @@ def test_chartng_cell_new_api_filter_params(new_api_statistics, nocache, freezer
     request = new_api_mock.call['requests'][-1]
     assert 'start=2020-10-01' in request.url and 'end=2020-11-03' in request.url
 
+    cell.time_range = 'range-template'
+    cell.save()
+    cell.get_chart()
+    request = new_api_mock.call['requests'][-1]
+    assert 'start' not in urllib.parse.parse_qs(urllib.parse.urlparse(request.url).query)
+    assert 'end' not in urllib.parse.parse_qs(urllib.parse.urlparse(request.url).query)
+
+    cell.time_range_start_template = 'today|add_days:"7"|adjust_to_week_monday'
+    cell.save()
+    cell.get_chart()
+    request = new_api_mock.call['requests'][-1]
+    assert 'start=2020-03-09' in request.url
+    assert 'end' not in urllib.parse.parse_qs(urllib.parse.urlparse(request.url).query)
+
+    cell.time_range_end_template = 'today|add_days:"14"|adjust_to_week_monday'
+    cell.save()
+    cell.get_chart()
+    request = new_api_mock.call['requests'][-1]
+    assert 'start=2020-03-09' in request.url and 'end=2020-03-16' in request.url
+
 
 @with_httmock(new_api_mock)
 def test_chartng_cell_new_api_filter_params_month(new_api_statistics, nocache, freezer):