dataviz: use select2 widget for all filters (#71885) #83

Open
vdeniaud wants to merge 3 commits from wip/71885-select2-dataviz into main
6 changed files with 355 additions and 94 deletions

View File

@ -14,8 +14,9 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import dataclasses
import datetime
from collections import OrderedDict
from collections import OrderedDict, defaultdict
from django import forms
from django.conf import settings
@ -25,12 +26,27 @@ from django.db.models import Q
from django.db.models.fields import BLANK_CHOICE_DASH
from django.template import Context, Template, TemplateSyntaxError, VariableDoesNotExist
from django.utils.translation import gettext_lazy as _
from gadjo.forms.widgets import MultiSelectWidget
from combo.utils import cache_during_request, requests, spooler
from .fields import StaticField
from .models import ChartCell, ChartFiltersCell, ChartNgCell
from .widgets import MultipleSelect2Widget, Select2Widget
@dataclasses.dataclass
class Choice:
id: str
label: str
group: str = None
@staticmethod
def get_field_choices(choices):
choices_by_group = defaultdict(list)
for choice in choices:
choices_by_group[choice.group].append((choice.id, choice.label))
return list(choices_by_group.items())
class ChartForm(forms.ModelForm):
@ -59,11 +75,12 @@ def trigger_statistics_list_refresh():
class ChartFiltersMixin:
ajax_choices = True
time_intervals = (
('week', _('Week')),
('month', _('Month')),
('year', _('Year')),
('weekday', _('Week day')),
Choice('week', _('Week')),
Choice('month', _('Month')),
Choice('year', _('Year')),
Choice('weekday', _('Week day')),
)
def get_filter_fields(self, cell):
@ -92,57 +109,68 @@ class ChartFiltersMixin:
return fields
def build_choice_field(self, cell, filter_, initial):
@classmethod
def get_filter_options(cls, cell, filter_, initial):
filter_id = filter_['id']
has_option_groups = isinstance(filter_['options'][0], list)
if filter_['options'] and has_option_groups:
choices = {
group: [(opt['id'], opt['label']) for opt in options] for group, options in filter_['options']
}
choices_to_complete = choices[None] = choices.get(None, [])
choices = list(choices.items())
else:
choices = [(option['id'], option['label']) for option in filter_['options']]
choices_to_complete = choices
filter_options = filter_['options']
if not isinstance(filter_options[0], list):
# no option groups, add empty one for consistency
filter_options = [(None, filter_options)]
choices = [
Choice(id=opt['id'], label=opt['label'], group=group)
for group, options in filter_options
for opt in options
]
if filter_id == 'time_interval':
self.extend_time_interval_choices(choices)
cls.extend_time_interval_choices(choices)
required = filter_.get('required', False)
multiple = filter_.get('multiple')
if not required:
choices_to_complete.insert(0, BLANK_CHOICE_DASH[0])
if not required and not multiple:
choices.insert(0, Choice(*BLANK_CHOICE_DASH[0]))
extra_variables = cell.page.get_extra_variables_keys()
variable_choices = [('variable:' + key, key) for key in extra_variables]
variable_choices = [
Choice(id='variable:' + key, label=key, group=_('Page variables')) for key in extra_variables
]
if has_option_groups:
possible_choices = {choice[0] for _, group_choices in choices for choice in group_choices}
else:
possible_choices = {choice[0] for choice in choices}
for choice in initial if isinstance(initial, list) else [initial]:
if not choice:
continue
if choice.startswith('variable:'):
variable = choice.replace('variable:', '')
if not variable in extra_variables:
variable_choices.append((choice, _('%s (unavailable)') % variable))
elif choice not in possible_choices:
choices_to_complete.append((choice, _('%s (unavailable)') % choice))
variable_choices.append(
Choice(id=choice, label=_('%s (unavailable)') % variable, group=_('Page variables'))
)
elif not any(x.id == choice for x in choices):
choices.append(Choice(id=choice, label=_('%s (unavailable)') % choice))
if variable_choices and not multiple and filter_id != 'time_interval':
choices.append((_('Page variables'), variable_choices))
choices.extend(variable_choices)
return choices
def build_choice_field(self, cell, filter_, initial):
multiple = filter_.get('multiple')
required = filter_.get('required', False)
choices = self.get_filter_options(cell, filter_, initial)
widget_class = MultipleSelect2Widget if multiple else Select2Widget
widget = widget_class(cell, filter_['id'], choices, initial, self.ajax_choices)
field_class = forms.MultipleChoiceField if multiple else forms.ChoiceField
widget_class = MultiSelectWidget if multiple else forms.Select
return field_class(
field = field_class(
label=filter_['label'],
choices=choices,
choices=Choice.get_field_choices(choices),
required=required,
initial=initial,
widget=widget_class,
)
field.widget = widget
field.dataviz_choices = choices
return field
def build_boolean_field(self, cell, filter_, initial):
return forms.BooleanField(
@ -151,11 +179,11 @@ class ChartFiltersMixin:
initial=bool(initial == 'true'),
)
def extend_time_interval_choices(self, choices):
choice_ids = {choice_id for choice_id, _ in choices}
if 'day' in choice_ids:
for choice in self.time_intervals:
if choice[0] not in choice_ids:
@classmethod
def extend_time_interval_choices(cls, choices):
if any(choice.id == 'day' for choice in choices):
for choice in cls.time_intervals:
if choice not in choices:
choices.append(choice)
def update_time_range_choices(self, statistic, exclude_template_choice=False):
@ -323,6 +351,7 @@ class ChartNgPartialForm(ChartFiltersMixin, forms.ModelForm):
class ChartFiltersForm(ChartFiltersMixin, forms.ModelForm):
ajax_choices = False
overridden_filters = StaticField()
prefix = 'filter'
@ -389,21 +418,12 @@ class ChartFiltersForm(ChartFiltersMixin, forms.ModelForm):
if field_name not in dynamic_fields or isinstance(field, forms.BooleanField):
continue
has_option_groups = isinstance(dynamic_fields[field_name].choices[0][1], list)
if has_option_groups and isinstance(field.choices[0][1], list):
new_choices = []
field_choices = {group_label: choices for group_label, choices in field.choices}
for group_label, choices in dynamic_fields[field_name].choices:
if group_label not in field_choices:
continue
new_choices.append(
(group_label, [x for x in choices if x in field_choices[group_label]])
)
dynamic_fields[field_name].choices = new_choices
else:
dynamic_fields[field_name].choices = [
x for x in dynamic_fields[field_name].choices if x in field.choices
]
dynamic_fields[field_name].dataviz_choices = [
x for x in dynamic_fields[field_name].dataviz_choices if x in field.dataviz_choices
]
dynamic_fields[field_name].choices = Choice.get_field_choices(
dynamic_fields[field_name].dataviz_choices
)
if dynamic_fields[field_name].choices == []:
del dynamic_fields[field_name]

View File

@ -781,8 +781,8 @@ class ChartFiltersCell(CellBase):
verbose_name = _('Filters')
class Media:
js = ('js/gadjo.multiselectwidget.js',)
css = {'all': ('css/gadjo.multiselectwidget.css',)}
js = ('xstatic/select2.min.js', 'xstatic/i18n/fr.js')
css = {'all': ('xstatic/select2.min.css',)}
@classmethod
def is_enabled(cls):

View File

@ -15,10 +15,18 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.conf.urls import re_path
from django.urls import path
from .views import ajax_gauge_count, dataviz_graph
from combo.urls_utils import manager_required
from .views import ajax_gauge_count, dataviz_choices, dataviz_graph
urlpatterns = [
re_path(r'^ajax/gauge-count/(?P<cell>[\w_-]+)/$', ajax_gauge_count, name='combo-ajax-gauge-count'),
re_path(r'^api/dataviz/graph/(?P<cell>[\w_-]+)/$', dataviz_graph, name='combo-dataviz-graph'),
path(
'api/dataviz/graph/<int:cell_id>/<filter_id>/ajax-choices',
manager_required(dataviz_choices),
name='combo-dataviz-choices',
),
]

View File

@ -14,10 +14,12 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import unicodedata
from django.core import signing
from django.core.cache import cache
from django.core.exceptions import PermissionDenied
from django.http import Http404, HttpResponse, HttpResponseBadRequest
from django.http import Http404, HttpResponse, HttpResponseBadRequest, JsonResponse
from django.shortcuts import render
from django.template import TemplateSyntaxError, VariableDoesNotExist
from django.utils.translation import gettext_lazy as _
@ -27,7 +29,7 @@ from requests.exceptions import HTTPError
from combo.utils import NothingInCacheException, get_templated_url, requests
from .forms import ChartNgPartialForm
from .forms import ChartFiltersMixin, ChartNgPartialForm, Choice
from .models import ChartNgCell, Gauge, MissingVariable, UnsupportedDataSet
@ -130,3 +132,58 @@ class DatavizGraphView(DetailView):
dataviz_graph = xframe_options_sameorigin(DatavizGraphView.as_view())
class DatavizChoicesView(DetailView):
model = ChartNgCell
pk_url_kwarg = 'cell_id'
def dispatch(self, *args, **kwargs):
self.cell = self.get_object()
filter_id = self.kwargs.get('filter_id')
for filter_ in self.cell.available_filters:
if filter_['id'] == filter_id:
self.filter = filter_
break
else:
raise Http404()
return super().dispatch(*args, **kwargs)
def get(self, request, *args, **kwargs):
search_term = request.GET.get('term', '')
search_term = unicodedata.normalize('NFKC', search_term).casefold()
try:
page_number = int(request.GET.get('page', 1))
except ValueError:
page_number = 1
initial = self.cell.filter_params.get(self.filter['id'], self.filter.get('default'))
objects = ChartFiltersMixin.get_filter_options(self.cell, self.filter, initial)
objects = [x for x in objects if search_term in unicodedata.normalize('NFKC', str(x)).casefold()]
return JsonResponse(
{
'results': self.format_results(objects, (page_number - 1) * 10, page_number * 10),
'pagination': {'more': bool(len(objects) >= page_number * 10)},
}
)
def format_results(self, objects, start_index, end_index):
page_objects = objects[start_index:end_index]
if start_index > 0:
last_displayed_group = objects[start_index - 1].group
for option in page_objects:
if option.group == last_displayed_group:
option.group = None
return [
{'text': group, 'children': [{'id': k, 'text': v} for k, v in choices]}
for group, choices in Choice.get_field_choices(page_objects)
]
dataviz_choices = DatavizChoicesView.as_view()

View File

@ -0,0 +1,64 @@
# combo - content management system
# Copyright (C) 2014-2022 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.conf import settings
from django.forms.widgets import Select, SelectMultiple
from django.urls import reverse
class Select2WidgetMixin:
class Media:
js = 'xstatic/select2.min.js'
css = {'all': ('xstatic/select2.min.css',)}
def __init__(self, cell, filter_id, choices, initial, ajax_choices):
from .forms import Choice
attrs = {}
if self.enable_select2(choices):
attrs['data-autocomplete'] = 'true'
attrs['lang'] = settings.LANGUAGE_CODE
if ajax_choices:
attrs['data-select2-url'] = reverse(
'combo-dataviz-choices', kwargs={'cell_id': cell.pk, 'filter_id': filter_id}
)
choices = self.filter_choices(choices, initial)
super().__init__(choices=Choice.get_field_choices(choices), attrs=attrs)
def enable_select2(self, choices):
return True
class Select2Widget(Select2WidgetMixin, Select):
min_choices = 20
@staticmethod
def filter_choices(choices, initial):
return [x for x in choices if x.id == initial]
def enable_select2(self, choices):
return bool(len(choices) > self.min_choices)
class MultipleSelect2Widget(Select2WidgetMixin, SelectMultiple):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.attrs['multiple'] = 'multiple'
@staticmethod
def filter_choices(choices, initial):
return [x for x in choices if x.id in (initial or [])]

View File

@ -1783,46 +1783,30 @@ def test_chartng_cell_manager_new_api(app, admin_user, new_api_statistics):
filter_multiple_stat = Statistic.objects.get(slug='filter-multiple')
resp.form[field_prefix + 'statistic'] = filter_multiple_stat.pk
manager_submit_cell(resp.form)
assert field_prefix + 'color$add_element' in resp.form.fields
assert field_prefix + 'color$remove_element' in resp.form.fields
resp.form[field_prefix + 'color'].select(text='Blue')
manager_submit_cell(resp.form)
cell.refresh_from_db()
assert cell.filter_params == {'color': ['blue']}
assert resp.form[field_prefix + 'color'].options == []
cell.filter_params = {'color': ['blue', 'green']}
cell.save()
resp = app.get('/manage/pages/%s/' % page.id)
assert resp.form.get(field_prefix + 'color', 0).value == 'blue'
assert resp.form.get(field_prefix + 'color', 1).value == 'green'
resp.form.get(field_prefix + 'color', 0).select(text='Red')
resp.form[field_prefix + 'color'].force_value(['blue', 'green'])
manager_submit_cell(resp.form)
assert resp.form[field_prefix + 'color'].value == ['green', 'blue']
assert resp.form[field_prefix + 'color'].options == [('green', True, 'Green'), ('blue', True, 'Blue')]
cell.refresh_from_db()
assert cell.filter_params == {'color': ['red', 'green']}
assert cell.filter_params == {'color': ['blue', 'green']}
color_filter = next(x for x in cell.statistic.filters if x['id'] == 'color')
color_filter['options'] = [{'id': 'black', 'label': 'Black'}, {'id': 'green', 'label': 'Green'}]
cell.statistic.save()
resp = app.get('/manage/pages/%s/' % page.id)
assert resp.form.get(field_prefix + 'color', 0).value == 'red'
assert resp.form.get(field_prefix + 'color', 1).value == 'green'
assert resp.form.get(field_prefix + 'color', 0).options == [
('', False, '---------'),
('black', False, 'Black'),
('green', False, 'Green'),
('red', True, 'red (unavailable)'),
assert resp.form[field_prefix + 'color'].value == ['green', 'blue']
assert resp.form[field_prefix + 'color'].options == [
('green', True, 'Green'),
('blue', True, 'blue (unavailable)'),
]
resp.form.get(field_prefix + 'color', 0).select(text='Green')
resp.form[field_prefix + 'color'].select_multiple(texts=[])
manager_submit_cell(resp.form)
cell.refresh_from_db()
assert cell.filter_params == {'color': ['green']}
resp.form[field_prefix + 'color'] = ''
manager_submit_cell(resp.form)
assert resp.form[field_prefix + 'color'].value == ''
assert resp.form[field_prefix + 'color'].value is None
cell.refresh_from_db()
assert cell.get_filter_params() == {}
@ -2115,14 +2099,6 @@ def test_chartng_cell_manager_new_api_page_variables(app, admin_user, new_api_st
time_interval_field = resp.form[field_prefix + 'time_interval']
assert [x[0] for x in time_interval_field.options] == ['day', 'month', 'year', 'week', 'weekday']
# no variables allowed for multiple choice field
cell.statistic = Statistic.objects.get(slug='filter-multiple')
cell.save()
resp = app.get('/manage/pages/%s/' % page.id)
color_field = resp.form[field_prefix + 'color']
assert [x[0] for x in color_field.options] == ['', 'red', 'green', 'blue']
def test_chartng_cell_manager_new_api_tabs(app, admin_user):
page = Page.objects.create(title='One', slug='index')
@ -3136,6 +3112,41 @@ def test_chart_filters_cell_select_filters(new_api_statistics, app, admin_user,
assert resp.forms[0].get(field_prefix + 'filters', index=0).value == 'ou'
@with_httmock(new_api_mock)
def test_chart_filters_cell_select2_choices(app, admin_user, new_api_statistics):
page = Page.objects.create(title='One', slug='index')
cell = ChartNgCell(page=page, order=1, placeholder='content')
cell.statistic = Statistic.objects.get(slug='filter-multiple')
cell.save()
ChartFiltersCell.objects.create(page=page, order=2, placeholder='content')
# multiple select2 is enabled without ajax
resp = app.get('/')
assert 'data-autocomplete' in resp.text
assert 'data-select2-url' not in resp.text
assert resp.form['filter-color'].options == [
('red', False, 'Red'),
('green', False, 'Green'),
('blue', False, 'Blue'),
]
cell.statistic = Statistic.objects.get(slug='option-groups')
cell.save()
# add choices to enable select2
form_filter = next(x for x in cell.statistic.filters if x['id'] == 'form')
form_filter['options'].append(
['Category C', [{'id': 'test-%s' % i, 'label': 'test %s' % i} for i in range(20)]]
)
cell.statistic.save()
# single select2 is enabled without ajax
resp = app.get('/')
assert 'data-autocomplete' in resp.text
assert 'data-select2-url' not in resp.text
assert len(resp.form['filter-form'].options) == 24
@with_httmock(new_api_mock)
@pytest.mark.freeze_time('2021-10-06')
def test_chartng_cell_api_view_get_parameters(app, normal_user, new_api_statistics, nocache):
@ -3367,3 +3378,104 @@ def test_chart_filters_cell_required_boolean(new_api_statistics, app, admin_user
# different value, boolean filter is hidden
resp = app.get('/')
assert not 'filter-test' in resp.form.fields
def test_chartng_cell_select2_choices(app, admin_user, new_api_statistics):
page = Page.objects.create(title='One', slug='index', extra_variables={'foo': 'bar'})
cell = ChartNgCell(page=page, order=1, placeholder='content')
cell.statistic = Statistic.objects.get(slug='option-groups')
cell.save()
app = login(app)
resp = app.get('/manage/pages/%s/' % page.id)
assert not resp.pyquery('select#id_cdataviz_chartngcell-%s-form' % cell.id).attr('data-select2-url')
# add choices to enable select2
form_filter = next(x for x in cell.statistic.filters if x['id'] == 'form')
form_filter['options'].append(
['Category C', [{'id': 'test-%s' % i, 'label': 'test %s' % i} for i in range(20)]]
)
cell.statistic.save()
resp = app.get('/manage/pages/%s/' % page.id)
field_prefix = 'cdataviz_chartngcell-%s-' % cell.id
assert resp.form[field_prefix + 'form'].options == []
resp.form[field_prefix + 'form'].force_value('test-0')
manager_submit_cell(resp.form)
assert resp.form[field_prefix + 'form'].options == [('test-0', True, 'test 0')]
url = resp.pyquery('select#id_cdataviz_chartngcell-%s-form' % cell.id).attr('data-select2-url')
resp = app.get(url)
assert resp.json['results'] == [
{'children': [{'id': '', 'text': '---------'}, {'id': 'all', 'text': 'All'}], 'text': None},
{'children': [{'id': 'test', 'text': 'Test'}], 'text': 'Category A'},
{'children': [{'id': 'test-2', 'text': 'test 2'}], 'text': 'Category B'},
{
'children': [
{'id': 'test-0', 'text': 'test 0'},
{'id': 'test-1', 'text': 'test 1'},
{'id': 'test-2', 'text': 'test 2'},
{'id': 'test-3', 'text': 'test 3'},
{'id': 'test-4', 'text': 'test 4'},
{'id': 'test-5', 'text': 'test 5'},
],
'text': 'Category C',
},
]
assert resp.json['pagination']['more'] is True
resp = app.get(url + '?page=2')
assert len(resp.json['results']) == 1
assert resp.json['results'][0]['text'] is None
assert len(resp.json['results'][0]['children']) == 10
assert resp.json['results'][0]['children'][0]['id'] == 'test-6'
assert resp.json['results'][0]['children'][-1]['id'] == 'test-15'
assert resp.json['pagination']['more'] is True
resp = app.get(url + '?page=3')
assert resp.json['results'] == [
{
'children': [
{'id': 'test-16', 'text': 'test 16'},
{'id': 'test-17', 'text': 'test 17'},
{'id': 'test-18', 'text': 'test 18'},
{'id': 'test-19', 'text': 'test 19'},
],
'text': None,
},
{'children': [{'id': 'variable:foo', 'text': 'foo'}], 'text': 'Page variables'},
]
assert resp.json['pagination']['more'] is False
resp = app.get(url + '?term=test 2')
assert resp.json['results'] == [
{'children': [{'id': 'test-2', 'text': 'test 2'}], 'text': 'Category B'},
{'children': [{'id': 'test-2', 'text': 'test 2'}], 'text': 'Category C'},
]
assert resp.json['pagination']['more'] is False
# no variables allowed for multiple choice field
cell.statistic = Statistic.objects.get(slug='filter-multiple')
cell.save()
resp = app.get('/manage/pages/%s/' % page.id)
url = resp.pyquery('select#id_cdataviz_chartngcell-%s-color' % cell.id).attr('data-select2-url')
resp = app.get(url)
assert resp.json == {
'pagination': {'more': False},
'results': [
{
'children': [
{'id': 'red', 'text': 'Red'},
{'id': 'green', 'text': 'Green'},
{'id': 'blue', 'text': 'Blue'},
],
'text': None,
}
],
}
# unknown filter
resp = app.get('/api/dataviz/graph/%s/unknown/ajax-choices' % cell.id, status=404)