entrouvert
/
combo
16
1
Fork 0
Code Issues Pull Requests 10 Releases Wiki Activity

dataviz: save available visualizations in db (#49173)

This commit is contained in:
Valentin Deniaud 2020-11-26 16:09:26 +01:00
parent 5db0931f10
commit 356bb767f5
10 changed files with 354 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
combo/apps/dataviz/__init__.py
View File

@ -14,13 +14,14 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import re
import django.apps
from django.core import checks
from django.conf import settings
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from combo.utils import requests
class AppConfig(django.apps.AppConfig):
name = 'combo.apps.dataviz'
@ -30,4 +31,33 @@ class AppConfig(django.apps.AppConfig):
from . import urls
return urls.urlpatterns
def hourly(self):
self.update_available_statistics()
def update_available_statistics(self):
from .models import Statistic
if not settings.KNOWN_SERVICES:
return
start_update = timezone.now()
bijoe_sites = settings.KNOWN_SERVICES.get('bijoe', {}).items()
for site_key, site_dict in bijoe_sites:
result = requests.get('/visualization/json/',
remote_service=site_dict, without_user=True,
headers={'accept': 'application/json'}).json()
for stat in result:
Statistic.objects.update_or_create(
slug=stat['slug'],
site_slug=site_key,
service_slug='bijoe',
defaults={
'label': stat['name'],
'url': stat['data-url'],
'site_title': site_dict.get('title', ''),
'available': True,
}
)
Statistic.objects.filter(last_update__lt=start_update).update(available=False)
default_app_config = 'combo.apps.dataviz.AppConfig'

26
combo/apps/dataviz/forms.py
View File

@ -16,7 +16,7 @@
from django import forms
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from django.db.models import Q
from combo.utils import requests
@ -43,24 +43,12 @@ class ChartForm(forms.ModelForm):
class ChartNgForm(forms.ModelForm):
class Meta:
model = ChartNgCell
fields = ('title', 'data_reference', 'chart_type', 'height', 'sort_order',
fields = ('title', 'statistic', 'chart_type', 'height', 'sort_order',
'hide_null_values')
def __init__(self, *args, **kwargs):
super(ChartNgForm, self).__init__(*args, **kwargs)
data_references = []
bijoe_sites = settings.KNOWN_SERVICES.get('bijoe').items()
for site_key, site_dict in bijoe_sites:
result = requests.get('/visualization/json/',
remote_service=site_dict, without_user=True,
headers={'accept': 'application/json'}).json()
if len(bijoe_sites) > 1:
label_prefix = _('%s: ') % site_dict.get('title')
else:
label_prefix = ''
data_references.extend([
('%s:%s' % (site_key, x['slug']), '%s%s' % (label_prefix, x['name']))
for x in result])
data_references.sort(key=lambda x: x[1])
self.fields['data_reference'].widget = forms.Select(choices=data_references)
super().__init__(*args, **kwargs)
q_filters = Q(available=True)
if self.instance.statistic:
q_filters |= Q(pk=self.instance.statistic.pk)
self.fields['statistic'].queryset = self.fields['statistic'].queryset.filter(q_filters)

42
combo/apps/dataviz/migrations/0012_auto_20201126_1557.py Normal file
View File

@ -0,0 +1,42 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.29 on 2020-11-26 14:57
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('dataviz', '0011_auto_20200813_1100'),
]
operations = [
migrations.CreateModel(
name='Statistic',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('slug', models.SlugField(max_length=256, verbose_name='Slug')),
('label', models.CharField(max_length=256, verbose_name='Label')),
('site_slug', models.SlugField(max_length=256, verbose_name='Site slug')),
('service_slug', models.SlugField(max_length=256, verbose_name='Service slug')),
('site_title', models.CharField(max_length=256, verbose_name='Site title')),
('url', models.URLField(verbose_name='Data URL')),
('last_update', models.DateTimeField(auto_now=True, null=True, verbose_name='Last update')),
('available', models.BooleanField(default=True, verbose_name='Available data')),
],
options={
'ordering': ['-available', 'site_title', 'label'],
},
),
migrations.AlterUniqueTogether(
name='statistic',
unique_together=set([('slug', 'site_slug', 'service_slug')]),
),
migrations.AddField(
model_name='chartngcell',
name='statistic',
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='cells', to='dataviz.Statistic', verbose_name='Data'),
),
]

40
combo/apps/dataviz/migrations/0013_update_chartng_cells.py Normal file
View File

@ -0,0 +1,40 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.29 on 2020-11-30 14:26
from __future__ import unicode_literals
from django.conf import settings
from django.db import migrations
def update_cells(apps, schema_editor):
Statistic = apps.get_model('dataviz', 'Statistic')
ChartNgCell = apps.get_model('dataviz', 'ChartNgCell')
bijoe_sites = settings.KNOWN_SERVICES.get('bijoe', {})
for cell in ChartNgCell.objects.filter(statistic__isnull=True):
if not cell.data_reference or not cell.cached_json:
continue
site_slug, slug = cell.data_reference.split(':')
statistic, _ = Statistic.objects.get_or_create(
slug=slug,
site_slug=site_slug,
service_slug='bijoe',
defaults={
'label': cell.cached_json['name'],
'url': cell.cached_json['data-url'],
'site_title': bijoe_sites.get(site_slug, {}).get('title'),
}
)
cell.statistic = statistic
cell.save()
class Migration(migrations.Migration):
dependencies = [
('dataviz', '0012_auto_20201126_1557'),
]
operations = [
migrations.RunPython(update_cells, migrations.RunPython.noop),
]

23
combo/apps/dataviz/migrations/0014_auto_20201130_1534.py Normal file
View File

@ -0,0 +1,23 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.29 on 2020-11-30 14:34
from __future__ import unicode_literals
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('dataviz', '0013_update_chartng_cells'),
]
operations = [
migrations.RemoveField(
model_name='chartngcell',
name='cached_json',
),
migrations.RemoveField(
model_name='chartngcell',
name='data_reference',
),
]

60
combo/apps/dataviz/models.py
View File

@ -103,11 +103,43 @@ class ChartCell(CellBase):
return context
class StatisticManager(models.Manager):
def get_by_natural_key(self, slug, site_slug, service_slug):
return self.get_or_create(slug=slug, site_slug=site_slug, service_slug=service_slug)[0]
class Statistic(models.Model):
slug = models.SlugField(_('Slug'), max_length=256)
label = models.CharField(_('Label'), max_length=256)
site_slug = models.SlugField(_('Site slug'), max_length=256)
service_slug = models.SlugField(_('Service slug'), max_length=256)
site_title = models.CharField(_('Site title'), max_length=256)
url = models.URLField(_('Data URL'))
available = models.BooleanField(_('Available data'), default=True)
last_update = models.DateTimeField(_('Last update'), null=True, auto_now=True)
objects = StatisticManager()
class Meta:
ordering = ['-available', 'site_title', 'label']
unique_together = ['slug', 'site_slug', 'service_slug']
def __str__(self):
name = _('%s: %s') % (self.site_title or self.site_slug, self.label)
if not self.available:
name = _('%s (unavailable)') % name
return name
def natural_key(self):
return (self.slug, self.site_slug, self.service_slug)
@register_cell_class
class ChartNgCell(CellBase):
data_reference = models.CharField(_('Data'), max_length=150)
statistic = models.ForeignKey(
verbose_name=_('Data'), to=Statistic, blank=False, null=True, on_delete=models.SET_NULL, related_name='cells'
)
title = models.CharField(_('Title'), max_length=150, blank=True)
cached_json = JSONField(blank=True)
chart_type = models.CharField(_('Chart Type'), max_length=20, default='bar',
choices=(
('bar', _('Bar')),
@ -154,28 +186,12 @@ class ChartNgCell(CellBase):
def get_additional_label(self):
return self.title
def is_relevant(self, context=None):
return bool(self.data_reference)
def save(self, *args, **kwargs):
if self.data_reference:
site_key, visualization_slug = self.data_reference.split(':')
site_dict = settings.KNOWN_SERVICES['bijoe'][site_key]
response_json = requests.get('/visualization/json/',
remote_service=site_dict, without_user=True,
headers={'accept': 'application/json'}).json()
if isinstance(response_json, dict):
# forward compatibility with possible API change
response_json = response_json.get('data')
for visualization in response_json:
slug = visualization.get('slug')
if slug == visualization_slug:
self.cached_json = visualization
return super(ChartNgCell, self).save(*args, **kwargs)
def is_relevant(self, context):
return bool(self.statistic)
def get_cell_extra_context(self, context):
ctx = super(ChartNgCell, self).get_cell_extra_context(context)
if self.chart_type == 'table' and self.cached_json:
if self.chart_type == 'table' and self.statistic:
try:
chart = self.get_chart(raise_if_not_cached=not(context.get('synchronous')))
except UnsupportedDataSet:
@ -193,7 +209,7 @@ class ChartNgCell(CellBase):
def get_chart(self, width=None, height=None, raise_if_not_cached=False):
response = requests.get(
self.cached_json['data-url'],
self.statistic.url,
cache_duration=300,
raise_if_not_cached=raise_if_not_cached)
response.raise_for_status()

4
combo/apps/dataviz/templates/combo/chartngcell.html
View File

@ -1,8 +1,6 @@
{% load i18n %}
{% if cell.title %}<h2>{{cell.title}}</h2>{% endif %}
{% if not cell.cached_json %}
<div class="warningnotice">{% trans "Unavailable data." %}</div>
{% elif cell.chart_type == "table" %}
{% if cell.chart_type == "table" %}
{{table|safe}}
{% else %}
<div style="min-height: {{cell.height}}px">

2
combo/apps/dataviz/templates/combo/chartngcell_form.html
View File

@ -1,6 +1,6 @@
<div style="position: relative">
{{ form.as_p }}
{% if cell.cached_json and cell.chart_type != "table" and cell.is_relevant %}
{% if cell.statistic and cell.chart_type != "table" %}
<div style="position: absolute; right: 0; top: 0; width: 300px; height: 150px">
<embed type="image/svg+xml" src="{% url 'combo-dataviz-graph' cell=cell.id %}?width=300&height=150"/>
</div>

2
combo/apps/dataviz/views.py
View File

@ -36,7 +36,7 @@ def dataviz_graph(request, *args, **kwargs):
raise PermissionDenied()
if not cell.is_visible(user=request.user):
raise PermissionDenied()
if not cell.cached_json:
if not cell.statistic:
raise Http404('misconfigured cell')
error_text = None
try:

258
tests/test_dataviz.py
View File

@ -2,15 +2,19 @@ import json
import mock
import pytest
from datetime import timedelta
from httmock import HTTMock, with_httmock
from requests.exceptions import HTTPError
from django.apps import apps
from django.contrib.auth.models import User, Group
from django.db import connection
from django.db.migrations.executor import MigrationExecutor
from django.test import override_settings
from django.utils import timezone
from combo.data.models import Page
from combo.apps.dataviz.models import Gauge, ChartNgCell, UnsupportedDataSet
from combo.apps.dataviz.models import Gauge, ChartNgCell, UnsupportedDataSet, Statistic
from .test_public import login, normal_user
@ -29,15 +33,6 @@ def cell():
return cell
@pytest.fixture(autouse=True)
def bijoe_settings(settings):
settings.KNOWN_SERVICES = {
"bijoe": {
"plop": {"title": "test", "url": "https://bijoe.example.com", "secret": "combo", "orig": "combo"}
}
}
def test_jsonp_gauge(app, cell):
with override_settings(TEMPLATE_VARS={'test_url': 'http://www.example.net'}):
resp = app.get('/')
@ -224,15 +219,27 @@ def bijoe_mock(url, request):
return {'content': json.dumps(response), 'request': request, 'status_code': 404}
@pytest.fixture
@with_httmock(bijoe_mock)
def test_chartng_cell(app):
def statistics(settings):
settings.KNOWN_SERVICES = {
"bijoe": {
"plop": {"title": "test", "url": "https://bijoe.example.com", "secret": "combo", "orig": "combo"}
}
}
appconfig = apps.get_app_config('dataviz')
appconfig.hourly()
assert Statistic.objects.count() == len(VISUALIZATION_JSON)
@with_httmock(bijoe_mock)
def test_chartng_cell(app, statistics):
page = Page(title='One', slug='index')
page.save()
cell = ChartNgCell(page=page, order=1)
cell.data_reference = 'plop:example'
cell.statistic = Statistic.objects.get(slug='example')
cell.save()
assert cell.cached_json == VISUALIZATION_JSON[0]
# bar
chart = cell.get_chart()
@ -260,9 +267,8 @@ def test_chartng_cell(app):
# data in Y
cell.chart_type = 'bar'
cell.data_reference = 'plop:second'
cell.statistic = Statistic.objects.get(slug='second')
cell.save()
assert cell.cached_json == VISUALIZATION_JSON[1]
chart = cell.get_chart()
assert chart.x_labels == ['web', 'mail', 'phone', 'email']
@ -270,9 +276,8 @@ def test_chartng_cell(app):
# data in X/Y
cell.chart_type = 'bar'
cell.data_reference = 'plop:third'
cell.statistic = Statistic.objects.get(slug='third')
cell.save()
assert cell.cached_json == VISUALIZATION_JSON[2]
chart = cell.get_chart()
assert chart.x_labels == ['web', 'mail', 'phone', 'email']
@ -283,16 +288,15 @@ def test_chartng_cell(app):
# single data point
cell.chart_type = 'bar'
cell.data_reference = 'plop:fourth'
cell.statistic = Statistic.objects.get(slug='fourth')
cell.save()
assert cell.cached_json == VISUALIZATION_JSON[3]
chart = cell.get_chart()
assert chart.x_labels == ['']
assert chart.raw_series == [([222], {'title': ''})]
# loop/X
cell.data_reference = 'plop:fifth'
cell.statistic = Statistic.objects.get(slug='fifth')
cell.save()
chart = cell.get_chart()
assert chart.x_labels == ['web', 'mail', 'phone', 'email']
@ -302,7 +306,7 @@ def test_chartng_cell(app):
]
# loop/Y
cell.data_reference = 'plop:sixth'
cell.statistic = Statistic.objects.get(slug='sixth')
cell.save()
chart = cell.get_chart()
assert chart.x_labels == ['web', 'mail', 'phone', 'email']
@ -312,39 +316,38 @@ def test_chartng_cell(app):
]
# loop/X/Y
cell.data_reference = 'plop:seventh'
cell.statistic = Statistic.objects.get(slug='seventh')
cell.save()
with pytest.raises(UnsupportedDataSet):
chart = cell.get_chart()
# duration
cell.data_reference = 'plop:eighth'
cell.statistic = Statistic.objects.get(slug='eighth')
cell.save()
chart = cell.get_chart()
# loop/X/Y
cell.data_reference = 'plop:nineth'
cell.statistic = Statistic.objects.get(slug='nineth')
cell.save()
with pytest.raises(UnsupportedDataSet):
chart = cell.get_chart()
# deleted visualization
cell.data_reference = 'plop:eleventh'
cell.statistic = Statistic.objects.get(slug='eleventh')
cell.save()
with pytest.raises(HTTPError):
chart = cell.get_chart()
@with_httmock(bijoe_mock)
def test_chartng_cell_hide_null_values(app):
def test_chartng_cell_hide_null_values(app, statistics):
page = Page(title='One', slug='index')
page.save()
cell = ChartNgCell(page=page, order=1)
cell.data_reference = 'plop:example'
cell.statistic = Statistic.objects.get(slug='example')
cell.hide_null_values = True
cell.save()
assert cell.cached_json == VISUALIZATION_JSON[0]
# bar
chart = cell.get_chart()
@ -372,9 +375,8 @@ def test_chartng_cell_hide_null_values(app):
# data in Y
cell.chart_type = 'bar'
cell.data_reference = 'plop:second'
cell.statistic = Statistic.objects.get(slug='second')
cell.save()
assert cell.cached_json == VISUALIZATION_JSON[1]
chart = cell.get_chart()
assert chart.x_labels == ['web', 'mail', 'email']
@ -382,9 +384,8 @@ def test_chartng_cell_hide_null_values(app):
# data in X/Y
cell.chart_type = 'bar'
cell.data_reference = 'plop:third'
cell.statistic = Statistic.objects.get(slug='third')
cell.save()
assert cell.cached_json == VISUALIZATION_JSON[2]
chart = cell.get_chart()
assert chart.x_labels == ['web', 'mail', 'phone', 'email']
@ -395,16 +396,15 @@ def test_chartng_cell_hide_null_values(app):
# single data point
cell.chart_type = 'bar'
cell.data_reference = 'plop:fourth'
cell.statistic = Statistic.objects.get(slug='fourth')
cell.save()
assert cell.cached_json == VISUALIZATION_JSON[3]
chart = cell.get_chart()
assert chart.x_labels == ['']
assert chart.raw_series == [([222], {'title': ''})]
# loop/X
cell.data_reference = 'plop:fifth'
cell.statistic = Statistic.objects.get(slug='fifth')
cell.save()
chart = cell.get_chart()
assert chart.x_labels == ['web', 'mail', 'phone', 'email']
@ -414,7 +414,7 @@ def test_chartng_cell_hide_null_values(app):
]
# loop/Y
cell.data_reference = 'plop:sixth'
cell.statistic = Statistic.objects.get(slug='sixth')
cell.save()
chart = cell.get_chart()
assert chart.x_labels == ['web', 'mail', 'phone', 'email']
@ -425,15 +425,14 @@ def test_chartng_cell_hide_null_values(app):
@with_httmock(bijoe_mock)
def test_chartng_cell_sort_order_alpha(app):
def test_chartng_cell_sort_order_alpha(app, statistics):
page = Page(title='One', slug='index')
page.save()
cell = ChartNgCell(page=page, order=1)
cell.data_reference = 'plop:example'
cell.statistic = Statistic.objects.get(slug='example')
cell.sort_order = 'alpha'
cell.save()
assert cell.cached_json == VISUALIZATION_JSON[0]
# bar
chart = cell.get_chart()
@ -461,9 +460,8 @@ def test_chartng_cell_sort_order_alpha(app):
# data in Y
cell.chart_type = 'bar'
cell.data_reference = 'plop:second'
cell.statistic = Statistic.objects.get(slug='second')
cell.save()
assert cell.cached_json == VISUALIZATION_JSON[1]
chart = cell.get_chart()
assert chart.x_labels == ['email', 'mail', 'phone', 'web']
@ -471,9 +469,8 @@ def test_chartng_cell_sort_order_alpha(app):
# data in X/Y
cell.chart_type = 'bar'
cell.data_reference = 'plop:third'
cell.statistic = Statistic.objects.get(slug='third')
cell.save()
assert cell.cached_json == VISUALIZATION_JSON[2]
chart = cell.get_chart()
assert chart.x_labels == ['web', 'mail', 'phone', 'email']
@ -484,16 +481,15 @@ def test_chartng_cell_sort_order_alpha(app):
# single data point
cell.chart_type = 'bar'
cell.data_reference = 'plop:fourth'
cell.statistic = Statistic.objects.get(slug='fourth')
cell.save()
assert cell.cached_json == VISUALIZATION_JSON[3]
chart = cell.get_chart()
assert chart.x_labels == ['']
assert chart.raw_series == [([222], {'title': ''})]
# loop/X
cell.data_reference = 'plop:fifth'
cell.statistic = Statistic.objects.get(slug='fifth')
cell.save()
chart = cell.get_chart()
assert chart.x_labels == ['web', 'mail', 'phone', 'email']
@ -503,7 +499,7 @@ def test_chartng_cell_sort_order_alpha(app):
]
# loop/Y
cell.data_reference = 'plop:sixth'
cell.statistic = Statistic.objects.get(slug='sixth')
cell.save()
chart = cell.get_chart()
assert chart.x_labels == ['web', 'mail', 'phone', 'email']
@ -514,15 +510,14 @@ def test_chartng_cell_sort_order_alpha(app):
@with_httmock(bijoe_mock)
def test_chartng_cell_sort_order_desc(app):
def test_chartng_cell_sort_order_desc(app, statistics):
page = Page(title='One', slug='index')
page.save()
cell = ChartNgCell(page=page, order=1)
cell.data_reference = 'plop:example'
cell.statistic = Statistic.objects.get(slug='example')
cell.sort_order = 'desc'
cell.save()
assert cell.cached_json == VISUALIZATION_JSON[0]
# bar
chart = cell.get_chart()
@ -550,9 +545,8 @@ def test_chartng_cell_sort_order_desc(app):
# data in Y
cell.chart_type = 'bar'
cell.data_reference = 'plop:second'
cell.statistic = Statistic.objects.get(slug='second')
cell.save()
assert cell.cached_json == VISUALIZATION_JSON[1]
chart = cell.get_chart()
assert chart.x_labels == ['web', 'mail', 'email', 'phone']
@ -560,9 +554,8 @@ def test_chartng_cell_sort_order_desc(app):
# data in X/Y
cell.chart_type = 'bar'
cell.data_reference = 'plop:third'
cell.statistic = Statistic.objects.get(slug='third')
cell.save()
assert cell.cached_json == VISUALIZATION_JSON[2]
chart = cell.get_chart()
assert chart.x_labels == ['web', 'mail', 'phone', 'email']
@ -573,16 +566,15 @@ def test_chartng_cell_sort_order_desc(app):
# single data point
cell.chart_type = 'bar'
cell.data_reference = 'plop:fourth'
cell.statistic = Statistic.objects.get(slug='fourth')
cell.save()
assert cell.cached_json == VISUALIZATION_JSON[3]
chart = cell.get_chart()
assert chart.x_labels == ['']
assert chart.raw_series == [([222], {'title': ''})]
# loop/X
cell.data_reference = 'plop:fifth'
cell.statistic = Statistic.objects.get(slug='fifth')
cell.save()
chart = cell.get_chart()
assert chart.x_labels == ['web', 'mail', 'phone', 'email']
@ -592,7 +584,7 @@ def test_chartng_cell_sort_order_desc(app):
]
# loop/Y
cell.data_reference = 'plop:sixth'
cell.statistic = Statistic.objects.get(slug='sixth')
cell.save()
chart = cell.get_chart()
assert chart.x_labels == ['web', 'mail', 'phone', 'email']
@ -603,12 +595,12 @@ def test_chartng_cell_sort_order_desc(app):
@with_httmock(bijoe_mock)
def test_chartng_cell_view(app, normal_user):
def test_chartng_cell_view(app, normal_user, statistics):
page = Page(title='One', slug='index')
page.save()
cell = ChartNgCell(page=page, order=1, placeholder='content')
cell.data_reference = 'plop:example'
cell.statistic = Statistic.objects.get(slug='example')
cell.save()
location = '/api/dataviz/graph/%s/' % cell.id
resp = app.get(location) # get data in cache
@ -649,7 +641,7 @@ def test_chartng_cell_view(app, normal_user):
assert '<td>222</td>' in resp.text
# unsupported dataset
cell.data_reference = 'plop:seventh'
cell.statistic = Statistic.objects.get(slug='seventh')
cell.save()
resp = app.get(location) # get data in cache
resp = app.get('/')
@ -661,7 +653,7 @@ def test_chartng_cell_view(app, normal_user):
assert 'Unsupported dataset' in resp.text
# durations
cell.data_reference = 'plop:eighth'
cell.statistic = Statistic.objects.get(slug='eighth')
cell.chart_type = 'table'
cell.save()
resp = app.get(location) # get data in cache
@ -680,7 +672,7 @@ def test_chartng_cell_view(app, normal_user):
assert '>1 day<' in resp.text
# percents
cell.data_reference = 'plop:tenth'
cell.statistic = Statistic.objects.get(slug='tenth')
cell.chart_type = 'table'
cell.save()
resp = app.get(location) # get data in cache
@ -693,55 +685,58 @@ def test_chartng_cell_view(app, normal_user):
assert '>10.0%<' in resp.text
# deleted visualization
cell.data_reference = 'plop:eleventh'
cell.statistic = Statistic.objects.get(slug='eleventh')
cell.save()
resp = app.get(location)
assert 'not found' in resp.text
# cell with missing cached_json (probably after import and missing
# bijoe visualisation)
# cell with no statistic chosen
cell.chart_type = 'table'
cell.statistic = None
cell.save()
ChartNgCell.objects.filter(id=cell.id).update(cached_json={})
resp = app.get('/')
assert 'warningnotice' in resp.text
assert not 'cell' in resp.text
@with_httmock(bijoe_mock)
def test_chartng_cell_manager(app, admin_user):
def test_chartng_cell_manager(app, admin_user, statistics):
page = Page(title='One', slug='index')
page.save()
Statistic.objects.create(
slug='unavailable-stat', label='Unavailable Stat', site_slug='plop', available=False
)
app = login(app)
cell = ChartNgCell(page=page, order=1, placeholder='content')
cell.data_reference = 'plop:example'
cell.statistic = Statistic.objects.get(slug='example')
cell.save()
resp = app.get('/manage/pages/%s/' % page.id)
assert resp.form['cdataviz_chartngcell-%s-data_reference' % cell.id].options == [
(u'plop:eighth', False, u'eighth visualization (duration)'),
(u'plop:eleventh', False, u'eleventh visualization (not found)'),
(u'plop:example', True, u'example visualization (X)'),
(u'plop:fifth', False, u'fifth visualization (loop/X)'),
(u'plop:fourth', False, u'fourth visualization (no axis)'),
(u'plop:nineth', False, u'nineth visualization (loop over varying dimensions)'),
(u'plop:second', False, u'second visualization (Y)'),
(u'plop:seventh', False, u'seventh visualization (loop/X/Y)'),
(u'plop:sixth', False, u'sixth visualization (loop/Y)'),
(u'plop:tenth', False, u'tenth visualization (percents)'),
(u'plop:third', False, u'third visualization (X/Y)'),
]
statistics_field = resp.form['cdataviz_chartngcell-%s-statistic' % cell.id]
# available visualizations and a blank choice
assert len(statistics_field.options) == len(VISUALIZATION_JSON) + 1
assert statistics_field.value == str(cell.statistic.pk)
assert statistics_field.options[1][2] == 'test: eighth visualization (duration)'
assert not 'Unavailable Stat' in resp.text
cell.statistic = Statistic.objects.get(slug='unavailable-stat')
cell.save()
resp = app.get('/manage/pages/%s/' % page.id)
statistics_field = resp.form['cdataviz_chartngcell-%s-statistic' % cell.id]
# available visualizations, a blank choice and the current unavailable visualization
assert len(statistics_field.options) == len(VISUALIZATION_JSON) + 2
assert 'Unavailable Stat' in resp.text
@with_httmock(bijoe_mock)
def test_table_cell(app, admin_user):
def test_table_cell(app, admin_user, statistics):
page = Page(title='One', slug='index')
page.save()
app = login(app)
cell = ChartNgCell(page=page, order=1, placeholder='content')
cell.data_reference = 'plop:example'
cell.statistic = Statistic.objects.get(slug='example')
cell.chart_type = 'table'
cell.save()
location = '/api/dataviz/graph/%s/' % cell.id
@ -749,28 +744,113 @@ def test_table_cell(app, admin_user):
resp = app.get('/')
assert resp.text.count('Total') == 1
cell.data_reference = 'plop:second'
cell.statistic = Statistic.objects.get(slug='second')
cell.save()
resp = app.get(location)
resp = app.get('/')
assert resp.text.count('Total') == 1
cell.data_reference = 'plop:third'
cell.statistic = Statistic.objects.get(slug='third')
cell.save()
resp = app.get(location)
resp = app.get('/')
assert '114' in resp.text
assert resp.text.count('Total') == 2
cell.data_reference = 'plop:fourth'
cell.statistic = Statistic.objects.get(slug='fourth')
cell.save()
resp = app.get(location)
resp = app.get('/')
assert resp.text.count('Total') == 0
# total of durations is not computed
cell.data_reference = 'plop:eigth'
cell.statistic = Statistic.objects.get(slug='eighth')
cell.save()
resp = app.get(location)
resp = app.get('/')
assert resp.text.count('Total') == 0
def test_dataviz_hourly_unavailable_statistic(statistics, nocache):
all_stats_count = Statistic.objects.count()
assert Statistic.objects.filter(available=True).count() == all_stats_count
def bijoe_mock_unavailable(url, request):
visualization_json = VISUALIZATION_JSON[2:]
return {'content': json.dumps(visualization_json), 'request': request, 'status_code': 200}
appconfig = apps.get_app_config('dataviz')
with HTTMock(bijoe_mock_unavailable):
appconfig.hourly()
assert Statistic.objects.filter(available=True).count() == all_stats_count - 2
def test_dataviz_import_cell():
page = Page.objects.create(title='One', slug='index')
cell = ChartNgCell.objects.create(page=page, order=1, slug='test', placeholder='content')
statistic = Statistic.objects.create(
slug='example', site_slug='plop', service_slug='bijoe', url='https://example.org'
)
cell.statistic = statistic
cell.save()
site_export = [page.get_serialized_page()]
cell.delete()
Page.load_serialized_pages(site_export)
cell = ChartNgCell.objects.get(slug='test')
assert cell.statistic.pk == statistic.pk
cell.delete()
statistic.delete()
Page.load_serialized_pages(site_export)
cell = ChartNgCell.objects.get(slug='test')
assert cell.statistic.slug == statistic.slug
assert cell.statistic.site_slug == statistic.site_slug
assert cell.statistic.service_slug == statistic.service_slug
def test_dataviz_cell_migration(settings):
settings.KNOWN_SERVICES = {
"bijoe": {
"plop": {"title": "test", "url": "https://bijoe.example.com", "secret": "combo", "orig": "combo"}
}
}
page = Page.objects.create(title='One', slug='index')
app = 'dataviz'
migrate_from = [(app, '0012_auto_20201126_1557')]
migrate_to = [(app, '0013_update_chartng_cells')]
executor = MigrationExecutor(connection)
old_apps = executor.loader.project_state(migrate_from).apps
executor.migrate(migrate_from)
Pagee = old_apps.get_model('data', 'Page')
page = Pagee.objects.get(pk=page.pk)
ChartNgCell = old_apps.get_model(app, 'ChartNgCell')
cell = ChartNgCell.objects.create(
page=page,
order=1,
data_reference='plop:example',
cached_json={
'data-url': 'https://bijoe.example.com/visualization/1/json/',
'path': 'https://bijoe.example.com/visualization/1/iframe/?signature=123',
'name': 'example visualization (X)',
'slug': 'example',
},
)
executor = MigrationExecutor(connection)
executor.migrate(migrate_to)
executor.loader.build_graph()
apps = executor.loader.project_state(migrate_to).apps
ChartNgCell = apps.get_model(app, 'ChartNgCell')
cell = ChartNgCell.objects.get(pk=cell.pk)
assert cell.statistic.slug == 'example'
assert cell.statistic.label == 'example visualization (X)'
assert cell.statistic.url == 'https://bijoe.example.com/visualization/1/json/'
assert cell.statistic.site_slug == 'plop'
assert cell.statistic.service_slug == 'bijoe'
assert cell.statistic.site_title == 'test'