entrouvert
/
combo
16
1
Fork 0
Code Issues Pull Requests 11 Releases Wiki Activity

dataviz: handle new api to get statistics from elsewhere (#48865)

This commit is contained in:
Valentin Deniaud 2020-11-30 14:10:39 +01:00
parent b615b7fa99
commit 18fdd8a8c4
4 changed files with 284 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
combo/apps/dataviz/__init__.py
View File

@ -39,24 +39,32 @@ class AppConfig(django.apps.AppConfig):
if not settings.KNOWN_SERVICES:
return
statistics_providers = settings.STATISTICS_PROVIDERS + ['bijoe']
start_update = timezone.now()
bijoe_sites = settings.KNOWN_SERVICES.get('bijoe', {}).items()
for site_key, site_dict in bijoe_sites:
result = requests.get('/visualization/json/',
remote_service=site_dict, without_user=True,
headers={'accept': 'application/json'}).json()
for stat in result:
Statistic.objects.update_or_create(
slug=stat['slug'],
site_slug=site_key,
service_slug='bijoe',
defaults={
'label': stat['name'],
'url': stat['data-url'],
'site_title': site_dict.get('title', ''),
'available': True,
}
)
for service in statistics_providers:
sites = settings.KNOWN_SERVICES.get(service, {}).items()
for site_key, site_dict in sites:
if service == 'bijoe':
result = requests.get('/visualization/json/',
remote_service=site_dict, without_user=True,
headers={'accept': 'application/json'}).json()
else:
result = requests.get('/api/statistics/',
remote_service=site_dict, without_user=True,
headers={'accept': 'application/json'}).json()['data']
for stat in result:
Statistic.objects.update_or_create(
slug=stat.get('slug') or stat['id'],
site_slug=site_key,
service_slug=service,
defaults={
'label': stat['name'],
'url': stat.get('data-url') or stat['url'],
'site_title': site_dict.get('title', ''),
'available': True,
}
)
Statistic.objects.filter(last_update__lt=start_update).update(available=False)

56
combo/apps/dataviz/models.py
View File

@ -177,7 +177,7 @@ class ChartNgCell(CellBase):
@classmethod
def is_enabled(self):
return hasattr(settings, 'KNOWN_SERVICES') and settings.KNOWN_SERVICES.get('bijoe')
return settings.KNOWN_SERVICES.get('bijoe') or settings.STATISTICS_PROVIDERS
def get_default_form_class(self):
from .forms import ChartNgForm
@ -202,7 +202,7 @@ class ChartNgCell(CellBase):
else:
ctx['table'] = chart.render_table(
transpose=bool(chart.axis_count == 2),
total=chart.compute_sum,
total=getattr(chart, 'compute_sum', True),
)
ctx['table'] = ctx['table'].replace('<table>', '<table class="main">')
return ctx
@ -211,6 +211,8 @@ class ChartNgCell(CellBase):
response = requests.get(
self.statistic.url,
cache_duration=300,
remote_service='auto',
without_user=True,
raise_if_not_cached=raise_if_not_cached)
response.raise_for_status()
response = response.json()
@ -229,18 +231,33 @@ class ChartNgCell(CellBase):
'table': pygal.Bar,
}[self.chart_type](config=pygal.Config(style=copy.copy(style)))
x_labels, y_labels, data = self.parse_response(response, chart)
chart.x_labels = x_labels
self.prepare_chart(chart, width, height)
if self.statistic.service_slug == 'bijoe':
x_labels, y_labels, data = self.parse_response(response, chart)
chart.x_labels = x_labels
self.prepare_chart(chart, width, height)
if chart.axis_count == 1:
if self.hide_null_values:
data = self.hide_values(chart, data)
if self.sort_order != 'none':
data = self.sort_values(chart, data)
if chart.compute_sum and self.chart_type == 'table':
data = self.add_total_to_line_table(chart, data)
self.add_data_to_chart(chart, data, y_labels)
if chart.axis_count == 1:
data = self.process_one_dimensional_data(chart, data)
self.add_data_to_chart(chart, data, y_labels)
else:
data = response['data']
chart.x_labels = data['x_labels']
chart.axis_count = min(len(data['series']), 2)
self.prepare_chart(chart, width, height)
if chart.axis_count == 1:
data['series'][0]['data'] = self.process_one_dimensional_data(
chart, data['series'][0]['data']
)
if self.chart_type == 'pie':
data["series"] = [
{"label": label, "data": [data]}
for label, data in zip(chart.x_labels, data["series"][0]["data"])
if data
]
for serie in data['series']:
chart.add(serie['label'], serie['data'])
return chart
@ -277,7 +294,6 @@ class ChartNgCell(CellBase):
else:
chart.axis_count = 2
chart.show_legend = bool(len(response['axis']) > 1)
chart.compute_sum = bool(response.get('measure') == 'integer' and chart.axis_count > 0)
formatter = self.get_value_formatter(response.get('unit'), response.get('measure'))
@ -296,6 +312,7 @@ class ChartNgCell(CellBase):
chart.config.explicit_size = True
chart.config.js = [os.path.join(settings.STATIC_URL, 'js/pygal-tooltips.js')]
chart.show_legend = bool(chart.axis_count > 1)
chart.truncate_legend = 30
# matplotlib tab10 palette
chart.config.style.colors = (
@ -319,6 +336,15 @@ class ChartNgCell(CellBase):
if width and width < 500:
chart.truncate_legend = 15
def process_one_dimensional_data(self, chart, data):
if self.hide_null_values:
data = self.hide_values(chart, data)
if self.sort_order != 'none':
data = self.sort_values(chart, data)
if getattr(chart, 'compute_sum', True) and self.chart_type == 'table':
data = self.add_total_to_line_table(chart, data)
return data
@staticmethod
def hide_values(chart, data):
x_labels, new_data = [], []
@ -344,7 +370,7 @@ class ChartNgCell(CellBase):
def add_total_to_line_table(chart, data):
# workaround pygal
chart.compute_sum = False
data.append(sum(data))
data.append(sum(x for x in data if x is not None))
chart.x_labels.append(gettext('Total'))
return data

3
combo/settings.py
View File

@ -351,6 +351,9 @@ COMBO_MAP_LAYER_ASSET_SLOTS = {}
# known services
KNOWN_SERVICES = {}
# known services exposing statistics
STATISTICS_PROVIDERS = []
# PWA Settings
PWA_VAPID_PUBLIK_KEY = None
PWA_VAPID_PRIVATE_KEY = None

215
tests/test_dataviz.py
View File

@ -233,6 +233,65 @@ def bijoe_mock(url, request):
return {'content': json.dumps(response), 'request': request, 'status_code': 200}
STATISTICS_LIST = {
'data': [
{
'url': 'https://authentic.example.com/api/statistics/one-serie/',
'name': 'One serie stat',
'id': 'one-serie',
},
{
'url': 'https://authentic.example.com/api/statistics/two-series/',
'name': 'Two series stat',
'id': 'two-series',
},
{
'url': 'https://authentic.example.com/api/statistics/no-data/',
'name': 'No data stat',
'id': 'no-data',
},
{
'url': 'https://authentic.example.com/api/statistics/not-found/',
'name': '404 not found stat',
'id': 'not-found',
},
]
}
def new_api_mock(url, request):
if url.path == '/visualization/json/': # nothing from bijoe
return {'content': b'{}', 'request': request, 'status_code': 200}
if url.path == '/api/statistics/':
return {'content': json.dumps(STATISTICS_LIST), 'request': request, 'status_code': 200}
if url.path == '/api/statistics/one-serie/':
response = {
'data': {
'series': [{'data': [None, 16, 2], 'label': 'Serie 1'}],
'x_labels': ['2020-10', '2020-11', '2020-12'],
},
}
return {'content': json.dumps(response), 'request': request, 'status_code': 200}
if url.path == '/api/statistics/two-series/':
response = {
'data': {
'series': [
{'data': [None, 16, 2], 'label': 'Serie 1'},
{'data': [2, 1, None], 'label': 'Serie 2'},
],
'x_labels': ['2020-10', '2020-11', '2020-12'],
},
}
return {'content': json.dumps(response), 'request': request, 'status_code': 200}
if url.path == '/api/statistics/no-data/':
response = {
'data': {'series': [], 'x_labels': []},
}
return {'content': json.dumps(response), 'request': request, 'status_code': 200}
if url.path == '/api/statistics/not-found/':
return {'content': b'', 'request': request, 'status_code': 404}
@pytest.fixture
@with_httmock(bijoe_mock)
def statistics(settings):
@ -246,6 +305,25 @@ def statistics(settings):
assert Statistic.objects.count() == len(VISUALIZATION_JSON)
@pytest.fixture
@with_httmock(new_api_mock)
def new_api_statistics(settings):
settings.KNOWN_SERVICES = {
'authentic': {
'connection': {
'title': 'Connection',
'url': 'https://authentic.example.com',
'secret': 'combo',
'orig': 'combo',
}
}
}
settings.STATISTICS_PROVIDERS = ['authentic']
appconfig = apps.get_app_config('dataviz')
appconfig.hourly()
assert Statistic.objects.count() == len(STATISTICS_LIST['data'])
@with_httmock(bijoe_mock)
def test_chartng_cell(app, statistics):
page = Page(title='One', slug='index')
@ -353,6 +431,47 @@ def test_chartng_cell(app, statistics):
chart = cell.get_chart()
@with_httmock(new_api_mock)
def test_chartng_cell_new_api(app, new_api_statistics):
page = Page.objects.create(title='One', slug='index')
cell = ChartNgCell(page=page, order=1)
cell.statistic = Statistic.objects.get(slug='one-serie')
cell.save()
chart = cell.get_chart()
assert chart.__class__.__name__ == 'Bar'
assert chart.x_labels == ['2020-10', '2020-11', '2020-12']
assert chart.raw_series == [([None, 16, 2], {'title': 'Serie 1'},)]
cell.chart_type = 'pie'
chart = cell.get_chart()
assert chart.__class__.__name__ == 'Pie'
assert chart.x_labels == ['2020-10', '2020-11', '2020-12']
assert chart.raw_series == [
([16], {'title': '2020-11'}),
([2], {'title': '2020-12'}),
]
cell.statistic = Statistic.objects.get(slug='two-series')
cell.save()
chart = cell.get_chart()
assert chart.x_labels == ['2020-10', '2020-11', '2020-12']
assert chart.raw_series == [([None, 16, 2], {'title': 'Serie 1'}), ([2, 1, None], {'title': 'Serie 2'})]
cell.statistic = Statistic.objects.get(slug='no-data')
cell.save()
chart = cell.get_chart()
assert chart.x_labels == []
assert chart.raw_series == []
cell.statistic = Statistic.objects.get(slug='not-found')
cell.save()
with pytest.raises(HTTPError):
chart = cell.get_chart()
@with_httmock(bijoe_mock)
def test_chartng_cell_hide_null_values(app, statistics):
page = Page(title='One', slug='index')
@ -445,6 +564,19 @@ def test_chartng_cell_hide_null_values(app, statistics):
assert chart.raw_series == [([], {'title': ''})]
@with_httmock(new_api_mock)
def test_chartng_cell_hide_null_values_new_api(app, new_api_statistics):
page = Page.objects.create(title='One', slug='index')
cell = ChartNgCell(page=page, order=1, hide_null_values=True)
cell.statistic = Statistic.objects.get(slug='one-serie')
cell.hide_null_values = True
cell.save()
chart = cell.get_chart()
assert chart.x_labels == ['2020-11', '2020-12']
assert chart.raw_series == [([16, 2], {'title': 'Serie 1'},)]
@with_httmock(bijoe_mock)
def test_chartng_cell_sort_order_alpha(app, statistics):
page = Page(title='One', slug='index')
@ -615,6 +747,19 @@ def test_chartng_cell_sort_order_desc(app, statistics):
]
@with_httmock(new_api_mock)
def test_chartng_cell_sort_order_new_api(app, new_api_statistics):
page = Page.objects.create(title='One', slug='index')
cell = ChartNgCell(page=page, order=1)
cell.statistic = Statistic.objects.get(slug='one-serie')
cell.sort_order = 'desc'
cell.save()
chart = cell.get_chart()
assert chart.x_labels == ['2020-11', '2020-12', '2020-10']
assert chart.raw_series == [([16, 2, None], {'title': 'Serie 1'},)]
@with_httmock(bijoe_mock)
def test_chartng_cell_view(app, normal_user, statistics):
page = Page(title='One', slug='index')
@ -719,6 +864,31 @@ def test_chartng_cell_view(app, normal_user, statistics):
assert not 'cell' in resp.text
@with_httmock(new_api_mock)
def test_chartng_cell_view_new_api(app, normal_user, new_api_statistics):
page = Page.objects.create(title='One', slug='index')
cell = ChartNgCell(page=page, order=1, placeholder='content')
cell.statistic = Statistic.objects.get(slug='one-serie')
cell.save()
location = '/api/dataviz/graph/%s/' % cell.id
resp = app.get('/')
assert 'min-height: 250px' in resp.text
assert location in resp.text
# table visualization
cell.chart_type = 'table'
cell.save()
resp = app.get('/')
assert '<td>18</td>' in resp.text
# deleted visualization
cell.statistic = Statistic.objects.get(slug='not-found')
cell.save()
resp = app.get(location)
assert 'not found' in resp.text
@with_httmock(bijoe_mock)
def test_chartng_cell_manager(app, admin_user, statistics):
page = Page(title='One', slug='index')
@ -749,6 +919,21 @@ def test_chartng_cell_manager(app, admin_user, statistics):
assert 'Unavailable Stat' in resp.text
@with_httmock(new_api_mock)
def test_chartng_cell_manager_new_api(app, admin_user, new_api_statistics):
page = Page.objects.create(title='One', slug='index')
cell = ChartNgCell(page=page, order=1, placeholder='content')
cell.statistic = Statistic.objects.get(slug='one-serie')
cell.save()
app = login(app)
resp = app.get('/manage/pages/%s/' % page.id)
statistics_field = resp.form['cdataviz_chartngcell-%s-statistic' % cell.id]
assert len(statistics_field.options) == len(STATISTICS_LIST['data']) + 1
assert statistics_field.value == str(cell.statistic.pk)
assert statistics_field.options[3][2] == 'Connection: One serie stat'
@with_httmock(bijoe_mock)
def test_table_cell(app, admin_user, statistics):
page = Page(title='One', slug='index')
@ -792,6 +977,25 @@ def test_table_cell(app, admin_user, statistics):
assert resp.text.count('Total') == 0
@with_httmock(new_api_mock)
def test_table_cell_new_api(app, admin_user, new_api_statistics):
page = Page.objects.create(title='One', slug='index')
cell = ChartNgCell(page=page, order=1, placeholder='content')
cell.statistic = Statistic.objects.get(slug='one-serie')
cell.chart_type = 'table'
cell.save()
app = login(app)
resp = app.get('/')
assert resp.text.count('Total') == 1
cell.statistic = Statistic.objects.get(slug='two-series')
cell.save()
resp = app.get('/')
assert '21' in resp.text
assert resp.text.count('Total') == 2
def test_dataviz_hourly_unavailable_statistic(statistics, nocache):
all_stats_count = Statistic.objects.count()
assert Statistic.objects.filter(available=True).count() == all_stats_count
@ -875,3 +1079,14 @@ def test_dataviz_cell_migration(settings):
assert cell.statistic.site_slug == 'plop'
assert cell.statistic.service_slug == 'bijoe'
assert cell.statistic.site_title == 'test'
@with_httmock(new_api_mock)
def test_dataviz_api_list_statistics(new_api_statistics):
statistic = Statistic.objects.get(slug='one-serie')
assert statistic.label == 'One serie stat'
assert statistic.site_slug == 'connection'
assert statistic.service_slug == 'authentic'
assert statistic.site_title == 'Connection'
assert statistic.url == 'https://authentic.example.com/api/statistics/one-serie/'
assert statistic.available