entrouvert
/
combo
16
1
Fork 0
Code Issues Pull Requests 12 Releases Wiki Activity

dataviz: add cells to display data from cubes using barcharts and HTML tables (#9098) 

It contains two new cells: CubesBarChart and CubesTable. You must have a cubes
server targeted by the CUBES_URL setting for enabling the cells.
This commit is contained in:
Benjamin Dauvergne 2015-11-23 22:48:14 +01:00
parent 85f0906aaf
commit 842f700b71
13 changed files with 603 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
combo/apps/dataviz/README Normal file
View File

@ -0,0 +1,36 @@
Data visualization cells
========================
Gauge cell
----------
FIXME
Cubes cells
-----------
Those cells are activated by setting the CUBES_URL setting to the root URL of a
Cubes[1] 1.1 server.
Cubes server is accessed using the requests library, you can define custom
parameters for the requests.get() calls by setting CUBES_REQUESTS_PARAMS, for
example to disable SSL certificate validation:
CUBES_REQUESTS_PARAMS = {'verify': False}
The CubesBarChart cell use the Chart.js library (through the XStatic-Chart.js
package) to render bar charts of the selected aggregated data. The y axis
measure the aggregate which is computed, the x axis is the dimension chosen
for the first drill-down axis. The second drilldown axis will be used to
generate multiple datasets, one by dimension point, i.e. results generated for
the second axis will be grouped along the first drilldown axis.
Ordering by drilldown axis is automatically done using implicit ordering defined
by the Cubes model.
The CubesTable render the same data as CubesBarChart but by using HTML tables.
The first drilldown axis is used for the row headers and the second drilldown
axis for the column headers. By using the two axis at the same time you can make
pivot tables.
[1]: https://pythonhosted.org/cubes/

14
combo/apps/dataviz/__init__.py
View File

@ -14,7 +14,12 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import re
import django.apps
from django.core import checks
from django.conf import settings
class AppConfig(django.apps.AppConfig):
name = 'combo.apps.dataviz'
@ -23,4 +28,13 @@ class AppConfig(django.apps.AppConfig):
from . import urls
return urls.urlpatterns
def ready(self):
@checks.register('settings')
def check_settings(**kwargs):
# Check if CUBES_URL is a proper URL string
if (getattr(settings, 'CUBES_URL', None) is not None
and (not isinstance(settings.CUBES_URL, str)
or not re.match(r'https?://', settings.CUBES_URL))):
yield checks.Error('settings.CUBES_URL must be an HTTP URL')
default_app_config = 'combo.apps.dataviz.AppConfig'

88
combo/apps/dataviz/forms.py Normal file
View File

@ -0,0 +1,88 @@
# combo - content management system
# Copyright (C) 2015 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import json
from django.utils.translation import ugettext_lazy as _
from django import forms
from django.conf import settings
from django.core.exceptions import ValidationError
from .models import CubesBarChart
from .utils import get_cubes, get_cube, get_drilldown
class CubesBarChartForm(forms.ModelForm):
EMPTY = [(u'', _('None'))]
class Meta:
model = CubesBarChart
fields = ('title', 'url', 'cube', 'aggregate1', 'drilldown1', 'drilldown2',
'other_parameters')
def __init__(self, *args, **kwargs):
super(CubesBarChartForm, self).__init__(*args, **kwargs)
for field in ('cube', 'aggregate1', 'drilldown1', 'drilldown2'):
self.fields[field] = forms.ChoiceField(
label=self.fields[field].label,
initial=self.fields[field].initial,
required=False,
choices=self.EMPTY)
if getattr(settings, 'CUBES_URL', None):
cube_choices = self.get_cubes_choices()
if cube_choices:
self.fields['cube'].choices = cube_choices
aggregate1_choices = self.get_aggregate_choices()
if aggregate1_choices:
self.fields['aggregate1'].choices = aggregate1_choices
drilldown_choices = self.get_drilldown_choices()
if drilldown_choices:
self.fields['drilldown1'].choices = drilldown_choices
self.fields['drilldown2'].choices = drilldown_choices
def clean_other_parameters(self):
other_parameters = self.cleaned_data['other_parameters']
if other_parameters:
try:
decoded = json.loads(other_parameters)
assert isinstance(decoded, dict)
for key, value in decoded.iteritems():
assert isinstance(key, unicode)
assert isinstance(value, unicode)
except (ValueError, AssertionError):
raise ValidationError(_('Other parameter must be a JSON object containing only '
'strings'))
return other_parameters
def get_cubes_choices(self):
cubes = get_cubes()
return self.EMPTY + [(cube['name'], cube.get('label')) for cube in cubes]
def get_aggregate_choices(self):
cube = self.instance.cube
if cube:
cube = get_cube(cube)
if cube:
return self.EMPTY + [(ag['name'], ag['label']) for ag in cube.get('aggregates', [])]
return []
def get_drilldown_choices(self):
cube = self.instance.cube
if cube:
choices = get_drilldown(cube)
if choices:
return self.EMPTY + choices
return []

64
combo/apps/dataviz/migrations/0003_cubesbarchart_cubestable.py Normal file
View File

@ -0,0 +1,64 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('auth', '0001_initial'),
('data', '0012_auto_20151029_1535'),
('dataviz', '0002_gauge_jsonp_data_source'),
]
operations = [
migrations.CreateModel(
name='CubesBarChart',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('placeholder', models.CharField(max_length=20)),
('order', models.PositiveIntegerField()),
('slug', models.SlugField(verbose_name='Slug', blank=True)),
('public', models.BooleanField(default=True, verbose_name='Public')),
('restricted_to_unlogged', models.BooleanField(default=False, verbose_name='Restrict to unlogged users')),
('title', models.CharField(max_length=150, null=True, verbose_name='Title', blank=True)),
('url', models.URLField(max_length=150, null=True, verbose_name='URL', blank=True)),
('cube', models.CharField(max_length=256, null=True, verbose_name='Cube', blank=True)),
('aggregate1', models.CharField(max_length=64, null=True, verbose_name='Aggregate', blank=True)),
('drilldown1', models.CharField(max_length=64, null=True, verbose_name='Drilldown 1', blank=True)),
('drilldown2', models.CharField(max_length=64, null=True, verbose_name='Drilldown 2', blank=True)),
('other_parameters', models.TextField(null=True, verbose_name='Other parameters', blank=True)),
('groups', models.ManyToManyField(to='auth.Group', verbose_name='Groups', blank=True)),
('page', models.ForeignKey(to='data.Page')),
],
options={
'verbose_name': 'Cubes Barchart',
},
bases=(models.Model,),
),
migrations.CreateModel(
name='CubesTable',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('placeholder', models.CharField(max_length=20)),
('order', models.PositiveIntegerField()),
('slug', models.SlugField(verbose_name='Slug', blank=True)),
('public', models.BooleanField(default=True, verbose_name='Public')),
('restricted_to_unlogged', models.BooleanField(default=False, verbose_name='Restrict to unlogged users')),
('title', models.CharField(max_length=150, null=True, verbose_name='Title', blank=True)),
('url', models.URLField(max_length=150, null=True, verbose_name='URL', blank=True)),
('cube', models.CharField(max_length=256, null=True, verbose_name='Cube', blank=True)),
('aggregate1', models.CharField(max_length=64, null=True, verbose_name='Aggregate', blank=True)),
('drilldown1', models.CharField(max_length=64, null=True, verbose_name='Drilldown 1', blank=True)),
('drilldown2', models.CharField(max_length=64, null=True, verbose_name='Drilldown 2', blank=True)),
('other_parameters', models.TextField(null=True, verbose_name='Other parameters', blank=True)),
('groups', models.ManyToManyField(to='auth.Group', verbose_name='Groups', blank=True)),
('page', models.ForeignKey(to='data.Page')),
],
options={
'verbose_name': 'Cubes Table',
},
bases=(models.Model,),
),
]

129
combo/apps/dataviz/models.py
View File

@ -14,9 +14,13 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import json
from collections import OrderedDict
from django.core.urlresolvers import reverse
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.conf import settings
from combo.data.models import CellBase
from combo.data.library import register_cell_class
@ -26,10 +30,8 @@ from combo.data.library import register_cell_class
class Gauge(CellBase):
title = models.CharField(_('Title'), max_length=150, blank=True, null=True)
url = models.URLField(_('URL'), max_length=150, blank=True, null=True)
data_source = models.CharField(_('Data Source'), max_length=150,
blank=True, null=True)
jsonp_data_source = models.BooleanField(_('Use JSONP to get data'),
default=True)
data_source = models.CharField(_('Data Source'), max_length=150, blank=True, null=True)
jsonp_data_source = models.BooleanField(_('Use JSONP to get data'), default=True)
max_value = models.PositiveIntegerField(_('Max Value'), blank=True, null=True)
template_name = 'combo/gauge-cell.html'
@ -55,3 +57,122 @@ class Gauge(CellBase):
'data_source_url': data_source_url,
'jsonp': self.jsonp_data_source,
}
class BaseCubesChart(CellBase):
title = models.CharField(_('Title'), max_length=150, blank=True, null=True)
url = models.URLField(_('URL'), max_length=150, blank=True, null=True)
cube = models.CharField(verbose_name=_('Cube'), max_length=256, blank=True, null=True)
aggregate1 = models.CharField(verbose_name=_('Aggregate'), max_length=64, blank=True, null=True)
drilldown1 = models.CharField(verbose_name=_('Drilldown 1'), max_length=64, blank=True,
null=True)
drilldown2 = models.CharField(verbose_name=_('Drilldown 2'), max_length=64, blank=True,
null=True)
other_parameters = models.TextField(verbose_name=_('Other parameters'), blank=True, null=True)
class Meta:
abstract = True
@classmethod
def is_enabled(self):
return bool(getattr(settings, 'CUBES_URL', None))
def get_additional_label(self):
return self.title
def get_default_form_class(self):
from .forms import CubesBarChartForm
return CubesBarChartForm
def get_cell_extra_context(self):
return {
'cell': self,
'title': self.title,
'url': self.url,
'aggregate': self.get_aggregate(),
}
def get_aggregate(self):
'''Get aggregate defined by chosen cube and the two drildown paths, request ordering of the
data by natural order of each axis.'''
from .utils import get_aggregate, get_cube, compute_levels
aggregate = get_aggregate(name=self.cube,
aggregate1=self.aggregate1,
drilldown1=self.drilldown1,
drilldown2=self.drilldown2,
other_parameters=(json.loads(self.other_parameters) if
self.other_parameters else None))
cube = get_cube(self.cube)
if not aggregate or not cube:
return
label_refs1 = []
key_refs1 = []
if self.drilldown1:
compute_levels(cube, self.drilldown1, label_refs=label_refs1, key_refs=key_refs1)
key_refs2 = []
label_refs2 = []
if self.drilldown2:
compute_levels(cube, self.drilldown2, label_refs=label_refs2, key_refs=key_refs2)
for ag in cube['aggregates']:
if ag['name'] != self.aggregate1:
continue
break
def cell_ref(cell, refs):
return tuple(cell[ref] for ref in refs)
keys1 = OrderedDict()
labels = OrderedDict()
datasets = OrderedDict()
for cell in aggregate['cells']:
label1 = u' / '.join(map(unicode, cell_ref(cell, label_refs1)))
key1 = cell_ref(cell, key_refs1)
labels[key1] = label1
keys1[key1] = 1
if key_refs2:
label2 = u' / '.join(map(unicode, cell_ref(cell, label_refs2)))
key2 = cell_ref(cell, key_refs2)
else:
label2 = ''
key2 = 1
dataset = datasets.setdefault(key2, {'label': label2,
'data': OrderedDict()})
value = cell[self.aggregate1]
dataset['data'][key1] = value
for dataset in datasets.itervalues():
dataset['data'] = [dataset['data'].get(key, 0) for key in keys1]
return {
'labels': labels.values(),
'datasets': [{
'label': dataset['label'],
'data': dataset['data'],
} for dataset in datasets.itervalues()]
}
@register_cell_class
class CubesBarChart(BaseCubesChart):
template_name = 'combo/cubes-barchart.html'
class Media:
js = ('xstatic/Chart.min.js', 'js/combo.cubes-barchart.js')
class Meta:
verbose_name = _('Cubes Barchart')
def get_cell_extra_context(self):
ctx = super(CubesBarChart, self).get_cell_extra_context()
# Need JSON serialization to pass data to Chart.js
ctx['json_aggregate'] = json.dumps(ctx['aggregate'])
return ctx
@register_cell_class
class CubesTable(BaseCubesChart):
template_name = 'combo/cubes-table.html'
class Meta:
verbose_name = _('Cubes Table')

89
combo/apps/dataviz/static/js/combo.cubes-barchart.js Normal file
View File

@ -0,0 +1,89 @@
$(function() {
var Colors = {};
Colors.spaced_hsla = function (i, n, s, l, a) {
var h = 360 * i/n;
return "hsla(" + h.toString() + ', ' + s.toString() + '%, ' + l.toString() + '%, ' + a.toString() + ')';
}
$('.combo-cube-aggregate').each(function(idx, elem) {
var cube_url = $(elem).data('cube-url');
var aggregate_url = $(elem).data('aggregate-url');
var model = null;
var ctx = $('canvas', elem)[0].getContext("2d");
var id = $(elem).data('combo-cube-aggregate-id');
var option = {
//Boolean - Whether the scale should start at zero, or an order of magnitude down from the lowest value
scaleBeginAtZero : true,
//Boolean - Whether grid lines are shown across the chart
scaleShowGridLines : true,
//String - Colour of the grid lines
scaleGridLineColor : "rgba(0,0,0,.05)",
//Number - Width of the grid lines
scaleGridLineWidth : 1,
//Boolean - Whether to show horizontal lines (except X axis)
scaleShowHorizontalLines: true,
//Boolean - Whether to show vertical lines (except Y axis)
scaleShowVerticalLines: true,
//Boolean - If there is a stroke on each bar
barShowStroke : true,
//Number - Pixel width of the bar stroke
barStrokeWidth : 2,
//Number - Spacing between each of the X value sets
barValueSpacing : 5,
//Number - Spacing between data sets within X values
barDatasetSpacing : 1,
//String - A legend template
legendTemplate : "ul",
multiTooltipTemplate: "<%= datasetLabel %>: <%= value %>",
responsive: true,
}
var data = window['combo_cube_aggregate_' + id];
// Set one color by dataset
var n = data.datasets.length;
for (var i = 0; i < n; i++) {
var dataset = data.datasets[i];
$.extend(dataset, {
fillColor: Colors.spaced_hsla(i, n, 100, 30, 0.5),
strokeColor: Colors.spaced_hsla(i, n, 100, 30, 0.75),
highlightFill: Colors.spaced_hsla(i, n, 100, 30, 0.75),
highlightStroke: Colors.spaced_hsla(i, n, 100, 30, 1)
})
}
var clone = function(obj){
var objClone = {};
for (var key in obj) {
if (obj.hasOwnProperty(key)) objClone[key] = obj[key];
};
return objClone;
}
var chart = new Chart(ctx).Bar(data, option);
if (chart.datasets.length == 1) {
// Set one color by bar
var n = chart.datasets[0].bars.length;
for (var i = 0; i < n; i++) {
var bar = chart.datasets[0].bars[i];
$.extend(bar, {
fillColor: Colors.spaced_hsla(i, n, 100, 30, 0.5),
strokeColor: Colors.spaced_hsla(i, n, 100, 30, 0.75),
highlightFill: Colors.spaced_hsla(i, n, 100, 30, 0.75),
highlightStroke: Colors.spaced_hsla(i, n, 100, 30, 1)
})
bar['_saved'] = clone(bar);
bar.update();
}
}
window.chart = chart;
})
})

12
combo/apps/dataviz/templates/combo/cubes-barchart.html Normal file
View File

@ -0,0 +1,12 @@
<script>
var combo_cube_aggregate_{{ cell.id }} = {{ json_aggregate|safe }};
</script>
<div
class="combo-cube-aggregate"
data-combo-cube-aggregate-id="{{ cell.id }}">
{% if title %}
{% if url %}<a href="{{url}}">{% endif %}{{title}}{% if url %}</a>{% endif %}
{% endif %}
<canvas style="width: 100%;">
</canvas>
</div>

23
combo/apps/dataviz/templates/combo/cubes-table.html Normal file
View File

@ -0,0 +1,23 @@
<table class="combo-cube-table">
{% if title %}
<caption>
{% if url %}<a href="{{url}}">{% endif %}{{title}}{% if url %}</a>{% endif %}
</caption>
{% endif %}
<thead>
<th></th>
{% for label in aggregate.labels %}
<td>{{ label }}</td>
{% endfor %}
</thead>
<tbody>
{% for dataset in aggregate.datasets %}
<tr>
<th>{{ dataset.label }}</th>
{% for value in dataset.data %}
<td>{{ value }}</td>
{% endfor %}
</tr>
{% endfor %}
</tbody>
</table>

3
combo/apps/dataviz/urls.py
View File

@ -18,7 +18,8 @@ from django.conf.urls import patterns, url
from .views import ajax_gauge_count
urlpatterns = patterns('',
urlpatterns = patterns(
'',
url(r'^ajax/gauge-count/(?P<cell>[\w_-]+)/$',
ajax_gauge_count, name='combo-ajax-gauge-count'),
)

146
combo/apps/dataviz/utils.py Normal file
View File

@ -0,0 +1,146 @@
# combo - content management system
# Copyright (C) 2015 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import requests
from requests.exceptions import RequestException
import urlparse
from django.conf import settings
def get_requests_params():
return getattr(settings, 'CUBES_REQUEST_PARAMS', {})
def get_cubes():
try:
r = requests.get(urlparse.urljoin(settings.CUBES_URL, 'cubes'), **get_requests_params)
except RequestException:
return []
try:
return r.json()
except ValueError:
return []
def get_cube(name):
model_url = urlparse.urljoin(settings.CUBES_URL, 'cube/%s/model' % name)
try:
r = requests.get(model_url, **get_requests_params)
except RequestException:
return None
try:
return r.json()
except ValueError:
return None
def get_drilldown(name):
cube = get_cube(name)
if not cube:
return []
l = []
for dimension in cube.get('dimensions', []):
dim_name = dimension['name']
dim_label = dimension.get('label') or dim_name
if dimension.get('levels'):
levels = {}
for level in dimension['levels']:
levels[level['name']] = level.get('label') or level['name']
if dimension.get('hierarchies'):
for hierarchy in dimension['hierarchies']:
h_name = hierarchy['name']
h_label = hierarchy.get('label') or h_name
if h_name == 'default':
h_label = ''
for level in hierarchy['levels']:
labels = filter(None, [dim_label, h_label, levels[level]])
label = ' - '.join(labels)
name = '%s@%s:%s' % (dim_name, h_name, level)
l.append((name, label))
else:
raise NotImplementedError
else:
l.append((dim_name, dim_label))
return l
def compute_levels(cube, drilldown, key_refs=None, label_refs=None):
from .utils import get_attribute_ref
dim = drilldown.split('@')[0]
hier = drilldown.split('@')[1].split(':')[0]
lev = drilldown.split(':')[1]
for dimension in cube['dimensions']:
if dimension['name'] != dim:
continue
level_label_refs = {}
level_key_refs = {}
for level in dimension['levels']:
level_key_refs[level['name']] = get_attribute_ref(level, level['key'])
level_label_refs[level['name']] = get_attribute_ref(level, level['label_attribute'])
for hierarchy in dimension['hierarchies']:
if hierarchy['name'] != hier:
continue
for level in hierarchy['levels']:
if key_refs is not None:
key_refs.append(level_key_refs[level])
if label_refs is not None:
label_refs.append(level_label_refs[level])
if level == lev:
break
break
break
def get_aggregate(name, aggregate1, drilldown1, drilldown2, other_parameters=None):
if not name:
return None
cube = get_cube(name)
aggregate_url = urlparse.urljoin(settings.CUBES_URL, 'cube/%s/aggregate' % name)
if not aggregate1:
return None
try:
params = {'aggregate': aggregate1}
drilldowns = []
key_refs = []
if drilldown1:
compute_levels(cube, drilldown1, key_refs=key_refs)
drilldowns.append(drilldown1)
if drilldown2:
compute_levels(cube, drilldown2, key_refs=key_refs)
drilldowns.append(drilldown2)
if drilldowns:
params['drilldown'] = drilldowns
if key_refs:
params['order'] = key_refs
if other_parameters:
params.update(other_parameters)
r = requests.get(aggregate_url, params=params, **get_requests_params)
except RequestException:
return None
try:
return r.json()
except ValueError:
return None
def get_attribute_ref(level, name):
for attribute in level['attributes']:
if attribute['name'] == name:
return attribute['ref']

4
combo/apps/dataviz/views.py
View File

@ -14,11 +14,9 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import json
import requests
from django.conf import settings
from django.http import Http404, HttpResponse
from django.http import HttpResponse
from .models import Gauge

2
combo/settings.py
View File

@ -63,6 +63,8 @@ INSTALLED_APPS = (
'combo.apps.wcs',
'combo.apps.publik',
'combo.apps.family',
'combo.apps.dataviz',
'xstatic.pkg.chart_js',
)
INSTALLED_APPS = plugins.register_plugins_apps(INSTALLED_APPS)

1
setup.py
View File

@ -109,6 +109,7 @@ setup(
'feedparser',
'django-jsonfield',
'requests',
'XStatic-Chart.js',
],
zip_safe=False,
cmdclass={