diff --git a/combo/apps/dataviz/README b/combo/apps/dataviz/README new file mode 100644 index 00000000..2ed2860c --- /dev/null +++ b/combo/apps/dataviz/README @@ -0,0 +1,36 @@ +Data visualization cells +======================== + +Gauge cell +---------- + +FIXME + +Cubes cells +----------- + +Those cells are activated by setting the CUBES_URL setting to the root URL of a +Cubes[1] 1.1 server. + +Cubes server is accessed using the requests library, you can define custom +parameters for the requests.get() calls by setting CUBES_REQUESTS_PARAMS, for +example to disable SSL certificate validation: + + CUBES_REQUESTS_PARAMS = {'verify': False} + +The CubesBarChart cell use the Chart.js library (through the XStatic-Chart.js +package) to render bar charts of the selected aggregated data. The y axis +measure the aggregate which is computed, the x axis is the dimension chosen +for the first drill-down axis. The second drilldown axis will be used to +generate multiple datasets, one by dimension point, i.e. results generated for +the second axis will be grouped along the first drilldown axis. + +Ordering by drilldown axis is automatically done using implicit ordering defined +by the Cubes model. + +The CubesTable render the same data as CubesBarChart but by using HTML tables. +The first drilldown axis is used for the row headers and the second drilldown +axis for the column headers. By using the two axis at the same time you can make +pivot tables. + +[1]: https://pythonhosted.org/cubes/ diff --git a/combo/apps/dataviz/__init__.py b/combo/apps/dataviz/__init__.py index 6712bb1c..4e67acd4 100644 --- a/combo/apps/dataviz/__init__.py +++ b/combo/apps/dataviz/__init__.py @@ -14,7 +14,12 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import re + import django.apps +from django.core import checks +from django.conf import settings + class AppConfig(django.apps.AppConfig): name = 'combo.apps.dataviz' @@ -23,4 +28,13 @@ class AppConfig(django.apps.AppConfig): from . import urls return urls.urlpatterns + def ready(self): + @checks.register('settings') + def check_settings(**kwargs): + # Check if CUBES_URL is a proper URL string + if (getattr(settings, 'CUBES_URL', None) is not None + and (not isinstance(settings.CUBES_URL, str) + or not re.match(r'https?://', settings.CUBES_URL))): + yield checks.Error('settings.CUBES_URL must be an HTTP URL') + default_app_config = 'combo.apps.dataviz.AppConfig' diff --git a/combo/apps/dataviz/forms.py b/combo/apps/dataviz/forms.py new file mode 100644 index 00000000..5a0dd3ef --- /dev/null +++ b/combo/apps/dataviz/forms.py @@ -0,0 +1,88 @@ +# combo - content management system +# Copyright (C) 2015 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import json + +from django.utils.translation import ugettext_lazy as _ +from django import forms +from django.conf import settings +from django.core.exceptions import ValidationError + +from .models import CubesBarChart +from .utils import get_cubes, get_cube, get_drilldown + + +class CubesBarChartForm(forms.ModelForm): + EMPTY = [(u'', _('None'))] + + class Meta: + model = CubesBarChart + fields = ('title', 'url', 'cube', 'aggregate1', 'drilldown1', 'drilldown2', + 'other_parameters') + + def __init__(self, *args, **kwargs): + super(CubesBarChartForm, self).__init__(*args, **kwargs) + for field in ('cube', 'aggregate1', 'drilldown1', 'drilldown2'): + self.fields[field] = forms.ChoiceField( + label=self.fields[field].label, + initial=self.fields[field].initial, + required=False, + choices=self.EMPTY) + if getattr(settings, 'CUBES_URL', None): + cube_choices = self.get_cubes_choices() + if cube_choices: + self.fields['cube'].choices = cube_choices + aggregate1_choices = self.get_aggregate_choices() + if aggregate1_choices: + self.fields['aggregate1'].choices = aggregate1_choices + drilldown_choices = self.get_drilldown_choices() + if drilldown_choices: + self.fields['drilldown1'].choices = drilldown_choices + self.fields['drilldown2'].choices = drilldown_choices + + def clean_other_parameters(self): + other_parameters = self.cleaned_data['other_parameters'] + if other_parameters: + try: + decoded = json.loads(other_parameters) + assert isinstance(decoded, dict) + for key, value in decoded.iteritems(): + assert isinstance(key, unicode) + assert isinstance(value, unicode) + except (ValueError, AssertionError): + raise ValidationError(_('Other parameter must be a JSON object containing only ' + 'strings')) + return other_parameters + + def get_cubes_choices(self): + cubes = get_cubes() + return self.EMPTY + [(cube['name'], cube.get('label')) for cube in cubes] + + def get_aggregate_choices(self): + cube = self.instance.cube + if cube: + cube = get_cube(cube) + if cube: + return self.EMPTY + [(ag['name'], ag['label']) for ag in cube.get('aggregates', [])] + return [] + + def get_drilldown_choices(self): + cube = self.instance.cube + if cube: + choices = get_drilldown(cube) + if choices: + return self.EMPTY + choices + return [] diff --git a/combo/apps/dataviz/migrations/0003_cubesbarchart_cubestable.py b/combo/apps/dataviz/migrations/0003_cubesbarchart_cubestable.py new file mode 100644 index 00000000..cd6ba1f9 --- /dev/null +++ b/combo/apps/dataviz/migrations/0003_cubesbarchart_cubestable.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('auth', '0001_initial'), + ('data', '0012_auto_20151029_1535'), + ('dataviz', '0002_gauge_jsonp_data_source'), + ] + + operations = [ + migrations.CreateModel( + name='CubesBarChart', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('placeholder', models.CharField(max_length=20)), + ('order', models.PositiveIntegerField()), + ('slug', models.SlugField(verbose_name='Slug', blank=True)), + ('public', models.BooleanField(default=True, verbose_name='Public')), + ('restricted_to_unlogged', models.BooleanField(default=False, verbose_name='Restrict to unlogged users')), + ('title', models.CharField(max_length=150, null=True, verbose_name='Title', blank=True)), + ('url', models.URLField(max_length=150, null=True, verbose_name='URL', blank=True)), + ('cube', models.CharField(max_length=256, null=True, verbose_name='Cube', blank=True)), + ('aggregate1', models.CharField(max_length=64, null=True, verbose_name='Aggregate', blank=True)), + ('drilldown1', models.CharField(max_length=64, null=True, verbose_name='Drilldown 1', blank=True)), + ('drilldown2', models.CharField(max_length=64, null=True, verbose_name='Drilldown 2', blank=True)), + ('other_parameters', models.TextField(null=True, verbose_name='Other parameters', blank=True)), + ('groups', models.ManyToManyField(to='auth.Group', verbose_name='Groups', blank=True)), + ('page', models.ForeignKey(to='data.Page')), + ], + options={ + 'verbose_name': 'Cubes Barchart', + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='CubesTable', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('placeholder', models.CharField(max_length=20)), + ('order', models.PositiveIntegerField()), + ('slug', models.SlugField(verbose_name='Slug', blank=True)), + ('public', models.BooleanField(default=True, verbose_name='Public')), + ('restricted_to_unlogged', models.BooleanField(default=False, verbose_name='Restrict to unlogged users')), + ('title', models.CharField(max_length=150, null=True, verbose_name='Title', blank=True)), + ('url', models.URLField(max_length=150, null=True, verbose_name='URL', blank=True)), + ('cube', models.CharField(max_length=256, null=True, verbose_name='Cube', blank=True)), + ('aggregate1', models.CharField(max_length=64, null=True, verbose_name='Aggregate', blank=True)), + ('drilldown1', models.CharField(max_length=64, null=True, verbose_name='Drilldown 1', blank=True)), + ('drilldown2', models.CharField(max_length=64, null=True, verbose_name='Drilldown 2', blank=True)), + ('other_parameters', models.TextField(null=True, verbose_name='Other parameters', blank=True)), + ('groups', models.ManyToManyField(to='auth.Group', verbose_name='Groups', blank=True)), + ('page', models.ForeignKey(to='data.Page')), + ], + options={ + 'verbose_name': 'Cubes Table', + }, + bases=(models.Model,), + ), + ] diff --git a/combo/apps/dataviz/models.py b/combo/apps/dataviz/models.py index 854c3a38..6195ace8 100644 --- a/combo/apps/dataviz/models.py +++ b/combo/apps/dataviz/models.py @@ -14,9 +14,13 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import json +from collections import OrderedDict + from django.core.urlresolvers import reverse from django.db import models from django.utils.translation import ugettext_lazy as _ +from django.conf import settings from combo.data.models import CellBase from combo.data.library import register_cell_class @@ -26,10 +30,8 @@ from combo.data.library import register_cell_class class Gauge(CellBase): title = models.CharField(_('Title'), max_length=150, blank=True, null=True) url = models.URLField(_('URL'), max_length=150, blank=True, null=True) - data_source = models.CharField(_('Data Source'), max_length=150, - blank=True, null=True) - jsonp_data_source = models.BooleanField(_('Use JSONP to get data'), - default=True) + data_source = models.CharField(_('Data Source'), max_length=150, blank=True, null=True) + jsonp_data_source = models.BooleanField(_('Use JSONP to get data'), default=True) max_value = models.PositiveIntegerField(_('Max Value'), blank=True, null=True) template_name = 'combo/gauge-cell.html' @@ -55,3 +57,122 @@ class Gauge(CellBase): 'data_source_url': data_source_url, 'jsonp': self.jsonp_data_source, } + + +class BaseCubesChart(CellBase): + title = models.CharField(_('Title'), max_length=150, blank=True, null=True) + url = models.URLField(_('URL'), max_length=150, blank=True, null=True) + cube = models.CharField(verbose_name=_('Cube'), max_length=256, blank=True, null=True) + aggregate1 = models.CharField(verbose_name=_('Aggregate'), max_length=64, blank=True, null=True) + drilldown1 = models.CharField(verbose_name=_('Drilldown 1'), max_length=64, blank=True, + null=True) + drilldown2 = models.CharField(verbose_name=_('Drilldown 2'), max_length=64, blank=True, + null=True) + other_parameters = models.TextField(verbose_name=_('Other parameters'), blank=True, null=True) + + class Meta: + abstract = True + + @classmethod + def is_enabled(self): + return bool(getattr(settings, 'CUBES_URL', None)) + + def get_additional_label(self): + return self.title + + def get_default_form_class(self): + from .forms import CubesBarChartForm + return CubesBarChartForm + + def get_cell_extra_context(self): + return { + 'cell': self, + 'title': self.title, + 'url': self.url, + 'aggregate': self.get_aggregate(), + } + + def get_aggregate(self): + '''Get aggregate defined by chosen cube and the two drildown paths, request ordering of the + data by natural order of each axis.''' + from .utils import get_aggregate, get_cube, compute_levels + aggregate = get_aggregate(name=self.cube, + aggregate1=self.aggregate1, + drilldown1=self.drilldown1, + drilldown2=self.drilldown2, + other_parameters=(json.loads(self.other_parameters) if + self.other_parameters else None)) + cube = get_cube(self.cube) + if not aggregate or not cube: + return + + label_refs1 = [] + key_refs1 = [] + if self.drilldown1: + compute_levels(cube, self.drilldown1, label_refs=label_refs1, key_refs=key_refs1) + key_refs2 = [] + label_refs2 = [] + if self.drilldown2: + compute_levels(cube, self.drilldown2, label_refs=label_refs2, key_refs=key_refs2) + for ag in cube['aggregates']: + if ag['name'] != self.aggregate1: + continue + break + + def cell_ref(cell, refs): + return tuple(cell[ref] for ref in refs) + + keys1 = OrderedDict() + labels = OrderedDict() + datasets = OrderedDict() + + for cell in aggregate['cells']: + label1 = u' / '.join(map(unicode, cell_ref(cell, label_refs1))) + key1 = cell_ref(cell, key_refs1) + labels[key1] = label1 + keys1[key1] = 1 + if key_refs2: + label2 = u' / '.join(map(unicode, cell_ref(cell, label_refs2))) + key2 = cell_ref(cell, key_refs2) + else: + label2 = '' + key2 = 1 + dataset = datasets.setdefault(key2, {'label': label2, + 'data': OrderedDict()}) + value = cell[self.aggregate1] + dataset['data'][key1] = value + for dataset in datasets.itervalues(): + dataset['data'] = [dataset['data'].get(key, 0) for key in keys1] + + return { + 'labels': labels.values(), + 'datasets': [{ + 'label': dataset['label'], + 'data': dataset['data'], + } for dataset in datasets.itervalues()] + } + + +@register_cell_class +class CubesBarChart(BaseCubesChart): + template_name = 'combo/cubes-barchart.html' + + class Media: + js = ('xstatic/Chart.min.js', 'js/combo.cubes-barchart.js') + + class Meta: + verbose_name = _('Cubes Barchart') + + def get_cell_extra_context(self): + ctx = super(CubesBarChart, self).get_cell_extra_context() + # Need JSON serialization to pass data to Chart.js + ctx['json_aggregate'] = json.dumps(ctx['aggregate']) + return ctx + + +@register_cell_class +class CubesTable(BaseCubesChart): + template_name = 'combo/cubes-table.html' + + class Meta: + verbose_name = _('Cubes Table') diff --git a/combo/apps/dataviz/static/js/combo.cubes-barchart.js b/combo/apps/dataviz/static/js/combo.cubes-barchart.js new file mode 100644 index 00000000..19a9c16f --- /dev/null +++ b/combo/apps/dataviz/static/js/combo.cubes-barchart.js @@ -0,0 +1,89 @@ +$(function() { + var Colors = {}; + + Colors.spaced_hsla = function (i, n, s, l, a) { + var h = 360 * i/n; + return "hsla(" + h.toString() + ', ' + s.toString() + '%, ' + l.toString() + '%, ' + a.toString() + ')'; + } + + $('.combo-cube-aggregate').each(function(idx, elem) { + var cube_url = $(elem).data('cube-url'); + var aggregate_url = $(elem).data('aggregate-url'); + var model = null; + var ctx = $('canvas', elem)[0].getContext("2d"); + var id = $(elem).data('combo-cube-aggregate-id'); + + var option = { + //Boolean - Whether the scale should start at zero, or an order of magnitude down from the lowest value + scaleBeginAtZero : true, + + //Boolean - Whether grid lines are shown across the chart + scaleShowGridLines : true, + + //String - Colour of the grid lines + scaleGridLineColor : "rgba(0,0,0,.05)", + + //Number - Width of the grid lines + scaleGridLineWidth : 1, + + //Boolean - Whether to show horizontal lines (except X axis) + scaleShowHorizontalLines: true, + + //Boolean - Whether to show vertical lines (except Y axis) + scaleShowVerticalLines: true, + + //Boolean - If there is a stroke on each bar + barShowStroke : true, + + //Number - Pixel width of the bar stroke + barStrokeWidth : 2, + + //Number - Spacing between each of the X value sets + barValueSpacing : 5, + + //Number - Spacing between data sets within X values + barDatasetSpacing : 1, + + //String - A legend template + legendTemplate : "ul", + multiTooltipTemplate: "<%= datasetLabel %>: <%= value %>", + responsive: true, + } + var data = window['combo_cube_aggregate_' + id]; + // Set one color by dataset + var n = data.datasets.length; + for (var i = 0; i < n; i++) { + var dataset = data.datasets[i]; + $.extend(dataset, { + fillColor: Colors.spaced_hsla(i, n, 100, 30, 0.5), + strokeColor: Colors.spaced_hsla(i, n, 100, 30, 0.75), + highlightFill: Colors.spaced_hsla(i, n, 100, 30, 0.75), + highlightStroke: Colors.spaced_hsla(i, n, 100, 30, 1) + }) + } + var clone = function(obj){ + var objClone = {}; + for (var key in obj) { + if (obj.hasOwnProperty(key)) objClone[key] = obj[key]; + }; + return objClone; + } + var chart = new Chart(ctx).Bar(data, option); + if (chart.datasets.length == 1) { + // Set one color by bar + var n = chart.datasets[0].bars.length; + for (var i = 0; i < n; i++) { + var bar = chart.datasets[0].bars[i]; + $.extend(bar, { + fillColor: Colors.spaced_hsla(i, n, 100, 30, 0.5), + strokeColor: Colors.spaced_hsla(i, n, 100, 30, 0.75), + highlightFill: Colors.spaced_hsla(i, n, 100, 30, 0.75), + highlightStroke: Colors.spaced_hsla(i, n, 100, 30, 1) + }) + bar['_saved'] = clone(bar); + bar.update(); + } + } + window.chart = chart; + }) +}) diff --git a/combo/apps/dataviz/templates/combo/cubes-barchart.html b/combo/apps/dataviz/templates/combo/cubes-barchart.html new file mode 100644 index 00000000..08f00662 --- /dev/null +++ b/combo/apps/dataviz/templates/combo/cubes-barchart.html @@ -0,0 +1,12 @@ + +
+ {% if title %} + {% if url %}{% endif %}{{title}}{% if url %}{% endif %} + {% endif %} + + +
diff --git a/combo/apps/dataviz/templates/combo/cubes-table.html b/combo/apps/dataviz/templates/combo/cubes-table.html new file mode 100644 index 00000000..7012083a --- /dev/null +++ b/combo/apps/dataviz/templates/combo/cubes-table.html @@ -0,0 +1,23 @@ + + {% if title %} + + {% endif %} + + + {% for label in aggregate.labels %} + + {% endfor %} + + + {% for dataset in aggregate.datasets %} + + + {% for value in dataset.data %} + + {% endfor %} + + {% endfor %} + +
+ {% if url %}{% endif %}{{title}}{% if url %}{% endif %} +
{{ label }}
{{ dataset.label }}{{ value }}
diff --git a/combo/apps/dataviz/urls.py b/combo/apps/dataviz/urls.py index 011cb0ba..8de07141 100644 --- a/combo/apps/dataviz/urls.py +++ b/combo/apps/dataviz/urls.py @@ -18,7 +18,8 @@ from django.conf.urls import patterns, url from .views import ajax_gauge_count -urlpatterns = patterns('', +urlpatterns = patterns( + '', url(r'^ajax/gauge-count/(?P[\w_-]+)/$', ajax_gauge_count, name='combo-ajax-gauge-count'), ) diff --git a/combo/apps/dataviz/utils.py b/combo/apps/dataviz/utils.py new file mode 100644 index 00000000..758cfdf9 --- /dev/null +++ b/combo/apps/dataviz/utils.py @@ -0,0 +1,146 @@ +# combo - content management system +# Copyright (C) 2015 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import requests +from requests.exceptions import RequestException +import urlparse + +from django.conf import settings + + +def get_requests_params(): + return getattr(settings, 'CUBES_REQUEST_PARAMS', {}) + + +def get_cubes(): + try: + r = requests.get(urlparse.urljoin(settings.CUBES_URL, 'cubes'), **get_requests_params) + except RequestException: + return [] + try: + return r.json() + except ValueError: + return [] + + +def get_cube(name): + model_url = urlparse.urljoin(settings.CUBES_URL, 'cube/%s/model' % name) + try: + r = requests.get(model_url, **get_requests_params) + except RequestException: + return None + try: + return r.json() + except ValueError: + return None + + +def get_drilldown(name): + cube = get_cube(name) + if not cube: + return [] + l = [] + for dimension in cube.get('dimensions', []): + dim_name = dimension['name'] + dim_label = dimension.get('label') or dim_name + if dimension.get('levels'): + levels = {} + for level in dimension['levels']: + levels[level['name']] = level.get('label') or level['name'] + if dimension.get('hierarchies'): + for hierarchy in dimension['hierarchies']: + h_name = hierarchy['name'] + h_label = hierarchy.get('label') or h_name + if h_name == 'default': + h_label = '' + for level in hierarchy['levels']: + labels = filter(None, [dim_label, h_label, levels[level]]) + label = ' - '.join(labels) + name = '%s@%s:%s' % (dim_name, h_name, level) + l.append((name, label)) + else: + raise NotImplementedError + else: + l.append((dim_name, dim_label)) + return l + + +def compute_levels(cube, drilldown, key_refs=None, label_refs=None): + from .utils import get_attribute_ref + dim = drilldown.split('@')[0] + hier = drilldown.split('@')[1].split(':')[0] + lev = drilldown.split(':')[1] + + for dimension in cube['dimensions']: + if dimension['name'] != dim: + continue + level_label_refs = {} + level_key_refs = {} + for level in dimension['levels']: + level_key_refs[level['name']] = get_attribute_ref(level, level['key']) + level_label_refs[level['name']] = get_attribute_ref(level, level['label_attribute']) + for hierarchy in dimension['hierarchies']: + if hierarchy['name'] != hier: + continue + for level in hierarchy['levels']: + if key_refs is not None: + key_refs.append(level_key_refs[level]) + if label_refs is not None: + label_refs.append(level_label_refs[level]) + if level == lev: + break + break + break + + +def get_aggregate(name, aggregate1, drilldown1, drilldown2, other_parameters=None): + if not name: + return None + cube = get_cube(name) + aggregate_url = urlparse.urljoin(settings.CUBES_URL, 'cube/%s/aggregate' % name) + if not aggregate1: + return None + try: + params = {'aggregate': aggregate1} + drilldowns = [] + key_refs = [] + if drilldown1: + compute_levels(cube, drilldown1, key_refs=key_refs) + drilldowns.append(drilldown1) + if drilldown2: + compute_levels(cube, drilldown2, key_refs=key_refs) + drilldowns.append(drilldown2) + if drilldowns: + params['drilldown'] = drilldowns + + if key_refs: + params['order'] = key_refs + if other_parameters: + params.update(other_parameters) + + r = requests.get(aggregate_url, params=params, **get_requests_params) + except RequestException: + return None + try: + return r.json() + except ValueError: + return None + + +def get_attribute_ref(level, name): + for attribute in level['attributes']: + if attribute['name'] == name: + return attribute['ref'] diff --git a/combo/apps/dataviz/views.py b/combo/apps/dataviz/views.py index 4fafda66..1e95d935 100644 --- a/combo/apps/dataviz/views.py +++ b/combo/apps/dataviz/views.py @@ -14,11 +14,9 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import json import requests -from django.conf import settings -from django.http import Http404, HttpResponse +from django.http import HttpResponse from .models import Gauge diff --git a/combo/settings.py b/combo/settings.py index 599808f9..82921e60 100644 --- a/combo/settings.py +++ b/combo/settings.py @@ -63,6 +63,8 @@ INSTALLED_APPS = ( 'combo.apps.wcs', 'combo.apps.publik', 'combo.apps.family', + 'combo.apps.dataviz', + 'xstatic.pkg.chart_js', ) INSTALLED_APPS = plugins.register_plugins_apps(INSTALLED_APPS) diff --git a/setup.py b/setup.py index a3a872ff..ed9fd8b1 100644 --- a/setup.py +++ b/setup.py @@ -109,6 +109,7 @@ setup( 'feedparser', 'django-jsonfield', 'requests', + 'XStatic-Chart.js', ], zip_safe=False, cmdclass={