misc: add visualization import/export (#30854)
This commit is contained in:
parent
b371cff33e
commit
44b2aec236
|
|
@ -0,0 +1,38 @@
|
|||
# bijoe - BI dashboard
|
||||
# Copyright (C) 2020 Entr'ouvert
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify it
|
||||
# under the terms of the GNU Affero General Public License as published
|
||||
# by the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from bijoe.utils import export_site
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'Export the site'
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
'--output', metavar='FILE', default=None,
|
||||
help='name of a file to write output to')
|
||||
|
||||
def handle(self, *args, **options):
|
||||
if options['output']:
|
||||
output = open(options['output'], 'w')
|
||||
else:
|
||||
output = sys.stdout
|
||||
json.dump(export_site(), output, indent=4)
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
# bijoe - BI dashboard
|
||||
# Copyright (C) 2020 Entr'ouvert
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify it
|
||||
# under the terms of the GNU Affero General Public License as published
|
||||
# by the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from bijoe.utils import import_site
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'Import an exported site'
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
'filename', metavar='FILENAME', type=str,
|
||||
help='name of file to import')
|
||||
parser.add_argument(
|
||||
'--clean', action='store_true', default=False,
|
||||
help='Clean site before importing')
|
||||
parser.add_argument(
|
||||
'--if-empty', action='store_true', default=False,
|
||||
help='Import only if site is empty')
|
||||
|
||||
def handle(self, filename, **options):
|
||||
if filename == '-':
|
||||
fd = sys.stdin
|
||||
else:
|
||||
fd = open(filename)
|
||||
import_site(json.load(fd), if_empty=options['if_empty'], clean=options['clean'])
|
||||
|
|
@ -6,10 +6,22 @@
|
|||
<a>{% trans "Homepage" %}</a>
|
||||
{% endblock %}
|
||||
|
||||
{% block appbar %}
|
||||
<h2>{% trans "Visualizations" %}</h2>
|
||||
<span class="actions">
|
||||
<a class="extra-actions-menu-opener"></a>
|
||||
</span>
|
||||
<ul class="extra-actions-menu">
|
||||
<li><a rel="popup" href="{% url 'visualizations-import' %}">{% trans 'Import' %}</a></li>
|
||||
<li><a download href="{% url 'visualizations-export' %}">{% trans 'Export' %}</a></li>
|
||||
</ul>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
{% if visualizations %}
|
||||
<h2>{% trans "Visualizations" %}</h2>
|
||||
{% include "bijoe/visualizations_list.html" %}
|
||||
{% else %}
|
||||
{% trans "No visualizations to display yet." %}
|
||||
{% endif %}
|
||||
{% if warehouses %}
|
||||
<h2>{% trans "Data sources" %}</h2>
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
<a rel="popup" class="button" href="{% url "rename-visualization" pk=object.pk %}">{% trans "Rename" %}</a>
|
||||
<a rel="popup" class="button" href="{% url "delete-visualization" pk=object.pk %}">{% trans "Delete" %}</a>
|
||||
<a class="button" href="{% url "visualization-ods" pk=object.pk %}">{% trans "Export as ODS" %}</a>
|
||||
<a download class="button" href="{% url "export-visualization" pk=object.pk %}">{% trans "Export as JSON" %}</a>
|
||||
<a href="{{ iframe_url }}" class="button">{% trans "URL for IFRAME" %}</a>
|
||||
{% endblock %}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,14 @@
|
|||
{% endblock %}
|
||||
|
||||
{% block appbar %}
|
||||
<h2>{% trans "Visualizations" %}</h2>
|
||||
<h2>{% trans "Visualizations" %}</h2>
|
||||
<span class="actions">
|
||||
<a class="extra-actions-menu-opener"></a>
|
||||
</span>
|
||||
<ul class="extra-actions-menu">
|
||||
<li><a rel="popup" href="{% url 'visualizations-import' %}">{% trans 'Import' %}</a></li>
|
||||
<li><a download href="{% url 'visualizations-export' %}">{% trans 'Export' %}</a></li>
|
||||
</ul>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
{% extends "bijoe/base.html" %}
|
||||
{% load i18n %}
|
||||
|
||||
{% block appbar %}
|
||||
<h2>{% trans "Visualizations Import" %}</h2>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<form method="post" enctype="multipart/form-data">
|
||||
{% csrf_token %}
|
||||
{{ form.as_p }}
|
||||
<div class="buttons">
|
||||
<button class="submit-button">{% trans "Import" %}</button>
|
||||
<a class="cancel" href="{% url 'homepage' %}">{% trans 'Cancel' %}</a>
|
||||
</div>
|
||||
</form>
|
||||
{% endblock %}
|
||||
|
|
@ -19,14 +19,13 @@ import glob
|
|||
import json
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import connection
|
||||
from django.db import connection, transaction
|
||||
from django.utils.translation import ugettext as _
|
||||
try:
|
||||
from functools import lru_cache
|
||||
except ImportError:
|
||||
from django.utils.lru_cache import lru_cache
|
||||
|
||||
|
||||
from .schemas import Warehouse
|
||||
|
||||
|
||||
|
|
@ -68,3 +67,29 @@ def human_join(l):
|
|||
if len(l) > 2:
|
||||
l = u', '.join(l[:-1]), l[-1]
|
||||
return _(u'{0} and {1}').format(l[0], l[1])
|
||||
|
||||
|
||||
def export_site():
|
||||
from bijoe.visualization.models import Visualization
|
||||
|
||||
return {'visualizations': [v.export_json() for v in Visualization.objects.all()]}
|
||||
|
||||
|
||||
def import_site(data, if_empty=False, clean=False):
|
||||
from bijoe.visualization.models import Visualization
|
||||
|
||||
if if_empty and Visualization.objects.exists():
|
||||
return
|
||||
|
||||
if clean:
|
||||
Visualization.objects.all().delete()
|
||||
|
||||
results = {'created': 0, 'updated': 0}
|
||||
with transaction.atomic():
|
||||
for data in data.get('visualizations', []):
|
||||
created = Visualization.import_json(data)
|
||||
if created:
|
||||
results['created'] += 1
|
||||
else:
|
||||
results['updated'] += 1
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -235,3 +235,7 @@ class CubeForm(forms.Form):
|
|||
raise ValidationError({'loop': _('You cannot use the same dimension for looping and'
|
||||
' grouping')})
|
||||
return cleaned_data
|
||||
|
||||
|
||||
class VisualizationsImportForm(forms.Form):
|
||||
visualizations_json = forms.FileField(label=_('Visualizations Export File'))
|
||||
|
|
|
|||
|
|
@ -54,6 +54,23 @@ class Visualization(models.Model):
|
|||
def natural_key(self):
|
||||
return (self.slug,)
|
||||
|
||||
def export_json(self):
|
||||
visualization = {
|
||||
'slug': self.slug,
|
||||
'name': self.name,
|
||||
'parameters': self.parameters
|
||||
}
|
||||
return visualization
|
||||
|
||||
@classmethod
|
||||
def import_json(cls, data):
|
||||
defaults = {
|
||||
'name': data['name'],
|
||||
'parameters': data['parameters']
|
||||
}
|
||||
_, created = cls.objects.update_or_create(slug=data['slug'], defaults=defaults)
|
||||
return created
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
if not self.slug:
|
||||
slug = base_slug = slugify(self.name)[:40].strip('-')
|
||||
|
|
|
|||
|
|
@ -23,6 +23,10 @@ urlpatterns = [
|
|||
views.visualizations, name='visualizations'),
|
||||
url(r'^json/$',
|
||||
views.visualizations_json, name='visualizations-json'),
|
||||
url(r'^import/$',
|
||||
views.visualizations_import, name='visualizations-import'),
|
||||
url(r'^export$',
|
||||
views.visualizations_export, name='visualizations-export'),
|
||||
url(r'^warehouse/(?P<warehouse>[^/]*)/$', views.warehouse, name='warehouse'),
|
||||
url(r'^warehouse/(?P<warehouse>[^/]*)/(?P<cube>[^/]*)/$', views.cube, name='cube'),
|
||||
url(r'^warehouse/(?P<warehouse>[^/]*)/(?P<cube>[^/]*)/iframe/$', views.cube_iframe,
|
||||
|
|
@ -36,4 +40,5 @@ urlpatterns = [
|
|||
url(r'(?P<pk>\d+)/ods/$', views.visualization_ods, name='visualization-ods'),
|
||||
url(r'(?P<pk>\d+)/rename/$', views.rename_visualization, name='rename-visualization'),
|
||||
url(r'(?P<pk>\d+)/delete/$', views.delete_visualization, name='delete-visualization'),
|
||||
url(r'(?P<pk>\d+)/export$', views.export_visualization, name='export-visualization'),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -21,9 +21,11 @@ import hashlib
|
|||
import json
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib import messages
|
||||
from django.utils.encoding import force_text
|
||||
from django.utils.text import slugify
|
||||
from django.views.generic.edit import CreateView, DeleteView, UpdateView
|
||||
from django.utils.translation import ungettext, ugettext as _
|
||||
from django.views.generic.edit import CreateView, DeleteView, UpdateView, FormView
|
||||
from django.views.generic.list import MultipleObjectMixin
|
||||
from django.views.generic import DetailView, ListView, View, TemplateView
|
||||
from django.shortcuts import redirect
|
||||
|
|
@ -35,7 +37,7 @@ from django.views.decorators.clickjacking import xframe_options_exempt
|
|||
from rest_framework import generics
|
||||
from rest_framework.response import Response
|
||||
|
||||
from ..utils import get_warehouses
|
||||
from bijoe.utils import get_warehouses, import_site, export_site
|
||||
from ..engine import Engine
|
||||
from . import models, forms, signature
|
||||
from .utils import Visualization
|
||||
|
|
@ -54,6 +56,7 @@ class WarehouseView(views.AuthorizationMixin, TemplateView):
|
|||
raise Http404
|
||||
ctx['warehouse'] = Engine(warehouse)
|
||||
ctx['cubes'] = sorted(ctx['warehouse'].cubes, key=lambda cube: cube.label.strip().lower())
|
||||
ctx['visualizations_exist'] = models.Visualization.objects.exists()
|
||||
return ctx
|
||||
|
||||
|
||||
|
|
@ -343,6 +346,60 @@ class VisualizationJSONView(generics.GenericAPIView):
|
|||
})
|
||||
|
||||
|
||||
class ExportVisualizationView(views.AuthorizationMixin, DetailView):
|
||||
model = models.Visualization
|
||||
|
||||
def get(self, request, *args, **kwargs):
|
||||
response = HttpResponse(content_type='application/json')
|
||||
json.dump({'visualizations': [self.get_object().export_json()]}, response, indent=2)
|
||||
return response
|
||||
|
||||
|
||||
class VisualizationsImportView(views.AuthorizationMixin, FormView):
|
||||
form_class = forms.VisualizationsImportForm
|
||||
template_name = 'bijoe/visualizations_import.html'
|
||||
success_url = reverse_lazy('homepage')
|
||||
|
||||
def form_valid(self, form):
|
||||
try:
|
||||
visualizations_json = json.loads(
|
||||
force_text(self.request.FILES['visualizations_json'].read()))
|
||||
except ValueError:
|
||||
form.add_error('visualizations_json', _('File is not in the expected JSON format.'))
|
||||
return self.form_invalid(form)
|
||||
|
||||
results = import_site(visualizations_json)
|
||||
|
||||
if results.get('created') == 0 and results.get('updated') == 0:
|
||||
messages.info(self.request, _('No visualizations were found.'))
|
||||
else:
|
||||
if results.get('created') == 0:
|
||||
message1 = _('No visualization created.')
|
||||
else:
|
||||
message1 = ungettext(
|
||||
'A visualization has been created.',
|
||||
'%(count)d visualizations have been created.',
|
||||
results['created']) % {'count': results['created']}
|
||||
if results.get('updated') == 0:
|
||||
message2 = _('No visualization updated.')
|
||||
else:
|
||||
message2 = ungettext(
|
||||
'A visualization has been updated.',
|
||||
'%(count)d visualizations have been updated.',
|
||||
results['updated']) % {'count': results['updated']}
|
||||
messages.info(self.request, u'%s %s' % (message1, message2))
|
||||
|
||||
return super(VisualizationsImportView, self).form_valid(form)
|
||||
|
||||
|
||||
class VisualizationsExportView(views.AuthorizationMixin, View):
|
||||
|
||||
def get(self, request, *args, **kwargs):
|
||||
response = HttpResponse(content_type='application/json')
|
||||
json.dump(export_site(), response, indent=2)
|
||||
return response
|
||||
|
||||
|
||||
warehouse = WarehouseView.as_view()
|
||||
cube = CubeView.as_view()
|
||||
cube_iframe = xframe_options_exempt(CubeIframeView.as_view())
|
||||
|
|
@ -351,11 +408,14 @@ visualizations_json = VisualizationsJSONView.as_view()
|
|||
create_visualization = CreateVisualizationView.as_view()
|
||||
delete_visualization = DeleteVisualizationView.as_view()
|
||||
rename_visualization = RenameVisualization.as_view()
|
||||
export_visualization = ExportVisualizationView.as_view()
|
||||
visualization = VisualizationView.as_view()
|
||||
visualization_iframe = xframe_options_exempt(VisualizationIframeView.as_view())
|
||||
visualization_geojson = VisualizationGeoJSONView.as_view()
|
||||
visualization_ods = VisualizationODSView.as_view()
|
||||
visualization_json = VisualizationJSONView.as_view()
|
||||
visualizations_import = VisualizationsImportView.as_view()
|
||||
visualizations_export = VisualizationsExportView.as_view()
|
||||
|
||||
cube_iframe.mellon_no_passive = True
|
||||
visualization_iframe.mellon_no_passive = True
|
||||
|
|
|
|||
|
|
@ -0,0 +1,92 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
from django.core.management import call_command
|
||||
from django.utils.encoding import force_bytes
|
||||
from django.utils.six import StringIO
|
||||
|
||||
from bijoe.visualization.models import Visualization
|
||||
from bijoe.utils import import_site
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
def get_output_of_command(command, *args, **kwargs):
|
||||
old_stdout = sys.stdout
|
||||
output = sys.stdout = StringIO()
|
||||
call_command(command, *args, **kwargs)
|
||||
sys.stdout = old_stdout
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def test_import_export(schema1, app):
|
||||
parameters = {
|
||||
'cube': 'facts1',
|
||||
'warehouse': 'schema1',
|
||||
'measure': 'duration',
|
||||
'representation': 'table',
|
||||
'loop': '',
|
||||
'filters': {},
|
||||
'drilldown_x': 'date__yearmonth'
|
||||
}
|
||||
|
||||
def create_visu(i=0):
|
||||
Visualization.objects.create(name='test' + str(i), parameters=parameters)
|
||||
|
||||
for i in range(3):
|
||||
create_visu(i)
|
||||
output = get_output_of_command('export_site')
|
||||
assert len(json.loads(output)['visualizations']) == 3
|
||||
|
||||
import_site(data={}, clean=True)
|
||||
empty_output = get_output_of_command('export_site')
|
||||
assert len(json.loads(empty_output)['visualizations']) == 0
|
||||
|
||||
create_visu()
|
||||
old_stdin = sys.stdin
|
||||
sys.stdin = StringIO(json.dumps({}))
|
||||
assert Visualization.objects.count() == 1
|
||||
try:
|
||||
call_command('import_site', '-', clean=True)
|
||||
finally:
|
||||
sys.stdin = old_stdin
|
||||
assert Visualization.objects.count() == 0
|
||||
|
||||
with tempfile.NamedTemporaryFile() as f:
|
||||
f.write(force_bytes(output))
|
||||
f.flush()
|
||||
call_command('import_site', f.name)
|
||||
assert Visualization.objects.count() == 3
|
||||
for i in range(3):
|
||||
visu = Visualization.objects.get(name='test' + str(i))
|
||||
assert visu.parameters == parameters
|
||||
|
||||
visu = Visualization.objects.get(name='test0')
|
||||
slug = visu.slug
|
||||
visu_json = visu.export_json()
|
||||
visu_json['name'] = 'new_name'
|
||||
visu_json['parameters']['measure'] = 'test'
|
||||
result = import_site(data={'visualizations': [visu_json]})
|
||||
assert result['created'] == 0 and result['updated'] == 1
|
||||
visu = Visualization.objects.get(slug=slug)
|
||||
assert visu.name == 'new_name'
|
||||
new_params = visu.parameters
|
||||
assert new_params.pop('measure') == 'test'
|
||||
assert new_params == {k: v for k, v in parameters.items() if k != 'measure'}
|
||||
|
||||
import_site(data={}, if_empty=True)
|
||||
assert Visualization.objects.count() == 3
|
||||
|
||||
import_site(data={}, clean=True)
|
||||
tempdir = tempfile.mkdtemp('bijoe-test')
|
||||
empty_output = get_output_of_command('export_site', output=os.path.join(tempdir, 't.json'))
|
||||
assert os.path.exists(os.path.join(tempdir, 't.json'))
|
||||
shutil.rmtree(tempdir)
|
||||
|
|
@ -14,7 +14,11 @@
|
|||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import copy
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from webtest import Upload
|
||||
|
||||
from django.core.urlresolvers import reverse
|
||||
|
||||
|
|
@ -115,3 +119,70 @@ def test_visualization_creation_view(schema1, app, admin):
|
|||
|
||||
visu = Visualization.objects.get(name='test')
|
||||
assert visu.parameters['warehouse_slug'] == 'schema1_slug'
|
||||
|
||||
|
||||
def test_import_visualization(schema1, app, admin, visualization):
|
||||
login(app, admin)
|
||||
resp = app.get('/visualization/%s/' % visualization.id)
|
||||
resp = resp.click('Export as JSON')
|
||||
assert resp.headers['content-type'] == 'application/json'
|
||||
visualization_export = resp.text
|
||||
|
||||
# invalid json
|
||||
resp = app.get('/', status=200)
|
||||
resp = resp.click('Import')
|
||||
resp.form['visualizations_json'] = Upload('export.json', b'garbage', 'application/json')
|
||||
resp = resp.form.submit()
|
||||
assert 'File is not in the expected JSON format.' in resp.text
|
||||
|
||||
# empty json
|
||||
resp = app.get('/', status=200)
|
||||
resp = resp.click('Import')
|
||||
resp.form['visualizations_json'] = Upload('export.json', b'{}', 'application/json')
|
||||
resp = resp.form.submit().follow()
|
||||
assert 'No visualizations were found.' in resp.text
|
||||
|
||||
# existing visualization
|
||||
resp = app.get('/', status=200)
|
||||
resp = resp.click('Import')
|
||||
resp.form['visualizations_json'] = Upload('export.json', visualization_export.encode('utf-8'), 'application/json')
|
||||
resp = resp.form.submit().follow()
|
||||
assert 'No visualization created. A visualization has been updated.' in resp.text
|
||||
assert Visualization.objects.count() == 1
|
||||
|
||||
# new visualization
|
||||
Visualization.objects.all().delete()
|
||||
resp = app.get('/')
|
||||
resp = resp.click('Import')
|
||||
resp.form['visualizations_json'] = Upload('export.json', visualization_export.encode('utf-8'), 'application/json')
|
||||
resp = resp.form.submit().follow()
|
||||
assert 'A visualization has been created. No visualization updated.' in resp.text
|
||||
assert Visualization.objects.count() == 1
|
||||
|
||||
# multiple visualizations
|
||||
visualizations = json.loads(visualization_export)
|
||||
visualizations['visualizations'].append(copy.copy(visualizations['visualizations'][0]))
|
||||
visualizations['visualizations'].append(copy.copy(visualizations['visualizations'][0]))
|
||||
visualizations['visualizations'][1]['name'] = 'test 2'
|
||||
visualizations['visualizations'][1]['slug'] = 'test-2'
|
||||
visualizations['visualizations'][2]['name'] = 'test 3'
|
||||
visualizations['visualizations'][2]['slug'] = 'test-3'
|
||||
|
||||
resp = app.get('/', status=200)
|
||||
resp = resp.click('Import')
|
||||
resp.form['visualizations_json'] = Upload('export.json', json.dumps(visualizations).encode('utf-8'), 'application/json')
|
||||
resp = resp.form.submit().follow()
|
||||
assert '2 visualizations have been created. A visualization has been updated.' in resp.text
|
||||
assert Visualization.objects.count() == 3
|
||||
|
||||
# global export/import
|
||||
resp = app.get('/').click('Export')
|
||||
visualizations_export = resp.text
|
||||
Visualization.objects.all().delete()
|
||||
|
||||
resp = app.get('/')
|
||||
resp = resp.click('Import')
|
||||
resp.form['visualizations_json'] = Upload('export.json', visualizations_export.encode('utf-8'), 'application/json')
|
||||
resp = resp.form.submit().follow()
|
||||
assert '3 visualizations have been created. No visualization updated.' in resp.text
|
||||
assert Visualization.objects.count() == 3
|
||||
|
|
|
|||
Loading…
Reference in New Issue