base: add import/export UI (#15269)

Site export as well as connector export.

passerelle/base/forms.py

@@ -1,4 +1,5 @@
 from django import forms
+from django.utils.translation import ugettext_lazy as _
 
 from .models import ApiUser, AccessRight, AvailabilityParameters
 
@@ -27,3 +28,8 @@ class AvailabilityParametersForm(forms.ModelForm):
         widgets = {
             'notification_delays': forms.TextInput,
         }
+
+
+class ImportSiteForm(forms.Form):
+    site_json = forms.FileField(label=_('Site Export File'))
+    import_users = forms.BooleanField(label=_('Import users and access rights'), required=False)

passerelle/base/urls.py

@@ -2,7 +2,8 @@ from django.conf.urls import url
 
 from .views import ApiUserCreateView, ApiUserUpdateView, ApiUserDeleteView, \
         ApiUserListView, AccessRightDeleteView, AccessRightCreateView, \
-        LoggingParametersUpdateView, ManageAvailabilityView
+        LoggingParametersUpdateView, ManageAvailabilityView, ImportSiteView, \
+        ExportSiteView
 
 access_urlpatterns = [
     url(r'^$', ApiUserListView.as_view(), name='apiuser-list'),
@@ -20,3 +21,8 @@ access_urlpatterns = [
         ManageAvailabilityView.as_view(), name='manage-availability')
 
 ]
+
+import_export_urlpatterns = [
+    url(r'^import$', ImportSiteView.as_view(), name='import-site'),
+    url(r'^export$', ExportSiteView.as_view(), name='export-site'),
+]

passerelle/base/views.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import datetime
+import json
 
 from dateutil import parser as date_parser
 
@@ -25,15 +26,17 @@ from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
 from django.db.models import Q
 from django.forms import models as model_forms
 from django.views.generic import (
-    DetailView, ListView, CreateView, UpdateView, DeleteView, FormView)
-from django.http import Http404
+    View, DetailView, ListView, CreateView, UpdateView, DeleteView, FormView)
+from django.http import Http404, HttpResponse
 from django.utils.timezone import make_aware
 from django.utils.translation import ugettext_lazy as _
 
+from passerelle.compat import json_loads
+
 from .models import ApiUser, AccessRight, LoggingParameters, ResourceStatus, Job
-from .forms import ApiUserForm, AccessRightForm, AvailabilityParametersForm
+from .forms import ApiUserForm, AccessRightForm, AvailabilityParametersForm, ImportSiteForm
 from ..views import GenericConnectorMixin
-from ..utils import get_trusted_services
+from ..utils import get_trusted_services, import_site, export_site
 
 
 class ResourceView(DetailView):
@@ -277,3 +280,30 @@ class GenericJobView(GenericConnectorMixin, DetailView):
         except Job.DoesNotExist:
             raise Http404()
         return context
+
+
+class ImportSiteView(FormView):
+    template_name = 'passerelle/manage/import_site.html'
+    form_class = ImportSiteForm
+
+    def get_success_url(self):
+        return reverse('manage-home')
+
+    def form_valid(self, form):
+        try:
+            site_json = json_loads(self.request.FILES['site_json'].read())
+        except ValueError:
+            form.add_error('site_json', _('File is not in the expected JSON format.'))
+            return self.form_invalid(form)
+
+        results = import_site(site_json, overwrite=True,
+                              import_users=form.cleaned_data['import_users'])
+        return super(ImportSiteView, self).form_valid(form)
+
+
+class ExportSiteView(View):
+
+    def get(self, request, *args, **kwargs):
+        response = HttpResponse(content_type='application/json')
+        json.dump(export_site(), response, indent=2)
+        return response

passerelle/templates/passerelle/manage.html

@@ -4,9 +4,14 @@
 {% block appbar %}
   <h2>{% trans 'Web Services' %}</h2>
   <span class="actions">
+  <a class="extra-actions-menu-opener"></a>
   <a href="{% url 'apiuser-list' %}">{% trans 'Access Management' %}</a>
   <a href="{% url 'add-connector' %}">{% trans 'Add Connector' %}</a>
   </span>
+  <ul class="extra-actions-menu">
+    <li><a href="{% url 'import-site' %}" rel="popup">{% trans 'Import' %}</a></li>
+    <li><a download href="{% url 'export-site' %}">{% trans 'Export' %}</a></li>
+  </ul>
 {% endblock %}
 
 {% block content %}

passerelle/templates/passerelle/manage/import_site.html

@@ -0,0 +1,17 @@
+{% extends "passerelle/manage.html" %}
+{% load i18n %}
+
+{% block appbar %}
+<h2>{% trans "Site Import" %}</h2>
+{% endblock %}
+
+{% block content %}
+<form method="post" enctype="multipart/form-data">
+  {% csrf_token %}
+  {{ form.as_p }}
+  <div class="buttons">
+    <button class="submit-button">{% trans "Import" %}</button>
+    <a class="cancel" href="{% url 'manage-home' %}">{% trans 'Cancel' %}</a>
+  </div>
+</form>
+{% endblock %}

passerelle/templates/passerelle/manage/service_view.html

@@ -15,6 +15,7 @@
     {% endwith %}
 </h2>
 <span class="actions">
+<a class="extra-actions-menu-opener"></a>
 {% if object|can_edit:request.user and has_check_status %}
 <a rel="popup" href="{% url 'manage-availability' resource_type=object|resource_type resource_pk=object.id %}">{% trans 'availability check parameters' %}</a>
 {% endif %}
@@ -28,6 +29,9 @@
 <a rel="popup" href="{{ object.get_delete_url }}">{% trans 'delete' %}</a>
 {% endif %}
 </span>
+<ul class="extra-actions-menu">
+  <li><a download href="{% url 'export-connector' connector=object.get_connector_slug slug=object.slug %}">{% trans 'Export' %}</a></li>
+</ul>
 {% endblock %}
 
 {% block content %}

passerelle/urls.py

@@ -9,11 +9,11 @@ from django.views.static import serve as static_serve
 from .views import (HomePageView, ManageView, ManageAddView,
         GenericCreateConnectorView, GenericDeleteConnectorView,
         GenericEditConnectorView, GenericEndpointView, GenericConnectorView,
-        GenericViewLogsConnectorView, GenericLogView,
+        GenericViewLogsConnectorView, GenericLogView, GenericExportConnectorView,
         login, logout, menu_json)
 from .base.views import GenericViewJobsConnectorView, GenericJobView
 from .urls_utils import decorated_includes, required, app_enabled, manager_required
-from .base.urls import access_urlpatterns
+from .base.urls import access_urlpatterns, import_export_urlpatterns
 from .plugins import register_apps_urls
 
 from passerelle.apps.pastell import urls as pastell_urls
@@ -34,6 +34,8 @@ urlpatterns = [
 
     url(r'^manage/access/',
         decorated_includes(manager_required, include(access_urlpatterns))),
+    url(r'^manage/',
+        decorated_includes(manager_required, include(import_export_urlpatterns))),
 ]
 
 urlpatterns += required(
@@ -73,6 +75,8 @@ urlpatterns += [
                 GenericViewJobsConnectorView.as_view(), name='view-jobs-connector'),
             url(r'^(?P<slug>[\w,-]+)/jobs/(?P<job_pk>\d+)/$',
                 GenericJobView.as_view(), name='view-job'),
+            url(r'^(?P<slug>[\w,-]+)/export$',
+                GenericExportConnectorView.as_view(), name='export-connector'),
         ])))
 ]
 

passerelle/views.py

@@ -482,3 +482,11 @@ class GenericEndpointView(GenericConnectorMixin, SingleObjectMixin, View):
 
     def delete(self, request, *args, **kwargs):
         return self.get(request, *args, **kwargs)
+
+
+class GenericExportConnectorView(GenericConnectorMixin, DetailView):
+
+    def get(self, request, *args, **kwargs):
+        response = HttpResponse(content_type='application/json')
+        json.dump({'resources': [self.get_object().export_json()]}, response, indent=2)
+        return response

tests/test_manager.py

@@ -1,6 +1,8 @@
 import datetime
 import re
 
+from webtest import Upload
+
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
 from django.core.files import File
@@ -336,3 +338,82 @@ def test_jobs(app, admin_user):
     base_url = re.findall(r'data-job-base-url="(.*)"', resp.text)[0]
     resp = app.get(base_url + job_pk + '/')
     resp = app.get(base_url + '12345' + '/', status=404)
+
+
+def test_manager_import_export(app, admin_user):
+    data = StringIO('1;Foo\n2;Bar\n3;Baz')
+    csv = CsvDataSource.objects.create(csv_file=File(data, 't.csv'),
+           columns_keynames='id, text', slug='test', title='a title', description='a description')
+    csv2 = CsvDataSource.objects.create(csv_file=File(data, 't.csv'),
+           columns_keynames='id, text', slug='test2', title='a title', description='a description')
+    api = ApiUser.objects.create(username='public',
+                    fullname='public',
+                    description='access for all',
+                    keytype='', key='')
+    obj_type = ContentType.objects.get_for_model(csv)
+    AccessRight.objects.create(codename='can_access',
+                    apiuser=api,
+                    resource_type=obj_type,
+                    resource_pk=csv.pk,
+    )
+
+    # export site
+    app = login(app)
+    resp = app.get('/manage/')
+    resp = resp.click('Export')
+    assert resp.headers['content-type'] == 'application/json'
+    site_export = resp.text
+
+    # invalid json
+    resp = app.get('/manage/', status=200)
+    resp = resp.click('Import')
+    resp.form['site_json'] = Upload('export.json', b'garbage', 'application/json')
+    resp = resp.form.submit()
+    assert 'File is not in the expected JSON format.' in resp.text
+
+    # empty json
+    resp = app.get('/manage/', status=200)
+    resp = resp.click('Import')
+    resp.form['site_json'] = Upload('export.json', b'{}', 'application/json')
+    resp = resp.form.submit().follow()
+    assert CsvDataSource.objects.count() == 2
+
+    # import site
+    CsvDataSource.objects.all().delete()
+    resp = app.get('/manage/', status=200)
+    resp = resp.click('Import')
+    resp.form['site_json'] = Upload('export.json', site_export.encode('utf-8'), 'application/json')
+    resp = resp.form.submit().follow()
+    assert CsvDataSource.objects.count() == 2
+
+    # export connector
+    resp = app.get('/%s/%s/' % (csv.get_connector_slug(), csv.slug), status=200)
+    resp = resp.click('Export')
+    assert resp.headers['content-type'] == 'application/json'
+    connector_export = resp.text
+
+    # import connector
+    csv.delete()
+    resp = app.get('/manage/', status=200)
+    resp = resp.click('Import')
+    resp.form['site_json'] = Upload('export.json', connector_export.encode('utf-8'),
+                                     'application/json')
+    resp = resp.form.submit().follow()
+    assert CsvDataSource.objects.count() == 2
+    assert CsvDataSource.objects.filter(slug='test').exists()
+
+    # import users
+    ApiUser.objects.all().delete()
+    AccessRight.objects.all().delete()
+    resp = app.get('/manage/', status=200)
+    resp = resp.click('Import')
+    resp.form['site_json'] = Upload('export.json', site_export.encode('utf-8'), 'application/json')
+    resp = resp.form.submit().follow()
+    assert not ApiUser.objects.exists()
+    assert not AccessRight.objects.exists()
+    resp = resp.click('Import')
+    resp.form['import_users'] = True
+    resp.form['site_json'] = Upload('export.json', site_export.encode('utf-8'), 'application/json')
+    resp = resp.form.submit().follow()
+    assert ApiUser.objects.filter(username='public').exists()
+    assert AccessRight.objects.filter(codename='can_access').exists()