From 62220edc86bffe68763f8a53f01c0370640c19ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= Date: Sat, 8 Jan 2022 16:07:08 +0100 Subject: [PATCH] add section to create/deploy applications (#60699) --- README | 38 +++ hobo/applications/__init__.py | 0 hobo/applications/forms.py | 22 ++ hobo/applications/migrations/0001_initial.py | 95 ++++++ hobo/applications/migrations/__init__.py | 0 hobo/applications/models.py | 104 ++++++ .../hobo/applications/add-element.html | 45 +++ .../templates/hobo/applications/create.html | 14 + .../hobo/applications/edit-metadata.html | 14 + .../applications/element_confirm_delete.html | 20 ++ .../templates/hobo/applications/home.html | 42 +++ .../templates/hobo/applications/install.html | 17 + .../templates/hobo/applications/manifest.html | 62 ++++ hobo/applications/urls.py | 40 +++ hobo/applications/utils.py | 66 ++++ hobo/applications/views.py | 300 ++++++++++++++++++ hobo/environment/utils.py | 4 +- hobo/settings.py | 1 + hobo/static/css/applications.svg | 1 + hobo/static/css/build-application.svg | 1 + hobo/static/css/style.scss | 46 +++ hobo/urls.py | 2 + tests/test_application.py | 229 +++++++++++++ 23 files changed, 1162 insertions(+), 1 deletion(-) create mode 100644 hobo/applications/__init__.py create mode 100644 hobo/applications/forms.py create mode 100644 hobo/applications/migrations/0001_initial.py create mode 100644 hobo/applications/migrations/__init__.py create mode 100644 hobo/applications/models.py create mode 100644 hobo/applications/templates/hobo/applications/add-element.html create mode 100644 hobo/applications/templates/hobo/applications/create.html create mode 100644 hobo/applications/templates/hobo/applications/edit-metadata.html create mode 100644 hobo/applications/templates/hobo/applications/element_confirm_delete.html create mode 100644 hobo/applications/templates/hobo/applications/home.html create mode 100644 hobo/applications/templates/hobo/applications/install.html create mode 100644 hobo/applications/templates/hobo/applications/manifest.html create mode 100644 hobo/applications/urls.py create mode 100644 hobo/applications/utils.py create mode 100644 hobo/applications/views.py create mode 100644 hobo/static/css/applications.svg create mode 100644 hobo/static/css/build-application.svg create mode 100644 tests/test_application.py diff --git a/README b/README index 86432c2..2410fcf 100644 --- a/README +++ b/README @@ -186,3 +186,41 @@ isort is used to format the imports, using those parameters: There is .pre-commit-config.yaml to use pre-commit to automatically run black and isort before commits. (execute `pre-commit install` to install the git hook.) + + +License +------- + +This program is free software: you can redistribute it and/or modify it under +the terms of the GNU Affero General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) any +later version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU Affero General Public License for more +details. + +You should have received a copy of the GNU Affero General Public License along +with this program. If not, see . + + +Combo embeds some other pieces of code or art, with their own authors and +copyright notices: + +Application images (hobo/static/css/*.svg) from the unDraw project: + # https://undraw.co/ + # + # All images, assets and vectors published on unDraw can be used for free. You + # can use them for noncommercial and commercial purposes. You do not need to ask + # permission from or provide credit to the creator or unDraw. + # + # More precisely, unDraw grants you an nonexclusive, worldwide copyright + # license to download, copy, modify, distribute, perform, and use the assets + # provided from unDraw for free, including for commercial purposes, without + # permission from or attributing the creator or unDraw. This license does not + # include the right to compile assets, vectors or images from unDraw to + # replicate a similar or competing service, in any form or distribute the assets + # in packs. This extends to automated and non-automated ways to link, embed, + # scrape, search or download the assets included on the website without our + # consent. diff --git a/hobo/applications/__init__.py b/hobo/applications/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hobo/applications/forms.py b/hobo/applications/forms.py new file mode 100644 index 0000000..1401aaa --- /dev/null +++ b/hobo/applications/forms.py @@ -0,0 +1,22 @@ +# hobo - portal to configure and deploy applications +# Copyright (C) 2015-2022 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django import forms +from django.utils.translation import ugettext_lazy as _ + + +class InstallForm(forms.Form): + bundle = forms.FileField(label=_('Application')) diff --git a/hobo/applications/migrations/0001_initial.py b/hobo/applications/migrations/0001_initial.py new file mode 100644 index 0000000..b279a9a --- /dev/null +++ b/hobo/applications/migrations/0001_initial.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2022-01-09 13:16 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +import django.db.models.deletion +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name='Application', + fields=[ + ( + 'id', + models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ('name', models.CharField(max_length=100, verbose_name='Name')), + ('slug', models.SlugField(max_length=100)), + ('description', models.TextField(blank=True, verbose_name='Description')), + ('editable', models.BooleanField(default=True)), + ('creation_timestamp', models.DateTimeField(default=django.utils.timezone.now)), + ('last_update_timestamp', models.DateTimeField(auto_now=True)), + ], + ), + migrations.CreateModel( + name='Element', + fields=[ + ( + 'id', + models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ('type', models.CharField(max_length=25, verbose_name='Type')), + ('slug', models.SlugField(max_length=500, verbose_name='Slug')), + ('name', models.CharField(max_length=500, verbose_name='Name')), + ('cache', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict)), + ], + ), + migrations.CreateModel( + name='Relation', + fields=[ + ( + 'id', + models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ('auto_dependency', models.BooleanField(default=False)), + ( + 'application', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to='applications.Application' + ), + ), + ( + 'element', + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='applications.Element'), + ), + ], + ), + migrations.CreateModel( + name='Version', + fields=[ + ( + 'id', + models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ('bundle', models.FileField(blank=True, null=True, upload_to='applications')), + ('creation_timestamp', models.DateTimeField(default=django.utils.timezone.now)), + ('last_update_timestamp', models.DateTimeField(auto_now=True)), + ( + 'deployment_status', + django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + ( + 'application', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to='applications.Application' + ), + ), + ], + ), + migrations.AddField( + model_name='application', + name='elements', + field=models.ManyToManyField( + blank=True, through='applications.Relation', to='applications.Element' + ), + ), + ] diff --git a/hobo/applications/migrations/__init__.py b/hobo/applications/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hobo/applications/models.py b/hobo/applications/models.py new file mode 100644 index 0000000..f4bc094 --- /dev/null +++ b/hobo/applications/models.py @@ -0,0 +1,104 @@ +# hobo - portal to configure and deploy applications +# Copyright (C) 2015-2022 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import urllib.parse + +from django.conf import settings +from django.contrib.postgres.fields import JSONField +from django.db import models +from django.utils.text import slugify +from django.utils.timezone import now +from django.utils.translation import ugettext_lazy as _ + +from hobo.environment.utils import get_installed_services +from hobo.signature import sign_url + +from .utils import Requests + +requests = Requests() + + +class Application(models.Model): + SUPPORTED_MODULES = ('wcs',) + + name = models.CharField(max_length=100, verbose_name=_('Name')) + slug = models.SlugField(max_length=100) + description = models.TextField(verbose_name=_('Description'), blank=True) + editable = models.BooleanField(default=True) + elements = models.ManyToManyField('Element', blank=True, through='Relation') + creation_timestamp = models.DateTimeField(default=now) + last_update_timestamp = models.DateTimeField(auto_now=True) + + def __repr__(self): + return '' % self.slug + + def save(self, *args, **kwargs): + if not self.slug: + base_slug = slugify(self.name)[:95] + slug = base_slug + i = 1 + while Application.objects.filter(slug=slug).exists(): + slug = '%s-%s' % (base_slug, i) + i += 1 + self.slug = slug + super().save(*args, **kwargs) + + +class Element(models.Model): + type = models.CharField(max_length=25, verbose_name=_('Type')) + slug = models.SlugField(max_length=500, verbose_name=_('Slug')) + name = models.CharField(max_length=500, verbose_name=_('Name')) + cache = JSONField(blank=True, default=dict) + + def __repr__(self): + return '' % (self.type, self.slug) + + +class Relation(models.Model): + application = models.ForeignKey(Application, on_delete=models.CASCADE) + element = models.ForeignKey(Element, on_delete=models.CASCADE) + auto_dependency = models.BooleanField(default=False) + + def __repr__(self): + return '' % (self.application.slug, self.element.type, self.element.slug) + + +class Version(models.Model): + application = models.ForeignKey(Application, on_delete=models.CASCADE) + bundle = models.FileField(upload_to='applications', blank=True, null=True) + creation_timestamp = models.DateTimeField(default=now) + last_update_timestamp = models.DateTimeField(auto_now=True) + deployment_status = JSONField(blank=True, default=dict) + + def __repr__(self): + return '' % self.application.slug + + def deploy(self): + bundle_content = self.bundle.read() + for service_id, services in getattr(settings, 'KNOWN_SERVICES', {}).items(): + if service_id not in Application.SUPPORTED_MODULES: + continue + service_objects = {x.get_base_url_path(): x for x in get_installed_services(types=[service_id])} + for service in services.values(): + if service_objects[service['url']].secondary: + continue + url = urllib.parse.urljoin(service['url'], 'api/export-import/bundle-import/') + response = requests.put(sign_url(url, service['secret']), data=bundle_content) + if not response.ok: + # TODO: report failures + continue + # TODO: look at response content for afterjob URLs to display a progress bar + pass diff --git a/hobo/applications/templates/hobo/applications/add-element.html b/hobo/applications/templates/hobo/applications/add-element.html new file mode 100644 index 0000000..616334e --- /dev/null +++ b/hobo/applications/templates/hobo/applications/add-element.html @@ -0,0 +1,45 @@ +{% extends "hobo/applications/home.html" %} +{% load i18n %} + +{% block appbar %} +

{{ type.text }}

+{% endblock %} + +{% block content %} +
+ {% csrf_token %} +
+ {% for element in elements %} + + {% endfor %} +
+
+ +
+
+ +{% trans "Cancel" %} +
+ + +
+{% endblock %} diff --git a/hobo/applications/templates/hobo/applications/create.html b/hobo/applications/templates/hobo/applications/create.html new file mode 100644 index 0000000..65ddc60 --- /dev/null +++ b/hobo/applications/templates/hobo/applications/create.html @@ -0,0 +1,14 @@ +{% extends "hobo/base.html" %} +{% load i18n %} + +{% block appbar %}

{% trans "Create Application" %}

{% endblock %} + +{% block content %} +
+{% csrf_token %} +{{ form.as_p }} +
+ +{% trans "Cancel" %} +
+{% endblock %} diff --git a/hobo/applications/templates/hobo/applications/edit-metadata.html b/hobo/applications/templates/hobo/applications/edit-metadata.html new file mode 100644 index 0000000..ee3f523 --- /dev/null +++ b/hobo/applications/templates/hobo/applications/edit-metadata.html @@ -0,0 +1,14 @@ +{% extends "hobo/base.html" %} +{% load i18n %} + +{% block appbar %}

{% trans "Metadata" %}

{% endblock %} + +{% block content %} + +{% csrf_token %} +{{ form.as_p }} +
+ +{% trans "Cancel" %} +
+{% endblock %} diff --git a/hobo/applications/templates/hobo/applications/element_confirm_delete.html b/hobo/applications/templates/hobo/applications/element_confirm_delete.html new file mode 100644 index 0000000..f12f02f --- /dev/null +++ b/hobo/applications/templates/hobo/applications/element_confirm_delete.html @@ -0,0 +1,20 @@ +{% extends "hobo/base.html" %} +{% load i18n %} + +{% block appbar %} +

{% blocktrans with title=object.element.name %}Removal of "{{ title }}"{% endblocktrans %}

+{% endblock %} + +{% block content %} + + {% csrf_token %} +

+ {% trans 'Are you sure you want to remove this element ?' %} +

+
+ + {% trans 'Cancel' %} +
+
+{% endblock %} + diff --git a/hobo/applications/templates/hobo/applications/home.html b/hobo/applications/templates/hobo/applications/home.html new file mode 100644 index 0000000..3f571db --- /dev/null +++ b/hobo/applications/templates/hobo/applications/home.html @@ -0,0 +1,42 @@ +{% extends "hobo/base.html" %} +{% load i18n %} + +{% block breadcrumb %} +{{ block.super }} +{% trans 'Applications' %} +{% endblock %} + +{% block appbar %} +

{% trans 'Applications' %}

+ + {% trans 'Install' %} + {% trans 'Create' %} + +{% endblock %} + +{% block content %} + +{% if object_list %} +
+{% for application in object_list %} +
+

{{ application.name }}

+

{{ application.description|default:"" }}

+ {% if application.editable %} +
+ {% trans "Edit" %} +
+ {% endif %} +
+{% endfor %} +
+{% else %} +
+
+

{% trans "You should find, install or build some applications." %}

+
+
+{% endif %} + +{% endblock %} diff --git a/hobo/applications/templates/hobo/applications/install.html b/hobo/applications/templates/hobo/applications/install.html new file mode 100644 index 0000000..549e85b --- /dev/null +++ b/hobo/applications/templates/hobo/applications/install.html @@ -0,0 +1,17 @@ +{% extends "hobo/base.html" %} +{% load i18n %} + +{% block appbar %} +

{% trans "Install" %}

+{% endblock %} + +{% block content %} +
+ {% csrf_token %} + {{ form.as_p }} +
+ + {% trans 'Cancel' %} +
+
+{% endblock %} diff --git a/hobo/applications/templates/hobo/applications/manifest.html b/hobo/applications/templates/hobo/applications/manifest.html new file mode 100644 index 0000000..9cb1d78 --- /dev/null +++ b/hobo/applications/templates/hobo/applications/manifest.html @@ -0,0 +1,62 @@ +{% extends "hobo/applications/home.html" %} +{% load i18n %} + +{% block breadcrumb %} +{{ block.super }} +{{ app.name }} +{% endblock %} + +{% block appbar %} +

{{ app.name }}

+ + {% trans 'Metadata' %} + +{% endblock %} + +{% block content %} + + + + {% if relations %} +
+ {% trans "Scan dependencies" %} +     + {% trans "Generate application bundle" %} + {% if versions %} +     + {% trans "Download" %} + {% endif %} +
+ {% else %} +
+
+

{% trans "You should now assemble the different parts of your application." %}

+
+
+ {% endif %} + +{% endblock %} + +{% block sidebar %} + +{% endblock %} diff --git a/hobo/applications/urls.py b/hobo/applications/urls.py new file mode 100644 index 0000000..ff62091 --- /dev/null +++ b/hobo/applications/urls.py @@ -0,0 +1,40 @@ +# hobo - portal to configure and deploy applications +# Copyright (C) 2015-2022 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.conf.urls import url + +from . import views + +urlpatterns = [ + url(r'^$', views.home, name='applications-home'), + url(r'^create/$', views.init, name='application-init'), + url(r'^install/$', views.install, name='application-install'), + url(r'^manifest/(?P[\w-]+)/$', views.manifest, name='application-manifest'), + url(r'^manifest/(?P[\w-]+)/metadata/$', views.metadata, name='application-metadata'), + url(r'^manifest/(?P[\w-]+)/scandeps/$', views.scandeps, name='application-scandeps'), + url(r'^manifest/(?P[\w-]+)/generate/$', views.generate, name='application-generate'), + url(r'^manifest/(?P[\w-]+)/download/$', views.download, name='application-download'), + url( + r'^manifest/(?P[\w-]+)/add/(?P[\w-]+)/$', + views.add_element, + name='application-add-element', + ), + url( + r'^manifest/(?P[\w-]+)/delete/(?P\d+)/$', + views.delete_element, + name='application-delete-element', + ), +] diff --git a/hobo/applications/utils.py b/hobo/applications/utils.py new file mode 100644 index 0000000..f6698f7 --- /dev/null +++ b/hobo/applications/utils.py @@ -0,0 +1,66 @@ +# hobo - portal to configure and deploy applications +# Copyright (C) 2015-2022 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import urllib + +import requests +from django.conf import settings +from django.utils.http import urlencode +from requests import Response +from requests import Session as RequestsSession +from requests.auth import AuthBase + +from hobo.signature import sign_url + + +class PublikSignature(AuthBase): + def __init__(self, secret): + self.secret = secret + + def __call__(self, request): + request.url = sign_url(request.url, self.secret) + return request + + +def get_known_service_for_url(url): + netloc = urllib.parse.urlparse(url).netloc + for services in settings.KNOWN_SERVICES.values(): + for service in services.values(): + remote_url = service.get('url') + if urllib.parse.urlparse(remote_url).netloc == netloc: + return service + return None + + +class Requests(RequestsSession): + def request(self, method, url, **kwargs): + remote_service = get_known_service_for_url(url) + kwargs['auth'] = PublikSignature(remote_service.get('secret')) + + # only keeps the path (URI) in url parameter, scheme and netloc are + # in remote_service + scheme, netloc, path, params, query, fragment = urllib.parse.urlparse(url) + url = urllib.parse.urlunparse(('', '', path, params, query, fragment)) + + query_params = {'orig': remote_service.get('orig')} + + remote_service_base_url = remote_service.get('url') + scheme, netloc, dummy, params, old_query, fragment = urllib.parse.urlparse(remote_service_base_url) + + query = urlencode(query_params) + url = urllib.parse.urlunparse((scheme, netloc, path, params, query, fragment)) + + return super().request(method, url, **kwargs) diff --git a/hobo/applications/views.py b/hobo/applications/views.py new file mode 100644 index 0000000..d7d1e2b --- /dev/null +++ b/hobo/applications/views.py @@ -0,0 +1,300 @@ +# hobo - portal to configure and deploy applications +# Copyright (C) 2015-2022 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import io +import json +import tarfile +import urllib.parse + +from django.conf import settings +from django.core.files.base import ContentFile +from django.http import HttpResponse, HttpResponseRedirect +from django.urls import reverse, reverse_lazy +from django.views.generic import FormView, ListView, TemplateView +from django.views.generic.edit import CreateView, DeleteView, UpdateView + +from hobo.environment.utils import get_installed_services + +from .forms import InstallForm +from .models import Application, Element, Relation, Version +from .utils import Requests + +requests = Requests() + + +class HomeView(ListView): + template_name = 'hobo/applications/home.html' + model = Application + + def get_queryset(self): + return super().get_queryset().order_by('name') + + +home = HomeView.as_view() + + +class InitView(CreateView): + template_name = 'hobo/applications/create.html' + model = Application + fields = ['name'] + + def get_success_url(self): + return reverse('application-manifest', kwargs={'app_slug': self.object.slug}) + + +init = InitView.as_view() + + +def get_object_types(): + object_types = [] + for service_id, services in getattr(settings, 'KNOWN_SERVICES', {}).items(): + if service_id not in Application.SUPPORTED_MODULES: + continue + service_objects = {x.get_base_url_path(): x for x in get_installed_services(types=[service_id])} + for service in services.values(): + if service_objects[service['url']].secondary: + continue + url = urllib.parse.urljoin(service['url'], 'api/export-import/') + response = requests.get(url) + if not response.ok: + continue + for object_type in response.json()['data']: + object_type['service'] = service + object_types.append(object_type) + return object_types + + +class ManifestView(TemplateView): + template_name = 'hobo/applications/manifest.html' + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + + context['app'] = Application.objects.get(slug=self.kwargs['app_slug']) + + context['relations'] = context['app'].relation_set.all().select_related('element') + context['versions'] = context['app'].version_set.exists() + context['types_by_service'] = {} + + type_labels = {} + for object_type in get_object_types(): + type_labels[object_type['id']] = object_type['singular'] + if object_type.get('minor'): + continue + service = object_type['service']['title'] + if service not in context['types_by_service']: + context['types_by_service'][service] = [] + context['types_by_service'][service].append(object_type) + + for relation in context['relations']: + relation.element.type_label = type_labels.get(relation.element.type) + + return context + + +manifest = ManifestView.as_view() + + +class MetadataView(UpdateView): + template_name = 'hobo/applications/edit-metadata.html' + model = Application + slug_url_kwarg = 'app_slug' + fields = ['name', 'slug', 'description'] + + def get_success_url(self): + return reverse('application-manifest', kwargs={'app_slug': self.object.slug}) + + +metadata = MetadataView.as_view() + + +class AppAddElementView(TemplateView): + template_name = 'hobo/applications/add-element.html' + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context['app'] = Application.objects.get(slug=self.kwargs['app_slug']) + for object_type in get_object_types(): + if object_type.get('id') == self.kwargs['type']: + context['type'] = object_type + url = object_type['urls']['list'] + response = requests.get(url) + context['elements'] = response.json()['data'] + break + return context + + def post(self, request, app_slug, type): + context = self.get_context_data() + app = context['app'] + element_infos = {x['id']: x for x in context['elements']} + for element_slug in request.POST.getlist('elements'): + element, created = Element.objects.get_or_create( + type=type, slug=element_slug, defaults={'name': element_infos[element_slug]['text']} + ) + element.name = element_infos[element_slug]['text'] + element.cache = element_infos[element_slug] + element.save() + relation, created = Relation.objects.get_or_create(application=app, element=element) + relation.auto_dependency = False + relation.save() + return HttpResponseRedirect(reverse('application-manifest', kwargs={'app_slug': app_slug})) + + +add_element = AppAddElementView.as_view() + + +class AppDeleteElementView(DeleteView): + model = Relation + template_name = 'hobo/applications/element_confirm_delete.html' + + def get_success_url(self): + return reverse('application-manifest', kwargs={'app_slug': self.kwargs['app_slug']}) + + +delete_element = AppDeleteElementView.as_view() + + +def scandeps(request, app_slug): + app = Application.objects.get(slug=app_slug) + app.relation_set.filter(auto_dependency=True).delete() + relations = app.relation_set.select_related('element') + elements = {(x.element.type, x.element.slug): x.element for x in relations} + finished = False + while not finished: + finished = True + for (type, slug), element in list(elements.items()): + dependencies_url = element.cache['urls'].get('dependencies') + element.done = True + if not dependencies_url: + continue + response = requests.get(dependencies_url) + for dependency in response.json()['data']: + if (dependency['type'], dependency['id']) in elements: + continue + finished = False + element, created = Element.objects.get_or_create( + type=dependency['type'], slug=dependency['id'], defaults={'name': dependency['text']} + ) + element.name = dependency['text'] + element.cache = dependency + element.save() + relation, created = Relation.objects.get_or_create(application=app, element=element) + if created: + relation.auto_dependency = True + relation.save() + elements[(element.type, element.slug)] = element + + return HttpResponseRedirect(reverse('application-manifest', kwargs={'app_slug': app_slug})) + + +def generate(request, app_slug): + app = Application.objects.get(slug=app_slug) + + version = Version(application=app) + version.save() + + tar_io = io.BytesIO() + with tarfile.open(mode='w', fileobj=tar_io) as tar: + manifest_json = { + 'application': app.name, + 'slug': app.slug, + 'description': app.description, + 'elements': [], + } + + for relation in app.relation_set.all().select_related('element'): + element = relation.element + manifest_json['elements'].append( + { + 'type': element.type, + 'slug': element.slug, + 'name': element.name, + 'auto-dependency': relation.auto_dependency, + } + ) + + response = requests.get(element.cache['urls']['export']) + tarinfo = tarfile.TarInfo('%s/%s' % (element.type, element.slug)) + tarinfo.mtime = version.last_update_timestamp.timestamp() + tarinfo.size = int(response.headers['content-length']) + tar.addfile(tarinfo, fileobj=io.BytesIO(response.content)) + + manifest_fd = io.BytesIO(json.dumps(manifest_json, indent=2).encode()) + tarinfo = tarfile.TarInfo('manifest.json') + tarinfo.size = len(manifest_fd.getvalue()) + tarinfo.mtime = version.last_update_timestamp.timestamp() + tar.addfile(tarinfo, fileobj=manifest_fd) + + version.bundle.save('%s.tar' % app_slug, content=ContentFile(tar_io.getvalue())) + version.save() + + return HttpResponseRedirect(reverse('application-manifest', kwargs={'app_slug': app_slug})) + + +def download(request, app_slug): + app = Application.objects.get(slug=app_slug) + version = app.version_set.order_by('-last_update_timestamp')[0] + response = HttpResponse(version.bundle, content_type='application/x-tar') + response['Content-Disposition'] = 'attachment; filename="%s"' % '%s.tar' % app_slug + return response + + +class Install(FormView): + form_class = InstallForm + template_name = 'hobo/applications/install.html' + success_url = reverse_lazy('applications-home') + + def form_valid(self, form): + tar_io = io.BytesIO(self.request.FILES['bundle'].read()) + tar = tarfile.open(fileobj=tar_io) + manifest = json.loads(tar.extractfile('manifest.json').read().decode()) + app, created = Application.objects.get_or_create( + slug=manifest.get('slug'), defaults={'name': manifest.get('application')} + ) + app.name = manifest.get('application') + app.description = manifest.get('description') + if created: + # mark as non-editable only newly deployed applications, this allows + # overwriting a local application and keep on developing it. + app.editable = False + else: + app.relation_set.all().delete() + app.save() + + for element_dict in manifest.get('elements'): + element, created = Element.objects.get_or_create( + type=element_dict['type'], slug=element_dict['slug'], defaults={'name': element_dict['name']} + ) + element.name = element_dict['name'] + element.save() + + relation = Relation( + application=app, + element=element, + auto_dependency=element_dict['auto-dependency'], + ) + relation.save() + + version = Version(application=app) + version.bundle.save('%s.tar' % app.slug, content=ContentFile(tar_io.getvalue())) + version.save() + + version.deploy() + + return super().form_valid(form) + + +install = Install.as_view() diff --git a/hobo/environment/utils.py b/hobo/environment/utils.py index 48370ec..0836e10 100644 --- a/hobo/environment/utils.py +++ b/hobo/environment/utils.py @@ -27,11 +27,13 @@ from hobo.multitenant.settings_loaders import KnownServices from hobo.profile.utils import get_profile_dict -def get_installed_services(): +def get_installed_services(types=None): from .models import AVAILABLE_SERVICES, Hobo installed_services = [] for available_service in AVAILABLE_SERVICES: + if types and available_service.Extra.service_id not in types: + continue if available_service is Hobo: installed_services.extend(available_service.objects.filter(local=False)) else: diff --git a/hobo/settings.py b/hobo/settings.py index 57192c2..718f0b5 100644 --- a/hobo/settings.py +++ b/hobo/settings.py @@ -42,6 +42,7 @@ INSTALLED_APPS = ( 'rest_framework', 'mellon', 'gadjo', + 'hobo.applications', 'hobo.debug', 'hobo.environment', 'hobo.franceconnect', diff --git a/hobo/static/css/applications.svg b/hobo/static/css/applications.svg new file mode 100644 index 0000000..b5bc78c --- /dev/null +++ b/hobo/static/css/applications.svg @@ -0,0 +1 @@ + diff --git a/hobo/static/css/build-application.svg b/hobo/static/css/build-application.svg new file mode 100644 index 0000000..36d63db --- /dev/null +++ b/hobo/static/css/build-application.svg @@ -0,0 +1 @@ + diff --git a/hobo/static/css/style.scss b/hobo/static/css/style.scss index dc09e9a..7266206 100644 --- a/hobo/static/css/style.scss +++ b/hobo/static/css/style.scss @@ -287,3 +287,49 @@ a.button.button-paragraph p { a.button.button-paragraph:hover p { color: white; } + +.application-elements { + max-height: 40vh; + overflow-y: scroll; + label { + display: block; + max-width: 40em; + } +} + +#application-empty { + background: url(build-application.svg) bottom right no-repeat; + background-size: auto 90%; + width: 100%; + height: 80vh; +} + +#no-applications { + background: url(applications.svg) bottom center no-repeat; + background-size: auto 90%; + width: 100%; + height: 80vh; +} + +#applications { + display: flex; + flex-wrap: wrap; + justify-content: space-between; + .application { + background: white; + flex: 1; + min-width: calc(50% - 0.5em); + max-width: calc(50% - 0.5em); + color: #3c3c33; + box-sizing: border-box; + border: 1px solid #ccc; + margin-bottom: 1rem; + padding: 0 1em; + h3 { + margin-top: 1em; + } + .buttons { + justify-content: flex-end; + } + } +} diff --git a/hobo/urls.py b/hobo/urls.py index 9839436..daa1517 100644 --- a/hobo/urls.py +++ b/hobo/urls.py @@ -18,6 +18,7 @@ from django.conf import settings from django.conf.urls import include, url from django.contrib import admin +from .applications.urls import urlpatterns as applications_urls from .debug.urls import urlpatterns as debug_urls from .emails.urls import urlpatterns as emails_urls from .environment.urls import urlpatterns as environment_urls @@ -43,6 +44,7 @@ urlpatterns = [ url(r'^seo/', decorated_includes(admin_required, include(seo_urls))), url(r'^sms/', decorated_includes(admin_required, include(sms_urls))), url(r'^debug/', decorated_includes(admin_required, include(debug_urls))), + url(r'^applications/', decorated_includes(admin_required, include(applications_urls))), url(r'^api/health/$', health_json, name='health-json'), url(r'^menu.json$', menu_json, name='menu_json'), url(r'^hobos.json$', hobo), diff --git a/tests/test_application.py b/tests/test_application.py new file mode 100644 index 0000000..60f04b6 --- /dev/null +++ b/tests/test_application.py @@ -0,0 +1,229 @@ +import io +import json +import tarfile + +import pytest +from httmock import HTTMock +from test_manager import login +from webtest import Upload + +from hobo.applications.models import Application +from hobo.environment.models import Wcs + +pytestmark = pytest.mark.django_db + +WCS_AVAILABLE_OBJECTS = { + "data": [ + { + "id": "forms", + "text": "Forms", + "singular": "Form", + "urls": {"list": "https://wcs.example.invalid/api/export-import/forms/"}, + }, + { + "id": "cards", + "text": "Card Models", + "singular": "Card Model", + "urls": {"list": "https://wcs.example.invalid/api/export-import/cards/"}, + }, + { + "id": "workflows", + "text": "Workflows", + "singular": "Workflow", + "urls": {"list": "https://wcs.example.invalid/api/export-import/workflows/"}, + }, + { + "id": "blocks", + "text": "Blocks", + "singular": "Block of fields", + "minor": True, + "urls": {"list": "https://wcs.example.invalid/api/export-import/blocks/"}, + }, + { + "id": "data-sources", + "text": "Data Sources", + "singular": "Data Source", + "minor": True, + "urls": {"list": "https://wcs.example.invalid/api/export-import/data-sources/"}, + }, + { + "id": "mail-templates", + "text": "Mail Templates", + "singular": "Mail Template", + "minor": True, + "urls": {"list": "https://wcs.example.invalid/api/export-import/mail-templates/"}, + }, + { + "id": "wscalls", + "text": "Webservice Calls", + "singular": "Webservice Call", + "minor": True, + "urls": {"list": "https://wcs.example.invalid/api/export-import/wscalls/"}, + }, + ] +} + +WCS_AVAILABLE_FORMS = { + "data": [ + { + "id": "test-form", + "text": "Test Form", + "type": "forms", + "urls": { + "export": "https://wcs.example.invalid/api/export-import/forms/test-form/", + "dependencies": "https://wcs.example.invalid/api/export-import/forms/test-form/dependencies/", + }, + }, + { + "id": "test2-form", + "text": "Second Test Form", + "type": "forms", + "urls": { + "export": "https://wcs.example.invalid/api/export-import/forms/test2-form/", + "dependencies": "https://wcs.example.invalid/api/export-import/forms/test2-form/dependencies/", + }, + }, + ] +} + +WCS_FORM_DEPENDENCIES = { + "data": [ + { + "id": "test-card", + "text": "Test Card", + "type": "cards", + "urls": { + "export": "https://wcs.example.invalid/api/export-import/cards/test-card/", + "dependencies": "https://wcs.example.invalid/api/export-import/cards/test-card/dependencies/", + }, + } + ] +} + + +def mocked_http(url, request): + assert '&signature=' in url.query + if url.netloc == 'wcs.example.invalid' and url.path == '/api/export-import/': + return {'content': json.dumps(WCS_AVAILABLE_OBJECTS), 'status_code': 200} + + if url.path == '/api/export-import/forms/': + return {'content': json.dumps(WCS_AVAILABLE_FORMS), 'status_code': 200} + + if url.path == '/api/export-import/forms/test-form/dependencies/': + return {'content': json.dumps(WCS_FORM_DEPENDENCIES), 'status_code': 200} + + if url.path.endswith('/dependencies/'): + return {'content': json.dumps({'data': []}), 'status_code': 200} + + if url.path == '/api/export-import/forms/test-form/': + return {'content': '', 'status_code': 200, 'headers': {'content-length': '10'}} + + if url.path == '/api/export-import/cards/test-card/': + return {'content': '', 'status_code': 200, 'headers': {'content-length': '10'}} + + if url.path == '/api/export-import/bundle-import/': + return {'content': '{}', 'status_code': 200} + + +def test_create_application(app, admin_user, settings): + Wcs.objects.create(base_url='https://wcs.example.invalid', slug='foobar', title='Foobar') + + settings.KNOWN_SERVICES = { + 'wcs': { + 'foobar': { + 'title': 'Foobar', + 'url': 'https://wcs.example.invalid/', + 'orig': 'example.org', + 'secret': 'xxx', + } + } + } + + login(app) + + resp = app.get('/applications/') + resp = resp.click('Create') + resp.form['name'] = 'Test' + resp = resp.form.submit() + + with HTTMock(mocked_http): + resp = resp.follow() + assert 'You should now assemble the different parts of your application.' in resp.text + + # edit metadata + resp = resp.click('Metadata') + resp.form['description'] = 'Lorem ipsum' + resp = resp.form.submit().follow() + + # add forms + assert '/add/forms/' in resp + resp = resp.click('Forms') + assert resp.form.fields['elements'][0]._value == 'test-form' + assert resp.form.fields['elements'][1]._value == 'test2-form' + resp.form.fields['elements'][0].checked = True + resp = resp.form.submit().follow() + assert Application.objects.get(slug='test').elements.count() == 1 + element = Application.objects.get(slug='test').elements.all()[0] + assert element.slug == 'test-form' + + resp = resp.click('Scan dependencies').follow() + assert Application.objects.get(slug='test').elements.count() == 2 + assert 'Test Card' in resp.text + + resp = resp.click('Generate application bundle').follow() + resp = resp.click('Download') + assert resp.content_type == 'application/x-tar' + # uncompressed tar, primitive check of contents + assert b'' in resp.content + assert b'' in resp.content + + +@pytest.fixture +def app_bundle(): + tar_io = io.BytesIO() + with tarfile.open(mode='w', fileobj=tar_io) as tar: + manifest_json = { + 'application': 'Test', + 'slug': 'test', + 'description': '', + 'elements': [ + {'type': 'forms', 'slug': 'test', 'name': 'test', 'auto-dependency': False}, + {'type': 'blocks', 'slug': 'test', 'name': 'test', 'auto-dependency': True}, + {'type': 'workflows', 'slug': 'test', 'name': 'test', 'auto-dependency': True}, + ], + } + manifest_fd = io.BytesIO(json.dumps(manifest_json, indent=2).encode()) + tarinfo = tarfile.TarInfo('manifest.json') + tarinfo.size = len(manifest_fd.getvalue()) + tar.addfile(tarinfo, fileobj=manifest_fd) + + return tar_io.getvalue() + + +def test_deploy_application(app, admin_user, settings, app_bundle): + Application.objects.all().delete() + Wcs.objects.create(base_url='https://wcs.example.invalid', slug='foobar', title='Foobar') + + settings.KNOWN_SERVICES = { + 'wcs': { + 'foobar': { + 'title': 'Foobar', + 'url': 'https://wcs.example.invalid/', + 'orig': 'example.org', + 'secret': 'xxx', + } + } + } + + login(app) + + resp = app.get('/applications/') + for i in range(2): + resp = resp.click('Install') + resp.form['bundle'] = Upload('app.tar', app_bundle, 'application/x-tar') + with HTTMock(mocked_http): + resp = resp.form.submit().follow() + + assert Application.objects.count() == 1 + assert Application.objects.get(slug='test').name == 'Test' + assert Application.objects.get(slug='test').elements.count() == 3