hobo/hobo/applications/views.py

818 lines
29 KiB
Python

# hobo - portal to configure and deploy applications
# Copyright (C) 2015-2022 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import dataclasses
import difflib
import io
import json
import tarfile
from django.contrib import messages
from django.core.files.base import ContentFile
from django.db.models import Prefetch
from django.http import Http404, HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404
from django.urls import reverse
from django.utils.formats import date_format
from django.utils.text import slugify
from django.utils.timezone import localtime, now
from django.utils.translation import gettext_lazy as _
from django.views.generic import DetailView, FormView, ListView, RedirectView, TemplateView
from django.views.generic.edit import CreateView, DeleteView, UpdateView
from hobo.environment.models import Variable
from .forms import GenerateForm, InstallForm, MetadataForm, VersionSelectForm
from .models import (
STATUS_CHOICES,
Application,
ApplicationError,
AsyncJob,
Element,
Parameter,
Relation,
UnlinkError,
Version,
get_object_type_index,
get_object_types,
)
from .utils import Requests
requests = Requests()
class HomeView(ListView):
template_name = 'hobo/applications/home.html'
model = Application
def get_queryset(self):
return super().get_queryset().order_by('name')
home = HomeView.as_view()
class InitView(CreateView):
template_name = 'hobo/applications/create.html'
model = Application
fields = ['name']
def get_success_url(self):
return reverse('application-manifest', kwargs={'app_slug': self.object.slug})
init = InitView.as_view()
class ManifestView(TemplateView):
template_name = 'hobo/applications/manifest.html'
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
context['app'] = get_object_or_404(Application, slug=self.kwargs['app_slug'])
context['relations'] = context['app'].relation_set.all().select_related('element')
context['last_version'] = context['app'].version_set.order_by('last_update_timestamp').last()
context['types_by_service'] = {}
type_labels = {}
object_types = get_object_types()
types = [o['id'] for o in object_types]
for object_type in object_types:
type_labels[object_type['id']] = object_type['singular']
if object_type.get('minor'):
continue
service = object_type['service']['title']
if service not in context['types_by_service']:
context['types_by_service'][service] = []
context['types_by_service'][service].append(object_type)
for relation in context['relations']:
relation.element.type_label = (
type_labels.get(relation.element.type) or _('Unknown (%s)') % relation.element.type
)
context['relations'] = sorted(
context['relations'],
key=lambda a: (
a.auto_dependency,
get_object_type_index(a.element.type, types),
slugify(a.element.name),
),
)
return context
manifest = ManifestView.as_view()
class VersionsView(ListView):
template_name = 'hobo/applications/versions.html'
def get_queryset(self):
self.app = get_object_or_404(Application, slug=self.kwargs['app_slug'])
return self.app.version_set.order_by('-last_update_timestamp').prefetch_related(
Prefetch('asyncjob_set', queryset=AsyncJob.objects.order_by('-creation_timestamp'))
)
def get_context_data(self, **kwargs):
kwargs['app'] = self.app
return super().get_context_data(**kwargs)
versions = VersionsView.as_view()
class VersionSelectView(FormView):
template_name = 'hobo/applications/version_select.html'
form_class = VersionSelectForm
def dispatch(self, request, *args, **kwargs):
self.application = get_object_or_404(Application, slug=self.kwargs['app_slug'])
self.version = get_object_or_404(Version, pk=self.kwargs['version_pk'], application=self.application)
return super().dispatch(request, *args, **kwargs)
def get_form_kwargs(self):
kwargs = super().get_form_kwargs()
kwargs['application'] = self.application
kwargs['version'] = self.version
return kwargs
def get_context_data(self, **kwargs):
kwargs['app'] = self.application
kwargs['version'] = self.version
return super().get_context_data(**kwargs)
def form_valid(self, form):
return HttpResponseRedirect(
'%s?version1=%s&version2=%s'
% (
reverse('application-version-compare', kwargs={'app_slug': self.application.slug}),
self.version.pk,
form.cleaned_data['version'].pk,
)
)
version_select = VersionSelectView.as_view()
class VersionCompareView(DetailView):
model = Application
template_name = 'hobo/applications/version_compare.html'
slug_url_kwarg = 'app_slug'
def get_context_data(self, **kwargs):
kwargs['app'] = self.object
id1 = self.request.GET.get('version1')
id2 = self.request.GET.get('version2')
mode = self.request.GET.get('mode') or 'manifest'
if not id1 or not id2:
raise Http404
if mode not in ['manifest', 'elements']:
raise Http404
version1 = get_object_or_404(Version, pk=id1, application=self.object)
version2 = get_object_or_404(Version, pk=id2, application=self.object)
if version1.last_update_timestamp > version2.last_update_timestamp:
version1, version2 = version2, version1
kwargs['mode'] = mode
kwargs['version1'] = version1
kwargs['version2'] = version2
kwargs['fromdesc'] = self.get_version_desc(version1)
kwargs['todesc'] = self.get_version_desc(version2)
kwargs.update(getattr(self, 'get_compare_%s_context' % mode)(version1, version2))
return super().get_context_data(**kwargs)
def get_manifest(self, version):
bundle = version.bundle.read()
tar_io = io.BytesIO(bundle)
with tarfile.open(fileobj=tar_io) as tar:
manifest = json.loads(tar.extractfile('manifest.json').read().decode())
# sort elements, to compare ordered lists
elements = manifest.get('elements') or []
object_types = get_object_types()
types = [o['id'] for o in object_types]
manifest['elements'] = sorted(
elements, key=lambda a: (a['auto-dependency'], types.index(a['type']), slugify(a['name']))
)
return manifest
def get_compare_manifest_context(self, version1, version2):
manifest1 = self.get_manifest(version1)
s1 = json.dumps(manifest1, sort_keys=True, indent=2)
manifest2 = self.get_manifest(version2)
s2 = json.dumps(manifest2, sort_keys=True, indent=2)
diff_serialization = difflib.HtmlDiff(wrapcolumn=160).make_table(
fromlines=s1.splitlines(True),
tolines=s2.splitlines(True),
)
return {
'diff_serialization': diff_serialization,
}
def get_compare_elements_context(self, version1, version2):
type_labels = {}
object_types = get_object_types()
types = [o['id'] for o in object_types]
for object_type in object_types:
type_labels[object_type['id']] = object_type['singular']
# take more recent version
bundle = version2.bundle.read()
tar_io = io.BytesIO(bundle)
with tarfile.open(fileobj=tar_io) as tar:
manifest = json.loads(tar.extractfile('manifest.json').read().decode())
# and extract elements
elements = manifest.get('elements') or []
# exclude roles, impossible to diff
elements = [e for e in elements if e['type'] != 'roles']
# sort elements
elements = sorted(
elements, key=lambda a: (a['auto-dependency'], types.index(a['type']), slugify(a['name']))
)
# and complete with real elements to have more information, such as redirect url
for element in elements:
element['type_label'] = type_labels.get(element['type'])
try:
real_element = Element.objects.get(type=element['type'], slug=element['slug'])
except Element.DoesNotExist:
element['real_element'] = None
continue
element['real_element'] = real_element
return {'elements': elements}
def get_version_desc(self, version):
return '{name} {number} ({timestamp})'.format(
name=_('Version'),
number=version.number,
timestamp=date_format(localtime(version.last_update_timestamp), format='DATETIME_FORMAT'),
)
version_compare = VersionCompareView.as_view()
class MetadataView(UpdateView):
template_name = 'hobo/applications/edit-metadata.html'
model = Application
slug_url_kwarg = 'app_slug'
form_class = MetadataForm
def get_queryset(self):
return super().get_queryset().filter(editable=True)
def get_success_url(self):
return reverse('application-manifest', kwargs={'app_slug': self.object.slug})
metadata = MetadataView.as_view()
@dataclasses.dataclass
class Category:
name: str
elements: list
class AppAddElementView(TemplateView):
template_name = 'hobo/applications/add-element.html'
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
context['app'] = get_object_or_404(Application, slug=self.kwargs['app_slug'], editable=True)
for object_type in get_object_types():
if object_type.get('id') == self.kwargs['type']:
context['type'] = object_type
url = object_type['urls']['list']
response = requests.get(url)
elements = response.json()['data']
for element in elements:
element['indentation'] = '\u00a0' * 2 * int(element.get('indent', '0'))
category_names = {el.get('category') or '' for el in elements}
categories = [
Category(
name=c,
elements=sorted(
[el for el in elements if el.get('category') == c],
key=lambda a: a.get('order', slugify(a['text'])),
),
)
for c in sorted(list(category_names))
if c
]
categories.append(
Category(
name=_('Uncategorized'),
elements=sorted(
[el for el in elements if not el.get('category')],
key=lambda a: a.get('order', slugify(a['text'])),
),
)
)
context['categories'] = categories
if len(categories) == 1:
categories[0].name = ''
break
return context
def post(self, request, app_slug, type):
context = self.get_context_data()
app = context['app']
element_infos = {x['id']: x for c in context['categories'] for x in c.elements}
for element_slug in request.POST.getlist('elements'):
element, dummy = Element.objects.get_or_create(
type=type, slug=element_slug, defaults={'name': element_infos[element_slug]['text']}
)
element.name = element_infos[element_slug]['text']
element.cache = element_infos[element_slug]
element.save()
relation, dummy = Relation.objects.get_or_create(application=app, element=element)
relation.auto_dependency = False
relation.save()
return HttpResponseRedirect(reverse('application-manifest', kwargs={'app_slug': app_slug}))
add_element = AppAddElementView.as_view()
class AppDeleteElementView(DeleteView):
model = Relation
template_name = 'hobo/applications/element_confirm_delete.html'
def get_queryset(self):
return super().get_queryset().filter(application__editable=True)
def get_success_url(self):
return reverse('application-manifest', kwargs={'app_slug': self.kwargs['app_slug']})
delete_element = AppDeleteElementView.as_view()
def scandeps(request, app_slug):
app = get_object_or_404(Application, slug=app_slug, editable=True)
job = AsyncJob(
label=_('Scanning for dependencies'),
application=app,
action='scandeps',
)
job.save()
job.run(spool=True)
if job.status == 'registered':
return HttpResponseRedirect(
reverse('application-async-job', kwargs={'app_slug': app_slug, 'pk': job.id})
)
return HttpResponseRedirect(reverse('application-manifest', kwargs={'app_slug': app_slug}))
class GenerateView(FormView):
form_class = GenerateForm
template_name = 'hobo/applications/generate.html'
def get_form_kwargs(self):
kwargs = super().get_form_kwargs()
self.app = get_object_or_404(Application, slug=self.kwargs['app_slug'], editable=True)
kwargs['latest_version'] = self.app.version_set.order_by('last_update_timestamp').last()
return kwargs
def form_valid(self, form):
app = self.app
version = Version(application=app)
version.number = form.get_cleaned_number()
version.notes = form.cleaned_data['notes']
version.save()
job = AsyncJob(
label=_('Creating application bundle'),
application=app,
version=version,
action='create_bundle',
)
job.save()
job.run(spool=True)
if job.status == 'registered':
return HttpResponseRedirect(
reverse('application-async-job', kwargs={'app_slug': app.slug, 'pk': job.id})
)
return super().form_valid(form)
def get_context_data(self, **kwargs):
kwargs['app_slug'] = self.kwargs['app_slug']
return super().get_context_data(**kwargs)
def get_success_url(self):
return reverse('application-manifest', kwargs={'app_slug': self.kwargs['app_slug']})
generate = GenerateView.as_view()
def download(request, app_slug, version_pk=None):
app = get_object_or_404(Application, slug=app_slug)
if version_pk is None:
version = app.version_set.order_by('last_update_timestamp').last()
else:
version = get_object_or_404(app.version_set, pk=version_pk)
response = HttpResponse(version.bundle, content_type='application/x-tar')
response['Content-Disposition'] = 'attachment; filename="%s"' % '%s-%s.tar' % (app_slug, version.number)
return response
class Install(FormView):
form_class = InstallForm
template_name = 'hobo/applications/install.html'
application = None
def form_valid(self, form):
tar_io = io.BytesIO(self.request.FILES['bundle'].read())
try:
with tarfile.open(fileobj=tar_io) as tar:
manifest = json.loads(tar.extractfile('manifest.json').read().decode())
if self.application and self.application.slug != manifest.get('slug'):
form.add_error(
'bundle',
_('Can not update this application, wrong slug (%s).') % manifest.get('slug'),
)
return self.form_invalid(form)
app, created = Application.objects.get_or_create(
slug=manifest.get('slug'), defaults={'name': manifest.get('application')}
)
self.application = app
app.name = manifest.get('application')
app.description = manifest.get('description')
app.documentation_url = manifest.get('documentation_url') or ''
if created:
# mark as non-editable only newly deployed applications, this allows
# overwriting a local application and keep on developing it.
app.editable = False
app.save()
icon = manifest.get('icon')
if icon:
app.icon.save(icon, tar.extractfile(icon), save=True)
else:
app.icon.delete()
except tarfile.TarError:
form.add_error('bundle', _('Invalid tar file.'))
return self.form_invalid(form)
# always create a new version on install or if previous version has not the same number
version_number = manifest.get('version_number') or 'unknown'
latest_version = app.version_set.order_by('last_update_timestamp').last()
if latest_version and latest_version.number == version_number:
version = latest_version
else:
version = Version(application=app)
version.number = version_number
version.notes = manifest.get('version_notes') or ''
version.bundle.save('%s.tar' % app.slug, content=ContentFile(tar_io.getvalue()))
version.save()
self.version = version
# check if some objects where locally modified or already exist outside the application
job = AsyncJob(
label=_('Check installation'),
application=self.application,
version=self.version,
action='check-first-install' if created else 'check-install',
)
job.save()
try:
job.run()
except ApplicationError:
pass
return super().form_valid(form)
def get_success_url(self):
return reverse(
'application-confirm-install',
kwargs={'app_slug': self.application.slug, 'version_pk': self.version.pk},
)
install = Install.as_view()
class Update(Install):
template_name = 'hobo/applications/update.html'
def get_context_data(self, **kwargs):
kwargs['app'] = self.application
return super().get_context_data(**kwargs)
def dispatch(self, request, *args, **kwargs):
self.application = get_object_or_404(Application, slug=kwargs['app_slug'], editable=False)
return super().dispatch(request, *args, **kwargs)
update = Update.as_view()
class ConfirmInstall(TemplateView):
template_name = 'hobo/applications/check-install.html'
def dispatch(self, *args, **kwargs):
self.application = get_object_or_404(Application, slug=self.kwargs['app_slug'])
self.version = get_object_or_404(Version, application=self.application, pk=self.kwargs['version_pk'])
self.last_job = self.version.asyncjob_set.order_by('creation_timestamp').last()
return super().dispatch(*args, **kwargs)
def get_context_data(self, **kwargs):
kwargs['app'] = self.application
kwargs['version'] = self.version
kwargs['last_job'] = self.last_job
if self.last_job:
diffs, not_found, no_history, legacy = self.last_job.get_diff_details()
kwargs['diffs'] = diffs
kwargs['not_found'] = not_found
kwargs['no_history'] = no_history
kwargs['legacy'] = legacy
return super().get_context_data(**kwargs)
def get(self, request, *args, **kwargs):
if self.last_job is None or self.last_job.action not in ['check-install', 'check-first-install']:
return self.install()
if self.last_job.status == 'completed':
no_history_only = True
no_legacy = True
for service in self.last_job.details.values():
for data in service.values():
if data.get('differences'):
no_history_only = False
if data.get('unknown_elements'):
no_history_only = False
if data.get('legacy_elements'):
no_history_only = False
no_legacy = False
if not data.get('no_history_elements'):
no_history_only = False
if no_history_only is True:
# legacy app, no application information found in elements history, jump to update page
return self.install()
if self.last_job.action == 'check-first-install' and no_legacy:
# first install, no legacy elements, jump to install page
return self.install()
return super().get(request, *args, **kwargs)
def post(self, request, *args, **kwargs):
return self.install()
def install(self):
# create elements and relations
bundle = self.version.bundle.read()
tar_io = io.BytesIO(bundle)
with tarfile.open(fileobj=tar_io) as tar:
manifest = json.loads(tar.extractfile('manifest.json').read().decode())
self.application.relation_set.all().delete()
for element_dict in manifest.get('elements'):
element, dummy = Element.objects.get_or_create(
type=element_dict['type'],
slug=element_dict['slug'],
defaults={'name': element_dict['name']},
)
element.name = element_dict['name']
element.save()
relation = Relation(
application=self.application,
element=element,
auto_dependency=element_dict['auto-dependency'],
)
relation.set_error('not-installed')
relation.save()
self.version.bundle.seek(0)
# and run deployment
job = AsyncJob(
label=_('Deploying application bundle'),
application=self.application,
version=self.version,
action='deploy',
)
job.save()
job.run(spool=True)
if job.status == 'registered':
return HttpResponseRedirect(
reverse('application-async-job', kwargs={'app_slug': self.application.slug, 'pk': job.id})
)
return HttpResponseRedirect(
reverse('application-manifest', kwargs={'app_slug': self.application.slug})
)
confirm_install = ConfirmInstall.as_view()
class Refresh(RedirectView):
def get_redirect_url(self, *args, **kwargs):
application = get_object_or_404(Application, slug=kwargs['app_slug'])
try:
application.refresh_elements(cache_only=True)
except ApplicationError:
pass
return reverse('application-manifest', kwargs={'app_slug': self.kwargs['app_slug']})
refresh = Refresh.as_view()
class AppDeleteView(DeleteView):
model = Application
template_name = 'hobo/applications/app_confirm_delete.html'
def delete(self, request, *args, **kwargs):
self.object = self.get_object()
success_url = self.get_success_url()
try:
self.object.unlink()
except UnlinkError as e:
messages.error(self.request, str(e))
return HttpResponseRedirect(
reverse('application-manifest', kwargs={'app_slug': self.kwargs['slug']})
)
self.object.delete()
return HttpResponseRedirect(success_url)
def get_success_url(self):
return reverse('applications-home')
delete = AppDeleteView.as_view()
class AsyncJobView(DetailView):
model = AsyncJob
template_name = 'hobo/applications/job.html'
def get_context_data(self, **kwargs):
kwargs['job_progression'] = {}
job = self.object
if job.action == 'deploy' and job.status != 'failed':
for service_id, services in job.progression_urls.items():
for service, url in services.items():
response = requests.get(url)
if not response.ok:
continue
kwargs['job_progression'][service] = response.json()
kwargs['job_progression'][service].update({'service_id': service_id})
kwargs['services_all_completed'] = all(
[s['data']['status'] == 'completed' for s in kwargs['job_progression'].values()]
)
failed = [
s['service_id'] for s in kwargs['job_progression'].values() if s['data']['status'] == 'failed'
]
if failed:
job.status = 'failed'
if len(failed) > 1:
job.exception = 'Failed to deploy modules %s' % ', '.join(failed)
else:
job.exception = 'Failed to deploy module %s' % failed[0]
job.completion_timestamp = now()
job.save()
kwargs['wait_for_services'] = True
kwargs['service_job_status_choices'] = {c[0]: c[1] for c in STATUS_CHOICES}
return super().get_context_data(**kwargs)
def get_redirect_url(self):
if self.object.action == 'deploy':
return reverse('application-refresh', kwargs={'app_slug': self.kwargs['app_slug']})
return reverse('application-manifest', kwargs={'app_slug': self.kwargs['app_slug']})
async_job = AsyncJobView.as_view()
class AsyncJobDiffsView(DetailView):
model = AsyncJob
template_name = 'hobo/applications/job_diffs.html'
def get_queryset(self):
return (
super()
.get_queryset()
.filter(
application__slug=self.kwargs['app_slug'],
action__in=['check-install', 'check-first-install'],
status='completed',
)
)
def get_context_data(self, **kwargs):
kwargs['app'] = self.object.application
diffs, not_found, no_history, legacy = self.object.get_diff_details()
kwargs['diffs'] = diffs
kwargs['not_found'] = not_found
kwargs['no_history'] = no_history
kwargs['legacy'] = legacy
kwargs['is_report'] = True
return super().get_context_data(**kwargs)
async_job_diffs = AsyncJobDiffsView.as_view()
class AddParameterView(CreateView):
template_name = 'hobo/applications/parameter-add.html'
model = Parameter
fields = ['label', 'name', 'default_value']
def dispatch(self, *args, **kwargs):
self.application = get_object_or_404(Application, slug=kwargs['app_slug'], editable=True)
return super().dispatch(*args, **kwargs)
def form_valid(self, form):
form.instance.application = self.application
form.instance.name = slugify(form.cleaned_data['name']).replace('-', '_')
return super().form_valid(form)
def get_success_url(self):
return reverse('application-manifest', kwargs={'app_slug': self.kwargs['app_slug']})
add_parameter = AddParameterView.as_view()
class EditParameterView(UpdateView):
template_name = 'hobo/applications/parameter-edit.html'
model = Parameter
fields = ['label', 'name', 'default_value']
def form_valid(self, form):
form.instance.name = slugify(form.cleaned_data['name']).replace('-', '_')
return super().form_valid(form)
def get_queryset(self):
return (
super()
.get_queryset()
.filter(application__editable=True, application__slug=self.kwargs['app_slug'])
)
def get_success_url(self):
return reverse('application-manifest', kwargs={'app_slug': self.kwargs['app_slug']})
edit_parameter = EditParameterView.as_view()
class DeleteParameterView(DeleteView):
template_name = 'hobo/applications/parameter-confirm-delete.html'
model = Parameter
def get_queryset(self):
return (
super()
.get_queryset()
.filter(application__editable=True, application__slug=self.kwargs['app_slug'])
)
def get_success_url(self):
return reverse('application-manifest', kwargs={'app_slug': self.kwargs['app_slug']})
delete_parameter = DeleteParameterView.as_view()
class ChangeParameterValueView(UpdateView):
template_name = 'hobo/applications/parameter-value-edit.html'
model = Variable
fields = ['value']
slug_field = 'name'
slug_url_kwarg = 'name'
def get_success_url(self):
return reverse('application-manifest', kwargs={'app_slug': self.kwargs['app_slug']})
change_parameter_value = ChangeParameterValueView.as_view()