# hobo - portal to configure and deploy applications # Copyright (C) 2015-2022 Entr'ouvert # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU Affero General Public License as published # by the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . import dataclasses import io import json import tarfile from django.core.files.base import ContentFile from django.db.models import Prefetch from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import get_object_or_404 from django.urls import reverse, reverse_lazy from django.utils.text import slugify from django.utils.timezone import now from django.utils.translation import gettext_lazy as _ from django.views.generic import DetailView, FormView, ListView, TemplateView from django.views.generic.edit import CreateView, DeleteView, UpdateView from .forms import GenerateForm, InstallForm, MetadataForm from .models import STATUS_CHOICES, Application, AsyncJob, Element, Relation, Version, get_object_types from .utils import Requests requests = Requests() class HomeView(ListView): template_name = 'hobo/applications/home.html' model = Application def get_queryset(self): return super().get_queryset().order_by('name') home = HomeView.as_view() class InitView(CreateView): template_name = 'hobo/applications/create.html' model = Application fields = ['name'] def get_success_url(self): return reverse('application-manifest', kwargs={'app_slug': self.object.slug}) init = InitView.as_view() class ManifestView(TemplateView): template_name = 'hobo/applications/manifest.html' def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context['app'] = get_object_or_404(Application, slug=self.kwargs['app_slug']) context['relations'] = context['app'].relation_set.all().select_related('element') context['last_version'] = context['app'].version_set.order_by('last_update_timestamp').last() context['types_by_service'] = {} type_labels = {} object_types = get_object_types() types = [o['id'] for o in object_types] for object_type in object_types: type_labels[object_type['id']] = object_type['singular'] if object_type.get('minor'): continue service = object_type['service']['title'] if service not in context['types_by_service']: context['types_by_service'][service] = [] context['types_by_service'][service].append(object_type) for relation in context['relations']: relation.element.type_label = type_labels.get(relation.element.type) context['relations'] = sorted( context['relations'], key=lambda a: (a.auto_dependency, types.index(a.element.type), slugify(a.element.name)), ) return context manifest = ManifestView.as_view() class VersionsView(ListView): template_name = 'hobo/applications/versions.html' def get_queryset(self): self.app = get_object_or_404(Application, slug=self.kwargs['app_slug']) return self.app.version_set.order_by('-last_update_timestamp').prefetch_related( Prefetch('asyncjob_set', queryset=AsyncJob.objects.order_by('-creation_timestamp')) ) def get_context_data(self, **kwargs): kwargs['app'] = self.app return super().get_context_data(**kwargs) versions = VersionsView.as_view() class MetadataView(UpdateView): template_name = 'hobo/applications/edit-metadata.html' model = Application slug_url_kwarg = 'app_slug' form_class = MetadataForm def get_queryset(self): return super().get_queryset().filter(editable=True) def get_success_url(self): return reverse('application-manifest', kwargs={'app_slug': self.object.slug}) metadata = MetadataView.as_view() @dataclasses.dataclass class Category: name: str elements: list class AppAddElementView(TemplateView): template_name = 'hobo/applications/add-element.html' def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context['app'] = get_object_or_404(Application, slug=self.kwargs['app_slug'], editable=True) for object_type in get_object_types(): if object_type.get('id') == self.kwargs['type']: context['type'] = object_type url = object_type['urls']['list'] response = requests.get(url) elements = response.json()['data'] category_names = {el.get('category') or '' for el in elements} categories = [ Category( name=c, elements=sorted( [el for el in elements if el.get('category') == c], key=lambda a: slugify(a['text']), ), ) for c in sorted(list(category_names)) if c ] categories.append( Category( name=_('Uncategorized'), elements=sorted( [el for el in elements if not el.get('category')], key=lambda a: slugify(a['text']), ), ) ) context['categories'] = categories break return context def post(self, request, app_slug, type): context = self.get_context_data() app = context['app'] element_infos = {x['id']: x for c in context['categories'] for x in c.elements} for element_slug in request.POST.getlist('elements'): element, created = Element.objects.get_or_create( type=type, slug=element_slug, defaults={'name': element_infos[element_slug]['text']} ) element.name = element_infos[element_slug]['text'] element.cache = element_infos[element_slug] element.save() relation, created = Relation.objects.get_or_create(application=app, element=element) relation.auto_dependency = False relation.save() return HttpResponseRedirect(reverse('application-manifest', kwargs={'app_slug': app_slug})) add_element = AppAddElementView.as_view() class AppDeleteElementView(DeleteView): model = Relation template_name = 'hobo/applications/element_confirm_delete.html' def get_queryset(self): return super().get_queryset().filter(application__editable=True) def get_success_url(self): return reverse('application-manifest', kwargs={'app_slug': self.kwargs['app_slug']}) delete_element = AppDeleteElementView.as_view() def scandeps(request, app_slug): app = get_object_or_404(Application, slug=app_slug, editable=True) job = AsyncJob( label=_('Scanning for dependencies'), application=app, action='scandeps', ) job.save() job.run(spool=True) if job.status == 'registered': return HttpResponseRedirect( reverse('application-async-job', kwargs={'app_slug': app_slug, 'pk': job.id}) ) return HttpResponseRedirect(reverse('application-manifest', kwargs={'app_slug': app_slug})) class GenerateView(FormView): form_class = GenerateForm template_name = 'hobo/applications/generate.html' def get_initial(self): self.app = get_object_or_404(Application, slug=self.kwargs['app_slug'], editable=True) version = self.app.version_set.order_by('last_update_timestamp').last() if version: self.initial['number'] = version.number self.initial['notes'] = version.notes return super().get_initial() def form_valid(self, form): app = self.app latest_version = app.version_set.order_by('last_update_timestamp').last() if latest_version and latest_version.number == form.cleaned_data['number']: version = latest_version else: version = Version(application=app) version.number = form.cleaned_data['number'] version.notes = form.cleaned_data['notes'] version.save() job = AsyncJob( label=_('Creating application bundle'), application=app, version=version, action='create_bundle', ) job.save() job.run(spool=True) if job.status == 'registered': return HttpResponseRedirect( reverse('application-async-job', kwargs={'app_slug': app.slug, 'pk': job.id}) ) return super().form_valid(form) def get_context_data(self, **kwargs): kwargs['app_slug'] = self.kwargs['app_slug'] return super().get_context_data(**kwargs) def get_success_url(self): return reverse('application-manifest', kwargs={'app_slug': self.kwargs['app_slug']}) generate = GenerateView.as_view() def download(request, app_slug, version_pk=None): app = get_object_or_404(Application, slug=app_slug) if version_pk is None: version = app.version_set.order_by('last_update_timestamp').last() else: version = get_object_or_404(app.version_set, pk=version_pk) response = HttpResponse(version.bundle, content_type='application/x-tar') response['Content-Disposition'] = 'attachment; filename="%s"' % '%s.tar' % app_slug return response class Install(FormView): form_class = InstallForm template_name = 'hobo/applications/install.html' success_url = reverse_lazy('applications-home') application = None def form_valid(self, form): tar_io = io.BytesIO(self.request.FILES['bundle'].read()) tar = tarfile.open(fileobj=tar_io) manifest = json.loads(tar.extractfile('manifest.json').read().decode()) if self.application and self.application.slug != manifest.get('slug'): form.add_error( 'bundle', _('Can not update this application, wrong slug (%s).') % manifest.get('slug') ) return self.form_invalid(form) app, created = Application.objects.get_or_create( slug=manifest.get('slug'), defaults={'name': manifest.get('application')} ) app.name = manifest.get('application') app.description = manifest.get('description') app.documentation_url = manifest.get('documentation_url') or '' if created: # mark as non-editable only newly deployed applications, this allows # overwriting a local application and keep on developing it. app.editable = False else: app.relation_set.all().delete() app.save() icon = manifest.get('icon') if icon: app.icon.save(icon, tar.extractfile(icon), save=True) else: app.icon.delete() for element_dict in manifest.get('elements'): element, created = Element.objects.get_or_create( type=element_dict['type'], slug=element_dict['slug'], defaults={'name': element_dict['name']} ) element.name = element_dict['name'] element.save() relation = Relation( application=app, element=element, auto_dependency=element_dict['auto-dependency'], ) relation.save() # always create a new version on install version_number = manifest.get('version_number') or 'unknown' latest_version = app.version_set.order_by('last_update_timestamp').last() if latest_version and latest_version.number == version_number: version = latest_version else: version = Version(application=app) version.number = version_number version.notes = manifest.get('version_notes') or '' version.bundle.save('%s.tar' % app.slug, content=ContentFile(tar_io.getvalue())) version.save() job = AsyncJob( label=_('Deploying application bundle'), application=app, version=version, action='deploy', ) job.save() job.run(spool=True) if job.status == 'registered': return HttpResponseRedirect( reverse('application-async-job', kwargs={'app_slug': app.slug, 'pk': job.id}) ) return super().form_valid(form) install = Install.as_view() class Update(Install): template_name = 'hobo/applications/update.html' def get_context_data(self, **kwargs): kwargs['app'] = self.application return super().get_context_data(**kwargs) def dispatch(self, request, *args, **kwargs): self.application = get_object_or_404(Application, slug=kwargs['app_slug'], editable=False) return super().dispatch(request, *args, **kwargs) update = Update.as_view() class AppDeleteView(DeleteView): model = Application template_name = 'hobo/applications/app_confirm_delete.html' def get_success_url(self): return reverse('applications-home') delete = AppDeleteView.as_view() class AsyncJobView(DetailView): model = AsyncJob template_name = 'hobo/applications/job.html' def get_context_data(self, **kwargs): kwargs['job_progression'] = {} job = self.object if job.action == 'deploy' and job.status != 'failed': for service_id, services in job.progression_urls.items(): for service, url in services.items(): response = requests.get(url) if not response.ok: continue kwargs['job_progression'][service] = response.json() kwargs['job_progression'][service].update({'service_id': service_id}) kwargs['services_all_completed'] = all( [s['data']['status'] == 'completed' for s in kwargs['job_progression'].values()] ) failed = [ s['service_id'] for s in kwargs['job_progression'].values() if s['data']['status'] == 'failed' ] if failed: job.status = 'failed' if len(failed) > 1: job.exception = 'Failed to deploy modules %s' % ', '.join(failed) else: job.exception = 'Failed to deploy module %s' % failed[0] job.completion_timestamp = now() job.save() kwargs['wait_for_services'] = True kwargs['service_job_status_choices'] = {c[0]: c[1] for c in STATUS_CHOICES} return super().get_context_data(**kwargs) def get_redirect_url(self): return reverse('application-manifest', kwargs={'app_slug': self.kwargs['app_slug']}) async_job = AsyncJobView.as_view()