applications: make dependency scanning and bundle creation async jobs (#70501)

debian/server/uwsgi.ini

@@ -13,6 +13,7 @@ chmod-socket = 666
 vacuum = true
 
 spooler-processes = 3
+spooler-python-import = hobo.applications.spooler
 spooler-python-import = hobo.provisionning.spooler
 spooler-max-tasks = 20
 

hobo/applications/migrations/0007_asyncjob.py

@@ -0,0 +1,55 @@
+# Generated by Django 3.2.13 on 2022-10-30 12:47
+
+import django.db.models.deletion
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('applications', '0006_documentation_url'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='AsyncJob',
+            fields=[
+                (
+                    'id',
+                    models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+                ),
+                ('label', models.CharField(max_length=100)),
+                (
+                    'status',
+                    models.CharField(
+                        choices=[
+                            ('registered', 'Registered'),
+                            ('running', 'Running'),
+                            ('failed', 'Failed'),
+                            ('completed', 'Completed'),
+                        ],
+                        default='registered',
+                        max_length=100,
+                    ),
+                ),
+                ('creation_timestamp', models.DateTimeField(default=django.utils.timezone.now)),
+                ('last_update_timestamp', models.DateTimeField(auto_now=True)),
+                ('completion_timestamp', models.DateTimeField(default=None, null=True)),
+                ('exception', models.TextField()),
+                ('action', models.CharField(max_length=100)),
+                (
+                    'application',
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to='applications.application'
+                    ),
+                ),
+                (
+                    'version',
+                    models.ForeignKey(
+                        null=True, on_delete=django.db.models.deletion.CASCADE, to='applications.version'
+                    ),
+                ),
+            ],
+        ),
+    ]

hobo/applications/models.py

@@ -17,13 +17,14 @@
 import io
 import json
 import os
+import sys
 import tarfile
 import urllib.parse
 
 from django.conf import settings
 from django.contrib.postgres.fields import JSONField
 from django.core.files.base import ContentFile
-from django.db import models
+from django.db import connection, models
 from django.utils.text import slugify
 from django.utils.timezone import now
 from django.utils.translation import ugettext_lazy as _
@@ -237,3 +238,52 @@ class Version(models.Model):
                 if not response.ok:
                     # TODO: report failures
                     continue
+
+
+class AsyncJob(models.Model):
+    label = models.CharField(max_length=100)
+    status = models.CharField(
+        max_length=100,
+        default='registered',
+        choices=[
+            ('registered', _('Registered')),
+            ('running', _('Running')),
+            ('failed', _('Failed')),
+            ('completed', _('Completed')),
+        ],
+    )
+    creation_timestamp = models.DateTimeField(default=now)
+    last_update_timestamp = models.DateTimeField(auto_now=True)
+    completion_timestamp = models.DateTimeField(default=None, null=True)
+    exception = models.TextField()
+
+    application = models.ForeignKey(Application, on_delete=models.CASCADE)
+    version = models.ForeignKey(Version, on_delete=models.CASCADE, null=True)
+    action = models.CharField(max_length=100)
+
+    raise_exception = True
+
+    def run(self, spool=False):
+        if 'uwsgi' in sys.modules and spool:
+            from hobo.applications.spooler import run_job
+
+            tenant = getattr(connection, 'tenant', None)
+            domain = getattr(tenant, 'domain_url', '')
+            run_job.spool(domain=domain.encode(), job_id=str(self.pk).encode())
+            return
+        self.status = 'running'
+        self.save()
+        try:
+            if self.action == 'scandeps':
+                self.application.scandeps()
+            elif self.action == 'create_bundle':
+                self.version.create_bundle()
+        except Exception as e:
+            if self.raise_exception:
+                raise
+            self.status = 'failed'
+            self.exception = traceback.format_exc()
+        finally:
+            self.status = 'completed'
+        self.completion_timestamp = now()
+        self.save()

hobo/applications/spooler.py

@@ -0,0 +1,31 @@
+# hobo - portal to configure and deploy applications
+# Copyright (C) 2015-2022 Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from uwsgidecorators import spool
+
+from hobo.provisionning.spooler import ensure_db, set_connection
+
+from .models import AsyncJob
+
+
+@spool
+@ensure_db
+def run_job(args):
+    set_connection(args['domain'])
+    job = AsyncJob.objects.get(id=args['job_id'])
+    job.raise_exception = False
+    job.run()
+    print('got job:', job)

hobo/applications/templates/hobo/applications/job.html

@@ -0,0 +1,30 @@
+{% extends "hobo/base.html" %}
+{% load i18n %}
+
+{% block appbar %}
+  <h2>{{ object.label }}</h2>
+{% endblock %}
+
+{% block content %}
+  {% if object.status == 'failed' %}
+    <div class="pk-error">
+      <p>{% trans "Error running the job." %}</p>
+      <p><a class="pk-button" href="{{ view.get_redirect_url }}">{% trans "Back" %}</a></p>
+    </div>
+  {% else %}
+    <p>{% trans "Please wait…" %}</p>
+  {% endif %}
+
+  <script>
+    $(function () {
+      if ("{{ object.status }}" == "completed") {
+        window.location = "{{ view.get_redirect_url }}";
+      } else {
+        setTimeout(function() {
+          window.location.reload();
+        }, 2000);
+      }
+    });
+  </script>
+
+{% endblock %}

hobo/applications/urls.py

@@ -44,4 +44,9 @@ urlpatterns = [
         views.delete_element,
         name='application-delete-element',
     ),
+    re_path(
+        r'^manifest/(?P<app_slug>[\w-]+)/job/(?P<pk>\d+)/$',
+        views.async_job,
+        name='application-async-job',
+    ),
 ]

hobo/applications/views.py

@@ -24,13 +24,14 @@ from django.core.files.base import ContentFile
 from django.http import HttpResponse, HttpResponseRedirect
 from django.shortcuts import get_object_or_404
 from django.urls import reverse, reverse_lazy
-from django.views.generic import FormView, ListView, TemplateView
+from django.utils.translation import ugettext_lazy as _
+from django.views.generic import DetailView, FormView, ListView, TemplateView
 from django.views.generic.edit import CreateView, DeleteView, UpdateView
 
 from hobo.environment.utils import get_installed_services
 
 from .forms import GenerateForm, InstallForm, MetadataForm
-from .models import Application, Element, Relation, Version
+from .models import Application, AsyncJob, Element, Relation, Version
 from .utils import Requests
 
 requests = Requests()
@@ -186,8 +187,17 @@ delete_element = AppDeleteElementView.as_view()
 
 
 def scandeps(request, app_slug):
-    app = Application.objects.get(slug=app_slug)
-    app.scandeps()
+    job = AsyncJob(
+        label=_('Scanning for dependencies'),
+        application=Application.objects.get(slug=app_slug),
+        action='scandeps',
+    )
+    job.save()
+    job.run(spool=True)
+    if job.status == 'registered':
+        return HttpResponseRedirect(
+            reverse('application-async-job', kwargs={'app_slug': app_slug, 'pk': job.id})
+        )
 
     return HttpResponseRedirect(reverse('application-manifest', kwargs={'app_slug': app_slug}))
 
@@ -217,7 +227,19 @@ class GenerateView(FormView):
         version.number = form.cleaned_data['number']
         version.notes = form.cleaned_data['notes']
         version.save()
-        version.create_bundle()
+
+        job = AsyncJob(
+            label=_('Creating application bundle'),
+            application=app,
+            version=version,
+            action='create_bundle',
+        )
+        job.save()
+        job.run(spool=True)
+        if job.status == 'registered':
+            return HttpResponseRedirect(
+                reverse('application-async-job', kwargs={'app_slug': app.slug, 'pk': job.id})
+            )
 
         return super().form_valid(form)
 
@@ -311,3 +333,14 @@ class AppDeleteView(DeleteView):
 
 
 delete = AppDeleteView.as_view()
+
+
+class AsyncJobView(DetailView):
+    model = AsyncJob
+    template_name = 'hobo/applications/job.html'
+
+    def get_redirect_url(self):
+        return reverse('application-manifest', kwargs={'app_slug': self.kwargs['app_slug']})
+
+
+async_job = AsyncJobView.as_view()