applications: display job progress on import (#70942)

hobo/applications/migrations/0009_job_progression.py

@@ -0,0 +1,17 @@
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('applications', '0008_element_error'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='asyncjob',
+            name='progression_urls',
+            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict),
+        ),
+    ]

hobo/applications/models.py

@@ -276,7 +276,7 @@ class Version(models.Model):
         self.bundle.save('%s.tar' % app.slug, content=ContentFile(tar_io.getvalue()))
         self.save()
 
-    def deploy(self):
+    def deploy(self, job=None):
         bundle_content = self.bundle.read()
         self.deploy_roles(bundle_content)
         for service_id, services in getattr(settings, 'KNOWN_SERVICES', {}).items():
@@ -292,8 +292,18 @@ class Version(models.Model):
                     raise DeploymentError(
                         _('Failed to deploy module %s (%s)') % (service_id, response.status_code)
                     )
-                # TODO: look at response content for afterjob URLs to display a progress bar
-                pass
+                if not job:
+                    continue
+                try:
+                    response_json = response.json()
+                except json.JSONDecodeError:
+                    continue
+                if not response_json.get('url'):
+                    continue
+                if service_id not in job.progression_urls:
+                    job.progression_urls[service_id] = {}
+                job.progression_urls[service_id][service['title']] = response_json['url']
+                job.save()
 
     def get_authentic_service(self):
         for service_id, services in getattr(settings, 'KNOWN_SERVICES', {}).items():
@@ -329,17 +339,20 @@ class Version(models.Model):
                     )
 
 
+STATUS_CHOICES = [
+    ('registered', _('Registered')),
+    ('running', _('Running')),
+    ('failed', _('Failed')),
+    ('completed', _('Completed')),
+]
+
+
 class AsyncJob(models.Model):
     label = models.CharField(max_length=100)
     status = models.CharField(
         max_length=100,
         default='registered',
-        choices=[
-            ('registered', _('Registered')),
-            ('running', _('Running')),
-            ('failed', _('Failed')),
-            ('completed', _('Completed')),
-        ],
+        choices=STATUS_CHOICES,
     )
     creation_timestamp = models.DateTimeField(default=now)
     last_update_timestamp = models.DateTimeField(auto_now=True)
@@ -349,6 +362,7 @@ class AsyncJob(models.Model):
     application = models.ForeignKey(Application, on_delete=models.CASCADE)
     version = models.ForeignKey(Version, on_delete=models.CASCADE, null=True)
     action = models.CharField(max_length=100)
+    progression_urls = JSONField(blank=True, default=dict)
 
     raise_exception = True
 
@@ -368,7 +382,7 @@ class AsyncJob(models.Model):
             elif self.action == 'create_bundle':
                 self.version.create_bundle()
             elif self.action == 'deploy':
-                self.version.deploy()
+                self.version.deploy(self)
         except ApplicationError as e:
             self.status = 'failed'
             self.exception = e.msg

hobo/applications/templates/hobo/applications/job.html

@@ -13,18 +13,26 @@
       <p><a class="pk-button" href="{{ view.get_redirect_url }}">{% trans "Back" %}</a></p>
     </div>
   {% else %}
-    <p>{% trans "Please wait…" %}</p>
+    <div class="pk-information">
+      <p>{% trans "Please wait…" %}</p>
+      {% for service, progression in job_progression.items %}
+        <dl class="job-status">
+          <dt>{{ service }}</dt>
+          <dd>{{ service_job_status_choices|get:progression.data.status }}: {{ progression.data.completion_status|default:'' }}</dd>
+        </dl>
+      {% endfor %}
+    </div>
   {% endif %}
 
   <script>
     $(function () {
-      if ("{{ object.status }}" == "completed") {
+      {% if object.status == 'completed' and wait_for_services and services_all_completed or object.status == 'completed' and not wait_for_services %}
         window.location = "{{ view.get_redirect_url }}";
-      } else if ("{{ object.status }}" != "failed") {
+      {% elif object.status != 'failed' %}
         setTimeout(function() {
           window.location.reload();
         }, 2000);
-      }
+      {% endif %}
     });
   </script>
 

hobo/applications/views.py

@@ -25,12 +25,13 @@ from django.http import HttpResponse, HttpResponseRedirect
 from django.shortcuts import get_object_or_404
 from django.urls import reverse, reverse_lazy
 from django.utils.text import slugify
+from django.utils.timezone import now
 from django.utils.translation import ugettext_lazy as _
 from django.views.generic import DetailView, FormView, ListView, TemplateView
 from django.views.generic.edit import CreateView, DeleteView, UpdateView
 
 from .forms import GenerateForm, InstallForm, MetadataForm
-from .models import Application, AsyncJob, Element, Relation, Version, get_object_types
+from .models import STATUS_CHOICES, Application, AsyncJob, Element, Relation, Version, get_object_types
 from .utils import Requests
 
 requests = Requests()
@@ -372,6 +373,35 @@ class AsyncJobView(DetailView):
     model = AsyncJob
     template_name = 'hobo/applications/job.html'
 
+    def get_context_data(self, **kwargs):
+        kwargs['job_progression'] = {}
+        job = self.object
+        if job.action == 'deploy' and job.status != 'failed':
+            for service_id, services in job.progression_urls.items():
+                for service, url in services.items():
+                    response = requests.get(url)
+                    if not response.ok:
+                        continue
+                    kwargs['job_progression'][service] = response.json()
+                    kwargs['job_progression'][service].update({'service_id': service_id})
+            kwargs['services_all_completed'] = all(
+                [s['data']['status'] == 'completed' for s in kwargs['job_progression'].values()]
+            )
+            failed = [
+                s['service_id'] for s in kwargs['job_progression'].values() if s['data']['status'] == 'failed'
+            ]
+            if failed:
+                job.status = 'failed'
+                if len(failed) > 1:
+                    job.exception = 'Failed to deploy modules %s' % ', '.join(failed)
+                else:
+                    job.exception = 'Failed to deploy module %s' % failed[0]
+                job.completion_timestamp = now()
+                job.save()
+            kwargs['wait_for_services'] = True
+            kwargs['service_job_status_choices'] = {c[0]: c[1] for c in STATUS_CHOICES}
+        return super().get_context_data(**kwargs)
+
     def get_redirect_url(self):
         return reverse('application-manifest', kwargs={'app_slug': self.kwargs['app_slug']})
 

hobo/settings.py

@@ -89,6 +89,10 @@ TEMPLATES = [
                 'django.contrib.messages.context_processors.messages',
                 'hobo.context_processors.template_vars',
                 'hobo.context_processors.hobo_json',
+                'publik_django_templatetags.wcs.context_processors.cards',
+            ],
+            'builtins': [
+                'publik_django_templatetags.publik.templatetags.publik',
             ],
         },
     },

tests/test_application.py

@@ -187,7 +187,10 @@ def mocked_http(url, request):
         return {'content': '<carddef/>', 'status_code': 200, 'headers': {'content-length': '10'}}
 
     if url.path == '/api/export-import/bundle-import/':
-        return {'content': '{}', 'status_code': 200}
+        return {
+            'content': json.dumps({'err': 0, 'url': 'https://wcs.example.invalid/api/jobs/job-uuid/'}),
+            'status_code': 200,
+        }
 
     return {'content': json.dumps({'data': []}), 'status_code': 200}
 
@@ -818,6 +821,9 @@ def test_deploy_application(app, admin_user, settings, app_bundle, app_bundle_wi
             assert version.number == '42.0'
         assert version.notes == 'foo bar blah'
         assert application.elements.count() == 3
+        job = AsyncJob.objects.latest('pk')
+        assert job.status == 'completed'
+        assert job.progression_urls == {'wcs': {'Foobar': 'https://wcs.example.invalid/api/jobs/job-uuid/'}}
         return version
 
     resp = app.get('/applications/')
@@ -971,3 +977,113 @@ def test_deploy_application_roles(app, admin_user, settings, app_bundle_roles):
         job = AsyncJob.objects.latest('pk')
         assert job.status == 'failed'
         assert job.exception == 'Failed to provision role test-role (500)'
+
+
+def test_job_status_page(app, admin_user, settings):
+    Wcs.objects.create(base_url='https://wcs.example.invalid', slug='foobar', title='Foobar')
+    application = Application.objects.create(name='Test', slug='test')
+
+    settings.KNOWN_SERVICES = {
+        'wcs': {
+            'foobar': {
+                'title': 'Foobar',
+                'url': 'https://wcs.example.invalid/',
+                'orig': 'example.org',
+                'secret': 'xxx',
+            }
+        }
+    }
+
+    login(app)
+
+    for action in ['scandeps', 'create_bundle']:
+        job = AsyncJob.objects.create(
+            label=action,
+            application=application,
+            action=action,
+            status='running',
+        )
+        resp = app.get('/applications/manifest/test/job/%s/' % job.pk)
+        assert 'Please wait…' in resp
+        assert 'window.location.reload' in resp
+        assert 'window.location = "/applications/manifest/test/"' not in resp
+
+        job.status = 'failed'
+        job.exception = 'foo bar exception'
+        job.save()
+        resp = app.get('/applications/manifest/test/job/%s/' % job.pk)
+        assert 'Error running the job.' in resp
+        assert 'Please wait…' not in resp
+        assert 'window.location.reload' not in resp
+        assert 'window.location = "/applications/manifest/test/"' not in resp
+
+    job = AsyncJob.objects.create(
+        label='deploy',
+        application=application,
+        action='deploy',
+        progression_urls={'wcs': {'Foobar': 'https://wcs.example.invalid/api/jobs/job-uuid/'}},
+        status='completed',  # completed, with async job on wcs part
+    )
+
+    def response_content(url, request):
+        if url.path.startswith('/api/jobs/'):
+            return {
+                'content': json.dumps({'err': 0, 'data': {'status': 'running', 'completion_status': '42%'}}),
+                'status_code': 200,
+            }
+        return mocked_http(url, request)
+
+    with HTTMock(response_content):
+        resp = app.get('/applications/manifest/test/job/%s/' % job.pk)
+        assert 'Please wait…' in resp
+        assert 'Running: 42%' in resp
+        assert 'window.location.reload' in resp
+        assert 'window.location = "/applications/manifest/test/"' not in resp
+
+    def response_content(url, request):
+        if url.path.startswith('/api/jobs/'):
+            return {
+                'content': json.dumps(
+                    {'err': 0, 'data': {'status': 'completed', 'completion_status': '100%'}}
+                ),
+                'status_code': 200,
+            }
+        return mocked_http(url, request)
+
+    with HTTMock(response_content):
+        resp = app.get('/applications/manifest/test/job/%s/' % job.pk)
+        assert 'Please wait…' in resp
+        assert 'Completed: 100%' in resp
+        assert 'window.location.reload' not in resp
+        assert 'window.location = "/applications/manifest/test/"' in resp
+
+    job.status = 'failed'
+    job.exception = 'foo bar exception'
+    job.save()
+    resp = app.get('/applications/manifest/test/job/%s/' % job.pk)
+    assert 'Error running the job.' in resp
+    assert 'Please wait…' not in resp
+    assert 'window.location.reload' not in resp
+    assert 'window.location = "/applications/manifest/test/"' not in resp
+
+    job.status = 'running'
+    job.exception = ''
+    job.save()
+
+    def response_content(url, request):
+        if url.path.startswith('/api/jobs/'):
+            return {
+                'content': json.dumps({'err': 0, 'data': {'status': 'failed', 'completion_status': '42%'}}),
+                'status_code': 200,
+            }
+        return mocked_http(url, request)
+
+    with HTTMock(response_content):
+        resp = app.get('/applications/manifest/test/job/%s/' % job.pk)
+        assert 'Error running the job.' in resp
+        assert 'Please wait…' not in resp
+        assert 'window.location.reload' not in resp
+        assert 'window.location = "/applications/manifest/test/"' not in resp
+        job.refresh_from_db()
+        assert job.status == 'failed'
+        assert job.exception == 'Failed to deploy module wcs'

tox.ini

@@ -68,6 +68,7 @@ deps:
 	sorl-thumbnail
     Pillow
 	black: pre-commit
+	git+https://git.entrouvert.org/publik-django-templatetags.git
 commands =
 	./getlasso3.sh
 	hobo: py.test {env:JUNIT:} {env:COVERAGE:} {env:NOMIGRATIONS:} {posargs:tests/}