applications: keep a cache of modules progression (#89124)

hobo/applications/migrations/0016_progression.py

@@ -0,0 +1,30 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('applications', '0015_auto_20231212_0933'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='asyncjob',
+            name='progression_cache',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.AlterField(
+            model_name='asyncjob',
+            name='status',
+            field=models.CharField(
+                choices=[
+                    ('registered', 'Registered'),
+                    ('running', 'Running'),
+                    ('waiting', 'Waiting for modules'),
+                    ('failed', 'Failed'),
+                    ('completed', 'Completed'),
+                ],
+                default='registered',
+                max_length=100,
+            ),
+        ),
+    ]

hobo/applications/models.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import collections
+import copy
 import io
 import json
 import os
@@ -566,6 +567,7 @@ class AsyncJob(models.Model):
     version = models.ForeignKey(Version, on_delete=models.CASCADE, null=True)
     action = models.CharField(max_length=100)
     progression_urls = JSONField(blank=True, default=dict)
+    progression_cache = JSONField(blank=True, default=dict)
     details = JSONField(blank=True, default=dict)
 
     raise_exception = True
@@ -674,11 +676,20 @@ class AsyncJob(models.Model):
         context = {'job_progression': {}}
         for service_id, services in self.progression_urls.items():
             for service, url in services.items():
+                if service_id not in self.progression_cache:
+                    self.progression_cache[service_id] = {}
+                cache = self.progression_cache[service_id].get(service) or {}
+                if (cache.get('data') or {}).get('status') == 'completed':
+                    context['job_progression'][service] = copy.deepcopy(cache)
+                    context['job_progression'][service].update({'service_id': service_id})
+                    continue
                 response = requests.get(url)
                 if not response.ok:
                     continue
                 context['job_progression'][service] = response.json()
                 context['job_progression'][service].update({'service_id': service_id})
+                self.progression_cache[service_id][service] = response.json()
+        self.save()
         context['services_all_completed'] = all(
             [s['data']['status'] == 'completed' for s in context['job_progression'].values()]
         )

pylint.rc

@@ -55,6 +55,7 @@ disable=
     unnecessary-lambda-assignment,
     unspecified-encoding,
     unsubscriptable-object,
+    unsupported-assignment-operation,
     unsupported-binary-operation,
     unsupported-membership-test,
     unused-argument,

tests/test_application.py

@@ -1844,6 +1844,11 @@ def test_job_status_page(app, admin_user, settings):
         assert 'Running: 42%' in resp
         assert 'window.location.reload' in resp
         assert 'window.location = "/applications/manifest/test/"' not in resp
+    job.refresh_from_db()
+    assert job.progression_cache == {
+        'wcs': {'Foobar': {'err': 0, 'data': {'status': 'running', 'completion_status': '42%'}}}
+    }
+    assert job.status == 'waiting'
 
     with StatefulHTTMock(httmock.remember_called(mocked_http)):
         resp = app.get('/applications/manifest/test/job/%s/' % job.pk)
@@ -1851,15 +1856,38 @@ def test_job_status_page(app, admin_user, settings):
         assert 'Completed: 100%' in resp
         assert 'window.location.reload' not in resp
         assert 'window.location = "/applications/manifest/test/refresh/"' in resp
+    job.refresh_from_db()
+    assert job.progression_cache == {
+        'wcs': {'Foobar': {'err': 0, 'data': {'status': 'completed', 'completion_status': '100%'}}}
+    }
+    assert job.status == 'completed'
+
+    # modules completed, don't call it anymore
+    job.status = 'waiting'
+    job.save()
+    resp = app.get('/applications/manifest/test/job/%s/' % job.pk)
+    assert 'Please wait…' in resp
+    assert 'Completed: 100%' in resp
+    assert 'window.location.reload' not in resp
+    assert 'window.location = "/applications/manifest/test/refresh/"' in resp
+    job.refresh_from_db()
+    assert job.progression_cache == {
+        'wcs': {'Foobar': {'err': 0, 'data': {'status': 'completed', 'completion_status': '100%'}}}
+    }
+    assert job.status == 'completed'
 
     job.status = 'failed'
     job.exception = 'foo bar exception'
+    job.progression_cache = {}
     job.save()
     resp = app.get('/applications/manifest/test/job/%s/' % job.pk)
     assert 'Error running the job.' in resp
     assert 'Please wait…' not in resp
     assert 'window.location.reload' not in resp
     assert 'window.location = "/applications/manifest/test/"' not in resp
+    job.refresh_from_db()
+    assert job.progression_cache == {}
+    assert job.status == 'failed'
 
     job.status = 'waiting'
     job.exception = ''
@@ -1879,9 +1907,12 @@ def test_job_status_page(app, admin_user, settings):
         assert 'Please wait…' not in resp
         assert 'window.location.reload' not in resp
         assert 'window.location = "/applications/manifest/test/"' not in resp
-        job.refresh_from_db()
-        assert job.status == 'failed'
-        assert job.exception == 'Failed to deploy module wcs'
+    job.refresh_from_db()
+    assert job.progression_cache == {
+        'wcs': {'Foobar': {'err': 0, 'data': {'status': 'failed', 'completion_status': '42%'}}}
+    }
+    assert job.status == 'failed'
+    assert job.exception == 'Failed to deploy module wcs'
 
 
 def test_create_application_parameters(app, admin_user, settings):