applications: refresh elements cache on deployment (#70897)

hobo/applications/models.py

@@ -106,11 +106,17 @@ class Application(models.Model):
             self.slug = slug
         super().save(*args, **kwargs)
 
-    def refresh_elements(self):
-        self.relation_set.filter(auto_dependency=True).delete()
+    def refresh_elements(self, cache_only=False):
+        if not cache_only:
+            self.relation_set.filter(auto_dependency=True).delete()
         remote_elements = {}
+        relations = self.relation_set.select_related('element')
+        elements = {(x.element.type, x.element.slug): (x.element, x) for x in relations}
+        current_object_types = {t for t, s in elements}
         for object_type in get_object_types():
-            if object_type.get('minor'):
+            if object_type['id'] not in current_object_types:
+                continue
+            if not cache_only and object_type.get('minor'):
                 continue
             url = object_type['urls']['list']
             response = requests.get(url)
@@ -119,16 +125,23 @@ class Application(models.Model):
                     _('Failed to get elements of type %s (%s)' % (object_type['id'], response.status_code))
                 )
             remote_elements[object_type['id']] = {x['id']: x for x in response.json()['data']}
-        relations = self.relation_set.select_related('element')
-        elements = {(x.element.type, x.element.slug): (x.element, x) for x in relations}
         for element, relation in elements.values():
-            if remote_elements[element.type].get(element.slug):
-                remote_element = remote_elements[element.type][element.slug]
-                if element.name != remote_element['text'] or element.cache != remote_element:
-                    element.name = remote_element['text']
-                    element.cache = remote_element
-                    element.save()
-                    elements[(element.type, element.slug)] = (element, relation)
+            if not remote_elements[element.type].get(element.slug):
+                continue
+            remote_element = remote_elements[element.type][element.slug]
+            if cache_only:
+                if element.cache == remote_element:
+                    continue
+                element.cache = remote_element
+                element.save()
+                elements[(element.type, element.slug)] = (element, relation)
+                continue
+            if element.name == remote_element['text'] and element.cache == remote_element:
+                continue
+            element.name = remote_element['text']
+            element.cache = remote_element
+            element.save()
+            elements[(element.type, element.slug)] = (element, relation)
         return elements
 
     def scandeps(self):
@@ -304,6 +317,7 @@ class Version(models.Model):
                     job.progression_urls[service_id] = {}
                 job.progression_urls[service_id][service['title']] = response_json['url']
                 job.save()
+        self.application.refresh_elements(cache_only=True)
 
     def get_authentic_service(self):
         for service_id, services in getattr(settings, 'KNOWN_SERVICES', {}).items():

tests/test_application.py

@@ -875,6 +875,59 @@ def test_deploy_application(app, admin_user, settings, app_bundle, app_bundle_wi
         assert job.status == 'failed'
         assert job.exception == 'Failed to deploy module wcs (500)'
 
+    form_def = {
+        "id": "test",
+        "text": "Test",
+        "type": "forms",
+        "urls": {
+            "export": "https://wcs.example.invalid/api/export-import/forms/test/",
+            "dependencies": "https://wcs.example.invalid/api/export-import/forms/test/dependencies/",
+            "redirect": "https://wcs.example.invalid/api/export-import/forms/test/redirect/",
+        },
+    }
+
+    def response_content(url, request):
+        if url.path == '/api/export-import/forms/':
+            return {
+                'content': {"data": [form_def]},
+                'status_code': 200,
+            }
+        return mocked_http(url, request)
+
+    resp = app.get('/applications/')
+    if action == 'Update':
+        with HTTMock(mocked_http):
+            resp = resp.click(href='manifest/test/')
+    resp = resp.click(action)
+    resp.form['bundle'] = Upload('app.tar', app_bundle, 'application/x-tar')
+    with HTTMock(response_content):
+        resp.form.submit()
+        application = Application.objects.get(slug='test')
+        elements = application.elements.all().order_by('type')
+        assert len(elements) == 3
+        assert elements[0].cache == {}
+        assert elements[1].cache == form_def
+        assert elements[2].cache == {}
+
+    def response_content(url, request):
+        if url.path == '/api/export-import/forms/':
+            return {'status_code': 500}
+        return mocked_http(url, request)
+
+    resp = app.get('/applications/')
+    if action == 'Update':
+        with HTTMock(mocked_http):
+            resp = resp.click(href='manifest/test/')
+    resp = resp.click(action)
+    resp.form['bundle'] = Upload('app.tar', app_bundle, 'application/x-tar')
+    with HTTMock(response_content):
+        with pytest.raises(ScanError) as e:
+            resp.form.submit()
+        assert str(e.value) == 'Failed to get elements of type forms (500)'
+        job = AsyncJob.objects.latest('pk')
+        assert job.status == 'failed'
+        assert job.exception == 'Failed to get elements of type forms (500)'
+
 
 def test_update_application(app, admin_user, settings, app_bundle):
     Wcs.objects.create(base_url='https://wcs.example.invalid', slug='foobar', title='Foobar')
@@ -987,7 +1040,16 @@ def test_deploy_application_roles(app, admin_user, settings, app_bundle_roles):
             )
             # then form
             assert 'wcs.example.invalid' in mocked_http.call['requests'][4].url
-            assert mocked_http.call['count'] == 5
+            assert mocked_http.call['requests'][5].url.startswith(
+                'https://wcs.example.invalid/api/export-import/?'
+            )
+            # then element refresh
+            available_objects = [o for o in WCS_AVAILABLE_OBJECTS['data'] if o['id'] in ['forms', 'roles']]
+            assert mocked_http.call['count'] == 5 + 1 + len(available_objects)
+            for i, object_type in enumerate(available_objects):
+                assert mocked_http.call['requests'][6 + i].url.startswith(
+                    'https://wcs.example.invalid/api/export-import/%s/?' % object_type['id']
+                )
 
     def response_content(url, request):
         if url.path == '/api/roles/':