cells: w.c.s. cells invalid report (#38009)

combo/apps/wcs/__init__.py

@@ -65,6 +65,14 @@ class AppConfig(django.apps.AppConfig):
         return urls.urlpatterns
 
     def hourly(self):
+        from combo.data.library import get_cell_classes
+        from combo.data.models import CellBase
+
+        cell_classes = [c for c in self.get_models() if c in get_cell_classes()]
+        for cell in CellBase.get_cells(cell_filter=lambda x: x in cell_classes, page__snapshot__isnull=True):
+            if hasattr(cell, 'check_validity'):
+                cell.check_validity()
+
         self.update_db_cache()
 
     def update_db_cache(self):
@@ -74,4 +82,5 @@ class AppConfig(django.apps.AppConfig):
                 cell_filter=lambda x: x in (WcsFormCell, WcsCategoryCell, WcsFormsOfCategoryCell)):
             cell.save()
 
+
 default_app_config = 'combo.apps.wcs.AppConfig'

combo/apps/wcs/models.py

@@ -19,7 +19,6 @@
 import copy
 import logging
 
-from django import template
 from django.conf import settings
 from django.db import models
 from django.forms import models as model_forms
@@ -34,6 +33,16 @@ from combo.utils import requests
 
 from .utils import get_wcs_json, is_wcs_enabled, get_wcs_services
 
+
+invalid_reason_codes = {
+    'wcs_form_not_defined': _('No form set'),
+    'wcs_form_not_found': _('Invalid form'),
+    'wcs_category_not_defined': _('No category set'),
+    'wcs_category_not_found': _('Invalid category'),
+    'wcs_data_failure': _('Failed to get data'),
+}
+
+
 @register_cell_class
 class WcsFormCell(CellBase):
     formdef_reference = models.CharField(_('Form'), max_length=150)
@@ -47,6 +56,7 @@ class WcsFormCell(CellBase):
     add_link_label = _('New form link')
     edit_link_label = _('Edit form link')
     add_as_link_code = 'form-link'
+    invalid_reason_codes = invalid_reason_codes
 
     is_enabled = classmethod(is_wcs_enabled)
 
@@ -58,20 +68,49 @@ class WcsFormCell(CellBase):
         return WcsFormCellForm
 
     def save(self, *args, **kwargs):
-        if self.formdef_reference:
+        if 'update_fields' in kwargs:
-            wcs_key, form_slug = self.formdef_reference.split(':')
+            # don't populate the cache
-            wcs_site = get_wcs_services().get(wcs_key)
+            return super(WcsFormCell, self).save(*args, **kwargs)
-            forms_response_json = get_wcs_json(wcs_site, 'api/formdefs/')
+
-            if isinstance(forms_response_json, dict):
+        def populate_cache():
-                # forward compability with future w.c.s. API
+            if self.formdef_reference:
-                forms_response_json = forms_response_json.get('data')
+                wcs_key, form_slug = self.formdef_reference.split(':')
-            for form in forms_response_json:
+                wcs_site = get_wcs_services().get(wcs_key)
-                slug = form.get('slug')
+                forms_response_json = get_wcs_json(wcs_site, 'api/formdefs/')
-                if slug == form_slug:
+
-                    self.cached_title = form.get('title')
+                if not forms_response_json:
-                    self.cached_url = form.get('url')
+                    # can not retrieve data, don't report cell as invalid
-                    self.cached_json = form
+                    self.mark_as_valid()
-        return super(WcsFormCell, self).save(*args, **kwargs)
+                    return
+
+                if isinstance(forms_response_json, dict):
+                    # forward compability with future w.c.s. API
+                    forms_response_json = forms_response_json.get('data')
+
+                form_found = False
+                for form in forms_response_json or []:
+                    slug = form.get('slug')
+                    if slug == form_slug:
+                        self.cached_title = form.get('title')
+                        self.cached_url = form.get('url')
+                        self.cached_json = form
+                        form_found = True
+                        break
+                if form_found:
+                    self.mark_as_valid()
+                    return
+                else:
+                    return self.mark_as_invalid('wcs_form_not_found', save=False)
+            else:
+                return self.mark_as_invalid('wcs_form_not_defined', save=False)
+
+        validity_info = populate_cache()
+        result = super(WcsFormCell, self).save(*args, **kwargs)
+        if validity_info:
+            # save validity_info after Cell insert
+            validity_info.content_object = self
+            validity_info.save()
+        return result
 
     def get_cell_extra_context(self, context):
         context = super(WcsFormCell, self).get_cell_extra_context(context)
@@ -129,21 +168,51 @@ class WcsCommonCategoryCell(CellBase):
     cached_description = models.TextField(_('Description'), blank=True)
     cached_url = models.URLField(_('Cached URL'))
 
+    invalid_reason_codes = invalid_reason_codes
+
     class Meta:
         abstract = True
 
     def save(self, *args, **kwargs):
-        if self.category_reference:
+        if 'update_fields' in kwargs:
-            wcs_key, category_slug = self.category_reference.split(':')
+            # don't populate the cache
-            wcs_site = get_wcs_services().get(wcs_key)
+            return super(WcsCommonCategoryCell, self).save(*args, **kwargs)
-            categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
+
-            for category in categories_response_json.get('data'):
+        def populate_cache():
-                slug = category.get('slug')
+            if self.category_reference:
-                if slug == category_slug:
+                wcs_key, category_slug = self.category_reference.split(':')
-                    self.cached_title = category.get('title')
+                wcs_site = get_wcs_services().get(wcs_key)
-                    self.cached_description = category.get('description') or ''
+                categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
-                    self.cached_url = category.get('url')
+
-        return super(WcsCommonCategoryCell, self).save(*args, **kwargs)
+                if not categories_response_json:
+                    # can not retrieve data, don't report cell as invalid
+                    self.mark_as_valid()
+                    return
+
+                category_found = False
+                for category in categories_response_json.get('data', []):
+                    slug = category.get('slug')
+                    if slug == category_slug:
+                        self.cached_title = category.get('title')
+                        self.cached_description = category.get('description') or ''
+                        self.cached_url = category.get('url')
+                        category_found = True
+                        break
+                if category_found:
+                    self.mark_as_valid()
+                    return
+                else:
+                    return self.mark_as_invalid('wcs_category_not_found', save=False)
+            else:
+                return self.mark_as_invalid('wcs_category_not_defined', save=False)
+
+        validity_info = populate_cache()
+        result = super(WcsCommonCategoryCell, self).save(*args, **kwargs)
+        if validity_info:
+            # save validity_info after Cell insert
+            validity_info.content_object = self
+            validity_info.save()
+        return result
 
     def get_additional_label(self):
         if not self.cached_title:
@@ -192,7 +261,8 @@ class WcsBlurpMixin(object):
     is_enabled = classmethod(is_wcs_enabled)
     cache_duration = 5
     api_url = None
-    warn_on_404 = True
+    warn_on_4xx = True
+    invalid_reason_codes = invalid_reason_codes
 
     def get_api_url(self, context):
         return self.api_url
@@ -242,9 +312,15 @@ class WcsBlurpMixin(object):
                 for item in wcs_site['data']:
                     item['site_slug'] = wcs_slug
 
-        if not 200 in returns: # not a single valid answer
+        if 200 not in returns:  # not a single valid answer
-            if returns != set([404]) or self.warn_on_404:
+            logging.debug('failed to get data from any %s (%r)', api_url, returns)
-                logging.error('failed to get data from any %s (%r)', api_url, returns)
+            if all([400 <= r < 500 for r in returns]) and self.warn_on_4xx:
+                # only 4xx errors, not a user cell, report the cell as invalid
+                self.mark_as_invalid('wcs_data_failure', force=False)
+            else:
+                self.mark_as_valid()
+        else:
+            self.mark_as_valid()
 
         return wcs_sites
 
@@ -293,7 +369,7 @@ class WcsDataBaseCell(CellBase, WcsBlurpMixin):
 
 
 class WcsUserDataBaseCell(WcsDataBaseCell):
-    warn_on_404 = False
+    warn_on_4xx = False
     user_dependant = True
 
     class Meta:
@@ -305,8 +381,34 @@ class WcsUserDataBaseCell(WcsDataBaseCell):
         return super(WcsUserDataBaseCell, self).is_visible(user)
 
 
+class CategoriesValidityMixin(object):
+    def check_validity(self):
+        categories = self.categories.get('data', [])
+
+        if not categories:
+            return
+
+        for category_reference in categories:
+            wcs_key, category_slug = category_reference.split(':')
+            wcs_site = get_wcs_services().get(wcs_key)
+            categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
+
+            if not categories_response_json:
+                # can not retrieve data, don't report cell as invalid
+                continue
+
+            category_found = any(
+                [c.get('slug') == category_slug
+                 for c in categories_response_json.get('data', [])])
+            if not category_found:
+                self.mark_as_invalid('wcs_category_not_found')
+                return
+
+        self.mark_as_valid()
+
+
 @register_cell_class
-class WcsCurrentFormsCell(WcsUserDataBaseCell):
+class WcsCurrentFormsCell(WcsUserDataBaseCell, CategoriesValidityMixin):
     variable_name = 'user_forms'
 
     categories = JSONField(_('Categories'), blank=True)
@@ -400,8 +502,9 @@ class WcsCurrentFormsCell(WcsUserDataBaseCell):
 
         return context
 
+
 @register_cell_class
-class WcsCurrentDraftsCell(WcsUserDataBaseCell):
+class WcsCurrentDraftsCell(WcsUserDataBaseCell, CategoriesValidityMixin):
     variable_name = 'current_drafts'
     template_name = 'combo/wcs/current_drafts.html'
 

tests/test_wcs.py

@@ -22,7 +22,8 @@ from django.utils.six.moves.urllib import parse as urlparse
 
 import mock
 
-from combo.data.models import CellBase, LinkListCell, Page
+from combo.data.library import get_cell_classes
+from combo.data.models import CellBase, LinkListCell, Page, ValidityInfo
 from combo.apps.search.engines import engines
 from combo.apps.wcs.models import (WcsFormCell, WcsCurrentFormsCell,
         WcsFormsOfCategoryCell, WcsCurrentDraftsCell, WcsCategoryCell,
@@ -299,6 +300,33 @@ def test_form_cell_save_cache():
     appconfig.update_db_cache()
     assert WcsFormCell.objects.get(id=cell.id).cached_title == 'form title'
 
+
+@wcs_present
+def test_form_cell_validity():
+    page = Page.objects.create(title='xxx', slug='test_form_cell_save_cache', template_name='standard')
+    cell = WcsFormCell.objects.create(page=page, placeholder='content', order=0)
+    validity_info = ValidityInfo.objects.latest('pk')
+    assert validity_info.invalid_reason_code == 'wcs_form_not_defined'
+    assert validity_info.invalid_since is not None
+
+    cell.formdef_reference = u'default:form-title'
+    cell.save()
+    assert ValidityInfo.objects.exists() is False
+
+    # can not retrieve formdefs, don't set cell as invalid
+    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
+        mock_json = mock.Mock(status_code=404)
+        requests_get.return_value = mock_json
+        cell.save()
+    assert ValidityInfo.objects.exists() is False
+
+    cell.formdef_reference = u'default:foobar'
+    cell.save()
+    validity_info = ValidityInfo.objects.latest('pk')
+    assert validity_info.invalid_reason_code == 'wcs_form_not_found'
+    assert validity_info.invalid_since is not None
+
+
 @wcs_present
 def test_form_cell_load():
     page = Page(title='xxx', slug='test_form_cell_save_cache', template_name='standard')
@@ -328,6 +356,33 @@ def test_category_cell_save_cache():
     assert cell.cached_title == 'Test 3'
     assert cell.get_additional_label() == 'Test 3'
 
+
+@wcs_present
+def test_category_cell_validity():
+    page = Page.objects.create(title='xxx', slug='test_category_cell_save_cache', template_name='standard')
+    cell = WcsCategoryCell.objects.create(page=page, placeholder='content', order=0)
+    validity_info = ValidityInfo.objects.latest('pk')
+    assert validity_info.invalid_reason_code == 'wcs_category_not_defined'
+    assert validity_info.invalid_since is not None
+
+    cell.category_reference = u'default:test-3'
+    cell.save()
+    assert ValidityInfo.objects.exists() is False
+
+    # can not retrieve categories, don't set cell as invalid
+    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
+        mock_json = mock.Mock(status_code=404)
+        requests_get.return_value = mock_json
+        cell.save()
+    assert ValidityInfo.objects.exists() is False
+
+    cell.category_reference = 'default:foobar'
+    cell.save()
+    validity_info = ValidityInfo.objects.latest('pk')
+    assert validity_info.invalid_reason_code == 'wcs_category_not_found'
+    assert validity_info.invalid_since is not None
+
+
 @wcs_present
 def test_form_cell_render():
     page = Page(title='xxx', slug='test_form_cell_render', template_name='standard')
@@ -426,15 +481,67 @@ def test_current_forms_cell_render(context):
     # check include drafts
     cell.include_drafts = False
     with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
+        mock_json = mock.Mock(status_code=200)
+        requests_get.return_value = mock_json
         cell.get_cell_extra_context(context)
     assert requests_get.call_args_list[0][0][0] == '/api/user/forms?limit=100&sort=desc'
 
     cell.include_drafts = True
     with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
+        mock_json = mock.Mock(status_code=200)
+        requests_get.return_value = mock_json
         cell.get_cell_extra_context(context)
     assert requests_get.call_args_list[0][0][0] == '/api/user/forms?limit=100&sort=desc&include-drafts=on'
 
 
+@wcs_present
+def test_current_forms_cell_validity(context):
+    page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
+    cell = WcsCurrentFormsCell.objects.create(page=page, placeholder='content', order=0)
+    context['request'].user = MockUser()
+    context['synchronous'] = True  # to get fresh content
+
+    cell.get_data(context)
+    assert ValidityInfo.objects.exists() is False
+
+    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
+        mock_json = mock.Mock(status_code=404)
+        requests_get.return_value = mock_json
+        cell.get_data(context)
+    assert ValidityInfo.objects.exists() is False
+
+
+@wcs_present
+def test_current_forms_cell_check_validity(context):
+    page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
+    cell = WcsCurrentFormsCell.objects.create(page=page, placeholder='content', order=0)
+
+    # no category
+    cell.check_validity()
+    assert ValidityInfo.objects.exists() is False
+
+    # valid categories
+    cell.categories = {'data': ['default:test-3', 'default:test-9']}
+    cell.save()
+    cell.check_validity()
+    assert ValidityInfo.objects.exists() is False
+
+    # can not retrieve categories, don't set cell as invalid
+    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
+        mock_json = mock.Mock(status_code=404)
+        requests_get.return_value = mock_json
+        cell.check_validity()
+    assert ValidityInfo.objects.exists() is False
+
+    # invalid category
+    cell.categories = {'data': ['default:foobar', 'default:test-9']}
+    cell.save()
+    cell.check_validity()
+    validity_info = ValidityInfo.objects.latest('pk')
+    assert validity_info.invalid_reason_code == 'wcs_category_not_found'
+    assert validity_info.invalid_since is not None
+
+
 @wcs_present
 def test_current_forms_cell_render_single_site(context):
     page = Page(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
@@ -542,6 +649,33 @@ def test_care_forms_cell_render(context):
     assert 'other' in data
 
 
+@wcs_present
+def test_care_forms_cell_validity(context):
+    page = Page.objects.create(title='xxx', slug='test_care_forms_cell_render', template_name='standard')
+    cell = WcsCareFormsCell.objects.create(page=page, placeholder='content', order=0)
+    context['synchronous'] = True  # to get fresh content
+    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
+        mock_json = mock.Mock(status_code=200)
+        requests_get.return_value = mock_json
+        cell.get_data(context)
+    assert ValidityInfo.objects.exists() is False
+
+    # can not retrieve data, don't set cell as invalid
+    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
+        mock_json = mock.Mock(status_code=500)
+        requests_get.return_value = mock_json
+        cell.get_data(context)
+    assert ValidityInfo.objects.exists() is False
+
+    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
+        mock_json = mock.Mock(status_code=404)
+        requests_get.return_value = mock_json
+        cell.get_data(context)
+    validity_info = ValidityInfo.objects.latest('pk')
+    assert validity_info.invalid_reason_code == 'wcs_data_failure'
+    assert validity_info.invalid_since is not None
+
+
 @wcs_present
 def test_care_forms_cell_render_single_site(context):
     page = Page(title='xxx', slug='test_care_forms_cell_render', template_name='standard')
@@ -579,6 +713,7 @@ def test_forms_of_category_cell_setup():
             (u'other:test-3', u'test2 : Test 3'),
             (u'other:test-9', u'test2 : Test 9')]
 
+
 @wcs_present
 def test_forms_of_category_cell_render(context):
     page = Page(title='xxx', slug='test_forms_of_category_cell_render', template_name='standard')
@@ -625,6 +760,48 @@ def test_forms_of_category_cell_render(context):
     assert cell.render_for_search() == ''
     assert len(list(cell.get_external_links_data())) == 2
 
+
+@wcs_present
+def test_forms_of_category_cell_validity(context):
+    page = Page.objects.create(title='xxx', slug='test_forms_of_category_cell_render', template_name='standard')
+    cell = WcsFormsOfCategoryCell.objects.create(page=page, placeholder='content', order=0)
+    validity_info = ValidityInfo.objects.latest('pk')
+    assert validity_info.invalid_reason_code == 'wcs_category_not_defined'
+    assert validity_info.invalid_since is not None
+    cell.category_reference = 'default:test-9'
+    cell.save()
+    assert ValidityInfo.objects.exists() is False
+
+    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
+        mock_json = mock.Mock(status_code=404)
+        requests_get.return_value = mock_json
+        cell.render(context)
+    validity_info = ValidityInfo.objects.latest('pk')
+    assert validity_info.invalid_reason_code == 'wcs_data_failure'
+    assert validity_info.invalid_since is not None
+    cell.mark_as_valid()
+
+    # can not retrieve data, don't set cell as invalid
+    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
+        mock_json = mock.Mock(status_code=500)
+        requests_get.return_value = mock_json
+        cell.render(context)
+    assert ValidityInfo.objects.exists() is False
+
+    # can not retrieve categories, don't set cell as invalid
+    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
+        mock_json = mock.Mock(status_code=404)
+        requests_get.return_value = mock_json
+        cell.save()
+    assert ValidityInfo.objects.exists() is False
+
+    cell.category_reference = 'default:foobar'
+    cell.save()
+    validity_info = ValidityInfo.objects.latest('pk')
+    assert validity_info.invalid_reason_code == 'wcs_category_not_found'
+    assert validity_info.invalid_since is not None
+
+
 @wcs_present
 def test_current_drafts_cell_render_unlogged(context):
     page = Page(title='xxx', slug='test_current_drafts_cell_render', template_name='standard')
@@ -666,6 +843,37 @@ def test_current_drafts_cell_render_logged_in(context):
     assert len(extra_context['drafts']) == 0
 
 
+@wcs_present
+def test_current_drafts_cell_check_validity(context):
+    page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
+    cell = WcsCurrentDraftsCell.objects.create(page=page, placeholder='content', order=0)
+
+    # no category
+    cell.check_validity()
+    assert ValidityInfo.objects.exists() is False
+
+    # valid categories
+    cell.categories = {'data': ['default:test-3', 'default:test-9']}
+    cell.save()
+    cell.check_validity()
+    assert ValidityInfo.objects.exists() is False
+
+    # can not retrieve vategories, don't set cell as invalid
+    with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
+        mock_json = mock.Mock(status_code=404)
+        requests_get.return_value = mock_json
+        cell.check_validity()
+    assert ValidityInfo.objects.exists() is False
+
+    # invalid category
+    cell.categories = {'data': ['default:foobar', 'default:test-9']}
+    cell.save()
+    cell.check_validity()
+    validity_info = ValidityInfo.objects.latest('pk')
+    assert validity_info.invalid_reason_code == 'wcs_category_not_found'
+    assert validity_info.invalid_since is not None
+
+
 @wcs_present
 def test_manager_forms_of_category_cell(app, admin_user):
     Page.objects.all().delete()
@@ -971,6 +1179,7 @@ def test_manager_add_edit_delete_list_link_item(app, admin_user):
     assert WcsFormCell.objects.count() == 0
 
 
+@wcs_present
 def test_import_export_pages_with_links():
     page = Page(title=u'bar', slug='bar', order=1)
     page.save()
@@ -998,3 +1207,19 @@ def test_import_export_pages_with_links():
     assert new_item.cached_title == item.cached_title
     assert new_item.cached_url == item.cached_url
     assert new_item.cached_json == item.cached_json
+
+
+@wcs_present
+def test_hourly():
+    appconfig = apps.get_app_config('wcs')
+    page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
+    cell_classes = [c for c in appconfig.get_models() if c in get_cell_classes()]
+    for klass in cell_classes:
+        klass.objects.create(page=page, placeholder='content', order=0)
+    for klass in cell_classes:
+        if klass in [WcsCurrentFormsCell, WcsCurrentDraftsCell]:
+            with mock.patch('combo.apps.wcs.models.%s.check_validity' % klass.__name__) as check_validity:
+                appconfig.hourly()
+            assert check_validity.call_args_list == [mock.call()]
+        else:
+            assert hasattr(klass, 'check_validity') is False