cells: w.c.s. cells invalid report (#38009)

This commit is contained in:
Lauréline Guérin 2020-02-17 16:15:21 +01:00
parent 6fa8bf883a
commit 7046049aa0
No known key found for this signature in database
GPG Key ID: 1FAB9B9B4F93D473
3 changed files with 371 additions and 34 deletions

View File

@ -65,6 +65,14 @@ class AppConfig(django.apps.AppConfig):
return urls.urlpatterns
def hourly(self):
from combo.data.library import get_cell_classes
from combo.data.models import CellBase
cell_classes = [c for c in self.get_models() if c in get_cell_classes()]
for cell in CellBase.get_cells(cell_filter=lambda x: x in cell_classes, page__snapshot__isnull=True):
if hasattr(cell, 'check_validity'):
cell.check_validity()
self.update_db_cache()
def update_db_cache(self):
@ -74,4 +82,5 @@ class AppConfig(django.apps.AppConfig):
cell_filter=lambda x: x in (WcsFormCell, WcsCategoryCell, WcsFormsOfCategoryCell)):
cell.save()
default_app_config = 'combo.apps.wcs.AppConfig'

View File

@ -19,7 +19,6 @@
import copy
import logging
from django import template
from django.conf import settings
from django.db import models
from django.forms import models as model_forms
@ -34,6 +33,16 @@ from combo.utils import requests
from .utils import get_wcs_json, is_wcs_enabled, get_wcs_services
invalid_reason_codes = {
'wcs_form_not_defined': _('No form set'),
'wcs_form_not_found': _('Invalid form'),
'wcs_category_not_defined': _('No category set'),
'wcs_category_not_found': _('Invalid category'),
'wcs_data_failure': _('Failed to get data'),
}
@register_cell_class
class WcsFormCell(CellBase):
formdef_reference = models.CharField(_('Form'), max_length=150)
@ -47,6 +56,7 @@ class WcsFormCell(CellBase):
add_link_label = _('New form link')
edit_link_label = _('Edit form link')
add_as_link_code = 'form-link'
invalid_reason_codes = invalid_reason_codes
is_enabled = classmethod(is_wcs_enabled)
@ -58,20 +68,49 @@ class WcsFormCell(CellBase):
return WcsFormCellForm
def save(self, *args, **kwargs):
if self.formdef_reference:
wcs_key, form_slug = self.formdef_reference.split(':')
wcs_site = get_wcs_services().get(wcs_key)
forms_response_json = get_wcs_json(wcs_site, 'api/formdefs/')
if isinstance(forms_response_json, dict):
# forward compability with future w.c.s. API
forms_response_json = forms_response_json.get('data')
for form in forms_response_json:
slug = form.get('slug')
if slug == form_slug:
self.cached_title = form.get('title')
self.cached_url = form.get('url')
self.cached_json = form
return super(WcsFormCell, self).save(*args, **kwargs)
if 'update_fields' in kwargs:
# don't populate the cache
return super(WcsFormCell, self).save(*args, **kwargs)
def populate_cache():
if self.formdef_reference:
wcs_key, form_slug = self.formdef_reference.split(':')
wcs_site = get_wcs_services().get(wcs_key)
forms_response_json = get_wcs_json(wcs_site, 'api/formdefs/')
if not forms_response_json:
# can not retrieve data, don't report cell as invalid
self.mark_as_valid()
return
if isinstance(forms_response_json, dict):
# forward compability with future w.c.s. API
forms_response_json = forms_response_json.get('data')
form_found = False
for form in forms_response_json or []:
slug = form.get('slug')
if slug == form_slug:
self.cached_title = form.get('title')
self.cached_url = form.get('url')
self.cached_json = form
form_found = True
break
if form_found:
self.mark_as_valid()
return
else:
return self.mark_as_invalid('wcs_form_not_found', save=False)
else:
return self.mark_as_invalid('wcs_form_not_defined', save=False)
validity_info = populate_cache()
result = super(WcsFormCell, self).save(*args, **kwargs)
if validity_info:
# save validity_info after Cell insert
validity_info.content_object = self
validity_info.save()
return result
def get_cell_extra_context(self, context):
context = super(WcsFormCell, self).get_cell_extra_context(context)
@ -129,21 +168,51 @@ class WcsCommonCategoryCell(CellBase):
cached_description = models.TextField(_('Description'), blank=True)
cached_url = models.URLField(_('Cached URL'))
invalid_reason_codes = invalid_reason_codes
class Meta:
abstract = True
def save(self, *args, **kwargs):
if self.category_reference:
wcs_key, category_slug = self.category_reference.split(':')
wcs_site = get_wcs_services().get(wcs_key)
categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
for category in categories_response_json.get('data'):
slug = category.get('slug')
if slug == category_slug:
self.cached_title = category.get('title')
self.cached_description = category.get('description') or ''
self.cached_url = category.get('url')
return super(WcsCommonCategoryCell, self).save(*args, **kwargs)
if 'update_fields' in kwargs:
# don't populate the cache
return super(WcsCommonCategoryCell, self).save(*args, **kwargs)
def populate_cache():
if self.category_reference:
wcs_key, category_slug = self.category_reference.split(':')
wcs_site = get_wcs_services().get(wcs_key)
categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
if not categories_response_json:
# can not retrieve data, don't report cell as invalid
self.mark_as_valid()
return
category_found = False
for category in categories_response_json.get('data', []):
slug = category.get('slug')
if slug == category_slug:
self.cached_title = category.get('title')
self.cached_description = category.get('description') or ''
self.cached_url = category.get('url')
category_found = True
break
if category_found:
self.mark_as_valid()
return
else:
return self.mark_as_invalid('wcs_category_not_found', save=False)
else:
return self.mark_as_invalid('wcs_category_not_defined', save=False)
validity_info = populate_cache()
result = super(WcsCommonCategoryCell, self).save(*args, **kwargs)
if validity_info:
# save validity_info after Cell insert
validity_info.content_object = self
validity_info.save()
return result
def get_additional_label(self):
if not self.cached_title:
@ -192,7 +261,8 @@ class WcsBlurpMixin(object):
is_enabled = classmethod(is_wcs_enabled)
cache_duration = 5
api_url = None
warn_on_404 = True
warn_on_4xx = True
invalid_reason_codes = invalid_reason_codes
def get_api_url(self, context):
return self.api_url
@ -242,9 +312,15 @@ class WcsBlurpMixin(object):
for item in wcs_site['data']:
item['site_slug'] = wcs_slug
if not 200 in returns: # not a single valid answer
if returns != set([404]) or self.warn_on_404:
logging.error('failed to get data from any %s (%r)', api_url, returns)
if 200 not in returns: # not a single valid answer
logging.debug('failed to get data from any %s (%r)', api_url, returns)
if all([400 <= r < 500 for r in returns]) and self.warn_on_4xx:
# only 4xx errors, not a user cell, report the cell as invalid
self.mark_as_invalid('wcs_data_failure', force=False)
else:
self.mark_as_valid()
else:
self.mark_as_valid()
return wcs_sites
@ -293,7 +369,7 @@ class WcsDataBaseCell(CellBase, WcsBlurpMixin):
class WcsUserDataBaseCell(WcsDataBaseCell):
warn_on_404 = False
warn_on_4xx = False
user_dependant = True
class Meta:
@ -305,8 +381,34 @@ class WcsUserDataBaseCell(WcsDataBaseCell):
return super(WcsUserDataBaseCell, self).is_visible(user)
class CategoriesValidityMixin(object):
def check_validity(self):
categories = self.categories.get('data', [])
if not categories:
return
for category_reference in categories:
wcs_key, category_slug = category_reference.split(':')
wcs_site = get_wcs_services().get(wcs_key)
categories_response_json = get_wcs_json(wcs_site, 'api/categories/')
if not categories_response_json:
# can not retrieve data, don't report cell as invalid
continue
category_found = any(
[c.get('slug') == category_slug
for c in categories_response_json.get('data', [])])
if not category_found:
self.mark_as_invalid('wcs_category_not_found')
return
self.mark_as_valid()
@register_cell_class
class WcsCurrentFormsCell(WcsUserDataBaseCell):
class WcsCurrentFormsCell(WcsUserDataBaseCell, CategoriesValidityMixin):
variable_name = 'user_forms'
categories = JSONField(_('Categories'), blank=True)
@ -400,8 +502,9 @@ class WcsCurrentFormsCell(WcsUserDataBaseCell):
return context
@register_cell_class
class WcsCurrentDraftsCell(WcsUserDataBaseCell):
class WcsCurrentDraftsCell(WcsUserDataBaseCell, CategoriesValidityMixin):
variable_name = 'current_drafts'
template_name = 'combo/wcs/current_drafts.html'

View File

@ -22,7 +22,8 @@ from django.utils.six.moves.urllib import parse as urlparse
import mock
from combo.data.models import CellBase, LinkListCell, Page
from combo.data.library import get_cell_classes
from combo.data.models import CellBase, LinkListCell, Page, ValidityInfo
from combo.apps.search.engines import engines
from combo.apps.wcs.models import (WcsFormCell, WcsCurrentFormsCell,
WcsFormsOfCategoryCell, WcsCurrentDraftsCell, WcsCategoryCell,
@ -299,6 +300,33 @@ def test_form_cell_save_cache():
appconfig.update_db_cache()
assert WcsFormCell.objects.get(id=cell.id).cached_title == 'form title'
@wcs_present
def test_form_cell_validity():
page = Page.objects.create(title='xxx', slug='test_form_cell_save_cache', template_name='standard')
cell = WcsFormCell.objects.create(page=page, placeholder='content', order=0)
validity_info = ValidityInfo.objects.latest('pk')
assert validity_info.invalid_reason_code == 'wcs_form_not_defined'
assert validity_info.invalid_since is not None
cell.formdef_reference = u'default:form-title'
cell.save()
assert ValidityInfo.objects.exists() is False
# can not retrieve formdefs, don't set cell as invalid
with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
mock_json = mock.Mock(status_code=404)
requests_get.return_value = mock_json
cell.save()
assert ValidityInfo.objects.exists() is False
cell.formdef_reference = u'default:foobar'
cell.save()
validity_info = ValidityInfo.objects.latest('pk')
assert validity_info.invalid_reason_code == 'wcs_form_not_found'
assert validity_info.invalid_since is not None
@wcs_present
def test_form_cell_load():
page = Page(title='xxx', slug='test_form_cell_save_cache', template_name='standard')
@ -328,6 +356,33 @@ def test_category_cell_save_cache():
assert cell.cached_title == 'Test 3'
assert cell.get_additional_label() == 'Test 3'
@wcs_present
def test_category_cell_validity():
page = Page.objects.create(title='xxx', slug='test_category_cell_save_cache', template_name='standard')
cell = WcsCategoryCell.objects.create(page=page, placeholder='content', order=0)
validity_info = ValidityInfo.objects.latest('pk')
assert validity_info.invalid_reason_code == 'wcs_category_not_defined'
assert validity_info.invalid_since is not None
cell.category_reference = u'default:test-3'
cell.save()
assert ValidityInfo.objects.exists() is False
# can not retrieve categories, don't set cell as invalid
with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
mock_json = mock.Mock(status_code=404)
requests_get.return_value = mock_json
cell.save()
assert ValidityInfo.objects.exists() is False
cell.category_reference = 'default:foobar'
cell.save()
validity_info = ValidityInfo.objects.latest('pk')
assert validity_info.invalid_reason_code == 'wcs_category_not_found'
assert validity_info.invalid_since is not None
@wcs_present
def test_form_cell_render():
page = Page(title='xxx', slug='test_form_cell_render', template_name='standard')
@ -426,15 +481,67 @@ def test_current_forms_cell_render(context):
# check include drafts
cell.include_drafts = False
with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
mock_json = mock.Mock(status_code=200)
requests_get.return_value = mock_json
cell.get_cell_extra_context(context)
assert requests_get.call_args_list[0][0][0] == '/api/user/forms?limit=100&sort=desc'
cell.include_drafts = True
with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
mock_json = mock.Mock(status_code=200)
requests_get.return_value = mock_json
cell.get_cell_extra_context(context)
assert requests_get.call_args_list[0][0][0] == '/api/user/forms?limit=100&sort=desc&include-drafts=on'
@wcs_present
def test_current_forms_cell_validity(context):
page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
cell = WcsCurrentFormsCell.objects.create(page=page, placeholder='content', order=0)
context['request'].user = MockUser()
context['synchronous'] = True # to get fresh content
cell.get_data(context)
assert ValidityInfo.objects.exists() is False
with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
mock_json = mock.Mock(status_code=404)
requests_get.return_value = mock_json
cell.get_data(context)
assert ValidityInfo.objects.exists() is False
@wcs_present
def test_current_forms_cell_check_validity(context):
page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
cell = WcsCurrentFormsCell.objects.create(page=page, placeholder='content', order=0)
# no category
cell.check_validity()
assert ValidityInfo.objects.exists() is False
# valid categories
cell.categories = {'data': ['default:test-3', 'default:test-9']}
cell.save()
cell.check_validity()
assert ValidityInfo.objects.exists() is False
# can not retrieve categories, don't set cell as invalid
with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
mock_json = mock.Mock(status_code=404)
requests_get.return_value = mock_json
cell.check_validity()
assert ValidityInfo.objects.exists() is False
# invalid category
cell.categories = {'data': ['default:foobar', 'default:test-9']}
cell.save()
cell.check_validity()
validity_info = ValidityInfo.objects.latest('pk')
assert validity_info.invalid_reason_code == 'wcs_category_not_found'
assert validity_info.invalid_since is not None
@wcs_present
def test_current_forms_cell_render_single_site(context):
page = Page(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
@ -542,6 +649,33 @@ def test_care_forms_cell_render(context):
assert 'other' in data
@wcs_present
def test_care_forms_cell_validity(context):
page = Page.objects.create(title='xxx', slug='test_care_forms_cell_render', template_name='standard')
cell = WcsCareFormsCell.objects.create(page=page, placeholder='content', order=0)
context['synchronous'] = True # to get fresh content
with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
mock_json = mock.Mock(status_code=200)
requests_get.return_value = mock_json
cell.get_data(context)
assert ValidityInfo.objects.exists() is False
# can not retrieve data, don't set cell as invalid
with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
mock_json = mock.Mock(status_code=500)
requests_get.return_value = mock_json
cell.get_data(context)
assert ValidityInfo.objects.exists() is False
with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
mock_json = mock.Mock(status_code=404)
requests_get.return_value = mock_json
cell.get_data(context)
validity_info = ValidityInfo.objects.latest('pk')
assert validity_info.invalid_reason_code == 'wcs_data_failure'
assert validity_info.invalid_since is not None
@wcs_present
def test_care_forms_cell_render_single_site(context):
page = Page(title='xxx', slug='test_care_forms_cell_render', template_name='standard')
@ -579,6 +713,7 @@ def test_forms_of_category_cell_setup():
(u'other:test-3', u'test2 : Test 3'),
(u'other:test-9', u'test2 : Test 9')]
@wcs_present
def test_forms_of_category_cell_render(context):
page = Page(title='xxx', slug='test_forms_of_category_cell_render', template_name='standard')
@ -625,6 +760,48 @@ def test_forms_of_category_cell_render(context):
assert cell.render_for_search() == ''
assert len(list(cell.get_external_links_data())) == 2
@wcs_present
def test_forms_of_category_cell_validity(context):
page = Page.objects.create(title='xxx', slug='test_forms_of_category_cell_render', template_name='standard')
cell = WcsFormsOfCategoryCell.objects.create(page=page, placeholder='content', order=0)
validity_info = ValidityInfo.objects.latest('pk')
assert validity_info.invalid_reason_code == 'wcs_category_not_defined'
assert validity_info.invalid_since is not None
cell.category_reference = 'default:test-9'
cell.save()
assert ValidityInfo.objects.exists() is False
with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
mock_json = mock.Mock(status_code=404)
requests_get.return_value = mock_json
cell.render(context)
validity_info = ValidityInfo.objects.latest('pk')
assert validity_info.invalid_reason_code == 'wcs_data_failure'
assert validity_info.invalid_since is not None
cell.mark_as_valid()
# can not retrieve data, don't set cell as invalid
with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
mock_json = mock.Mock(status_code=500)
requests_get.return_value = mock_json
cell.render(context)
assert ValidityInfo.objects.exists() is False
# can not retrieve categories, don't set cell as invalid
with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
mock_json = mock.Mock(status_code=404)
requests_get.return_value = mock_json
cell.save()
assert ValidityInfo.objects.exists() is False
cell.category_reference = 'default:foobar'
cell.save()
validity_info = ValidityInfo.objects.latest('pk')
assert validity_info.invalid_reason_code == 'wcs_category_not_found'
assert validity_info.invalid_since is not None
@wcs_present
def test_current_drafts_cell_render_unlogged(context):
page = Page(title='xxx', slug='test_current_drafts_cell_render', template_name='standard')
@ -666,6 +843,37 @@ def test_current_drafts_cell_render_logged_in(context):
assert len(extra_context['drafts']) == 0
@wcs_present
def test_current_drafts_cell_check_validity(context):
page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
cell = WcsCurrentDraftsCell.objects.create(page=page, placeholder='content', order=0)
# no category
cell.check_validity()
assert ValidityInfo.objects.exists() is False
# valid categories
cell.categories = {'data': ['default:test-3', 'default:test-9']}
cell.save()
cell.check_validity()
assert ValidityInfo.objects.exists() is False
# can not retrieve vategories, don't set cell as invalid
with mock.patch('combo.apps.wcs.models.requests.get') as requests_get:
mock_json = mock.Mock(status_code=404)
requests_get.return_value = mock_json
cell.check_validity()
assert ValidityInfo.objects.exists() is False
# invalid category
cell.categories = {'data': ['default:foobar', 'default:test-9']}
cell.save()
cell.check_validity()
validity_info = ValidityInfo.objects.latest('pk')
assert validity_info.invalid_reason_code == 'wcs_category_not_found'
assert validity_info.invalid_since is not None
@wcs_present
def test_manager_forms_of_category_cell(app, admin_user):
Page.objects.all().delete()
@ -971,6 +1179,7 @@ def test_manager_add_edit_delete_list_link_item(app, admin_user):
assert WcsFormCell.objects.count() == 0
@wcs_present
def test_import_export_pages_with_links():
page = Page(title=u'bar', slug='bar', order=1)
page.save()
@ -998,3 +1207,19 @@ def test_import_export_pages_with_links():
assert new_item.cached_title == item.cached_title
assert new_item.cached_url == item.cached_url
assert new_item.cached_json == item.cached_json
@wcs_present
def test_hourly():
appconfig = apps.get_app_config('wcs')
page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard')
cell_classes = [c for c in appconfig.get_models() if c in get_cell_classes()]
for klass in cell_classes:
klass.objects.create(page=page, placeholder='content', order=0)
for klass in cell_classes:
if klass in [WcsCurrentFormsCell, WcsCurrentDraftsCell]:
with mock.patch('combo.apps.wcs.models.%s.check_validity' % klass.__name__) as check_validity:
appconfig.hourly()
assert check_validity.call_args_list == [mock.call()]
else:
assert hasattr(klass, 'check_validity') is False