diff --git a/combo/apps/wcs/__init__.py b/combo/apps/wcs/__init__.py index d5412651..3e3c2c54 100644 --- a/combo/apps/wcs/__init__.py +++ b/combo/apps/wcs/__init__.py @@ -65,6 +65,14 @@ class AppConfig(django.apps.AppConfig): return urls.urlpatterns def hourly(self): + from combo.data.library import get_cell_classes + from combo.data.models import CellBase + + cell_classes = [c for c in self.get_models() if c in get_cell_classes()] + for cell in CellBase.get_cells(cell_filter=lambda x: x in cell_classes, page__snapshot__isnull=True): + if hasattr(cell, 'check_validity'): + cell.check_validity() + self.update_db_cache() def update_db_cache(self): @@ -74,4 +82,5 @@ class AppConfig(django.apps.AppConfig): cell_filter=lambda x: x in (WcsFormCell, WcsCategoryCell, WcsFormsOfCategoryCell)): cell.save() + default_app_config = 'combo.apps.wcs.AppConfig' diff --git a/combo/apps/wcs/models.py b/combo/apps/wcs/models.py index 854a3b7d..ce7a8057 100644 --- a/combo/apps/wcs/models.py +++ b/combo/apps/wcs/models.py @@ -19,7 +19,6 @@ import copy import logging -from django import template from django.conf import settings from django.db import models from django.forms import models as model_forms @@ -34,6 +33,16 @@ from combo.utils import requests from .utils import get_wcs_json, is_wcs_enabled, get_wcs_services + +invalid_reason_codes = { + 'wcs_form_not_defined': _('No form set'), + 'wcs_form_not_found': _('Invalid form'), + 'wcs_category_not_defined': _('No category set'), + 'wcs_category_not_found': _('Invalid category'), + 'wcs_data_failure': _('Failed to get data'), +} + + @register_cell_class class WcsFormCell(CellBase): formdef_reference = models.CharField(_('Form'), max_length=150) @@ -47,6 +56,7 @@ class WcsFormCell(CellBase): add_link_label = _('New form link') edit_link_label = _('Edit form link') add_as_link_code = 'form-link' + invalid_reason_codes = invalid_reason_codes is_enabled = classmethod(is_wcs_enabled) @@ -58,20 +68,49 @@ class WcsFormCell(CellBase): return WcsFormCellForm def save(self, *args, **kwargs): - if self.formdef_reference: - wcs_key, form_slug = self.formdef_reference.split(':') - wcs_site = get_wcs_services().get(wcs_key) - forms_response_json = get_wcs_json(wcs_site, 'api/formdefs/') - if isinstance(forms_response_json, dict): - # forward compability with future w.c.s. API - forms_response_json = forms_response_json.get('data') - for form in forms_response_json: - slug = form.get('slug') - if slug == form_slug: - self.cached_title = form.get('title') - self.cached_url = form.get('url') - self.cached_json = form - return super(WcsFormCell, self).save(*args, **kwargs) + if 'update_fields' in kwargs: + # don't populate the cache + return super(WcsFormCell, self).save(*args, **kwargs) + + def populate_cache(): + if self.formdef_reference: + wcs_key, form_slug = self.formdef_reference.split(':') + wcs_site = get_wcs_services().get(wcs_key) + forms_response_json = get_wcs_json(wcs_site, 'api/formdefs/') + + if not forms_response_json: + # can not retrieve data, don't report cell as invalid + self.mark_as_valid() + return + + if isinstance(forms_response_json, dict): + # forward compability with future w.c.s. API + forms_response_json = forms_response_json.get('data') + + form_found = False + for form in forms_response_json or []: + slug = form.get('slug') + if slug == form_slug: + self.cached_title = form.get('title') + self.cached_url = form.get('url') + self.cached_json = form + form_found = True + break + if form_found: + self.mark_as_valid() + return + else: + return self.mark_as_invalid('wcs_form_not_found', save=False) + else: + return self.mark_as_invalid('wcs_form_not_defined', save=False) + + validity_info = populate_cache() + result = super(WcsFormCell, self).save(*args, **kwargs) + if validity_info: + # save validity_info after Cell insert + validity_info.content_object = self + validity_info.save() + return result def get_cell_extra_context(self, context): context = super(WcsFormCell, self).get_cell_extra_context(context) @@ -129,21 +168,51 @@ class WcsCommonCategoryCell(CellBase): cached_description = models.TextField(_('Description'), blank=True) cached_url = models.URLField(_('Cached URL')) + invalid_reason_codes = invalid_reason_codes + class Meta: abstract = True def save(self, *args, **kwargs): - if self.category_reference: - wcs_key, category_slug = self.category_reference.split(':') - wcs_site = get_wcs_services().get(wcs_key) - categories_response_json = get_wcs_json(wcs_site, 'api/categories/') - for category in categories_response_json.get('data'): - slug = category.get('slug') - if slug == category_slug: - self.cached_title = category.get('title') - self.cached_description = category.get('description') or '' - self.cached_url = category.get('url') - return super(WcsCommonCategoryCell, self).save(*args, **kwargs) + if 'update_fields' in kwargs: + # don't populate the cache + return super(WcsCommonCategoryCell, self).save(*args, **kwargs) + + def populate_cache(): + if self.category_reference: + wcs_key, category_slug = self.category_reference.split(':') + wcs_site = get_wcs_services().get(wcs_key) + categories_response_json = get_wcs_json(wcs_site, 'api/categories/') + + if not categories_response_json: + # can not retrieve data, don't report cell as invalid + self.mark_as_valid() + return + + category_found = False + for category in categories_response_json.get('data', []): + slug = category.get('slug') + if slug == category_slug: + self.cached_title = category.get('title') + self.cached_description = category.get('description') or '' + self.cached_url = category.get('url') + category_found = True + break + if category_found: + self.mark_as_valid() + return + else: + return self.mark_as_invalid('wcs_category_not_found', save=False) + else: + return self.mark_as_invalid('wcs_category_not_defined', save=False) + + validity_info = populate_cache() + result = super(WcsCommonCategoryCell, self).save(*args, **kwargs) + if validity_info: + # save validity_info after Cell insert + validity_info.content_object = self + validity_info.save() + return result def get_additional_label(self): if not self.cached_title: @@ -192,7 +261,8 @@ class WcsBlurpMixin(object): is_enabled = classmethod(is_wcs_enabled) cache_duration = 5 api_url = None - warn_on_404 = True + warn_on_4xx = True + invalid_reason_codes = invalid_reason_codes def get_api_url(self, context): return self.api_url @@ -242,9 +312,15 @@ class WcsBlurpMixin(object): for item in wcs_site['data']: item['site_slug'] = wcs_slug - if not 200 in returns: # not a single valid answer - if returns != set([404]) or self.warn_on_404: - logging.error('failed to get data from any %s (%r)', api_url, returns) + if 200 not in returns: # not a single valid answer + logging.debug('failed to get data from any %s (%r)', api_url, returns) + if all([400 <= r < 500 for r in returns]) and self.warn_on_4xx: + # only 4xx errors, not a user cell, report the cell as invalid + self.mark_as_invalid('wcs_data_failure', force=False) + else: + self.mark_as_valid() + else: + self.mark_as_valid() return wcs_sites @@ -293,7 +369,7 @@ class WcsDataBaseCell(CellBase, WcsBlurpMixin): class WcsUserDataBaseCell(WcsDataBaseCell): - warn_on_404 = False + warn_on_4xx = False user_dependant = True class Meta: @@ -305,8 +381,34 @@ class WcsUserDataBaseCell(WcsDataBaseCell): return super(WcsUserDataBaseCell, self).is_visible(user) +class CategoriesValidityMixin(object): + def check_validity(self): + categories = self.categories.get('data', []) + + if not categories: + return + + for category_reference in categories: + wcs_key, category_slug = category_reference.split(':') + wcs_site = get_wcs_services().get(wcs_key) + categories_response_json = get_wcs_json(wcs_site, 'api/categories/') + + if not categories_response_json: + # can not retrieve data, don't report cell as invalid + continue + + category_found = any( + [c.get('slug') == category_slug + for c in categories_response_json.get('data', [])]) + if not category_found: + self.mark_as_invalid('wcs_category_not_found') + return + + self.mark_as_valid() + + @register_cell_class -class WcsCurrentFormsCell(WcsUserDataBaseCell): +class WcsCurrentFormsCell(WcsUserDataBaseCell, CategoriesValidityMixin): variable_name = 'user_forms' categories = JSONField(_('Categories'), blank=True) @@ -400,8 +502,9 @@ class WcsCurrentFormsCell(WcsUserDataBaseCell): return context + @register_cell_class -class WcsCurrentDraftsCell(WcsUserDataBaseCell): +class WcsCurrentDraftsCell(WcsUserDataBaseCell, CategoriesValidityMixin): variable_name = 'current_drafts' template_name = 'combo/wcs/current_drafts.html' diff --git a/tests/test_wcs.py b/tests/test_wcs.py index 8f00b5f8..3c0ae747 100644 --- a/tests/test_wcs.py +++ b/tests/test_wcs.py @@ -22,7 +22,8 @@ from django.utils.six.moves.urllib import parse as urlparse import mock -from combo.data.models import CellBase, LinkListCell, Page +from combo.data.library import get_cell_classes +from combo.data.models import CellBase, LinkListCell, Page, ValidityInfo from combo.apps.search.engines import engines from combo.apps.wcs.models import (WcsFormCell, WcsCurrentFormsCell, WcsFormsOfCategoryCell, WcsCurrentDraftsCell, WcsCategoryCell, @@ -299,6 +300,33 @@ def test_form_cell_save_cache(): appconfig.update_db_cache() assert WcsFormCell.objects.get(id=cell.id).cached_title == 'form title' + +@wcs_present +def test_form_cell_validity(): + page = Page.objects.create(title='xxx', slug='test_form_cell_save_cache', template_name='standard') + cell = WcsFormCell.objects.create(page=page, placeholder='content', order=0) + validity_info = ValidityInfo.objects.latest('pk') + assert validity_info.invalid_reason_code == 'wcs_form_not_defined' + assert validity_info.invalid_since is not None + + cell.formdef_reference = u'default:form-title' + cell.save() + assert ValidityInfo.objects.exists() is False + + # can not retrieve formdefs, don't set cell as invalid + with mock.patch('combo.apps.wcs.models.requests.get') as requests_get: + mock_json = mock.Mock(status_code=404) + requests_get.return_value = mock_json + cell.save() + assert ValidityInfo.objects.exists() is False + + cell.formdef_reference = u'default:foobar' + cell.save() + validity_info = ValidityInfo.objects.latest('pk') + assert validity_info.invalid_reason_code == 'wcs_form_not_found' + assert validity_info.invalid_since is not None + + @wcs_present def test_form_cell_load(): page = Page(title='xxx', slug='test_form_cell_save_cache', template_name='standard') @@ -328,6 +356,33 @@ def test_category_cell_save_cache(): assert cell.cached_title == 'Test 3' assert cell.get_additional_label() == 'Test 3' + +@wcs_present +def test_category_cell_validity(): + page = Page.objects.create(title='xxx', slug='test_category_cell_save_cache', template_name='standard') + cell = WcsCategoryCell.objects.create(page=page, placeholder='content', order=0) + validity_info = ValidityInfo.objects.latest('pk') + assert validity_info.invalid_reason_code == 'wcs_category_not_defined' + assert validity_info.invalid_since is not None + + cell.category_reference = u'default:test-3' + cell.save() + assert ValidityInfo.objects.exists() is False + + # can not retrieve categories, don't set cell as invalid + with mock.patch('combo.apps.wcs.models.requests.get') as requests_get: + mock_json = mock.Mock(status_code=404) + requests_get.return_value = mock_json + cell.save() + assert ValidityInfo.objects.exists() is False + + cell.category_reference = 'default:foobar' + cell.save() + validity_info = ValidityInfo.objects.latest('pk') + assert validity_info.invalid_reason_code == 'wcs_category_not_found' + assert validity_info.invalid_since is not None + + @wcs_present def test_form_cell_render(): page = Page(title='xxx', slug='test_form_cell_render', template_name='standard') @@ -426,15 +481,67 @@ def test_current_forms_cell_render(context): # check include drafts cell.include_drafts = False with mock.patch('combo.apps.wcs.models.requests.get') as requests_get: + mock_json = mock.Mock(status_code=200) + requests_get.return_value = mock_json cell.get_cell_extra_context(context) assert requests_get.call_args_list[0][0][0] == '/api/user/forms?limit=100&sort=desc' cell.include_drafts = True with mock.patch('combo.apps.wcs.models.requests.get') as requests_get: + mock_json = mock.Mock(status_code=200) + requests_get.return_value = mock_json cell.get_cell_extra_context(context) assert requests_get.call_args_list[0][0][0] == '/api/user/forms?limit=100&sort=desc&include-drafts=on' +@wcs_present +def test_current_forms_cell_validity(context): + page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard') + cell = WcsCurrentFormsCell.objects.create(page=page, placeholder='content', order=0) + context['request'].user = MockUser() + context['synchronous'] = True # to get fresh content + + cell.get_data(context) + assert ValidityInfo.objects.exists() is False + + with mock.patch('combo.apps.wcs.models.requests.get') as requests_get: + mock_json = mock.Mock(status_code=404) + requests_get.return_value = mock_json + cell.get_data(context) + assert ValidityInfo.objects.exists() is False + + +@wcs_present +def test_current_forms_cell_check_validity(context): + page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard') + cell = WcsCurrentFormsCell.objects.create(page=page, placeholder='content', order=0) + + # no category + cell.check_validity() + assert ValidityInfo.objects.exists() is False + + # valid categories + cell.categories = {'data': ['default:test-3', 'default:test-9']} + cell.save() + cell.check_validity() + assert ValidityInfo.objects.exists() is False + + # can not retrieve categories, don't set cell as invalid + with mock.patch('combo.apps.wcs.models.requests.get') as requests_get: + mock_json = mock.Mock(status_code=404) + requests_get.return_value = mock_json + cell.check_validity() + assert ValidityInfo.objects.exists() is False + + # invalid category + cell.categories = {'data': ['default:foobar', 'default:test-9']} + cell.save() + cell.check_validity() + validity_info = ValidityInfo.objects.latest('pk') + assert validity_info.invalid_reason_code == 'wcs_category_not_found' + assert validity_info.invalid_since is not None + + @wcs_present def test_current_forms_cell_render_single_site(context): page = Page(title='xxx', slug='test_current_forms_cell_render', template_name='standard') @@ -542,6 +649,33 @@ def test_care_forms_cell_render(context): assert 'other' in data +@wcs_present +def test_care_forms_cell_validity(context): + page = Page.objects.create(title='xxx', slug='test_care_forms_cell_render', template_name='standard') + cell = WcsCareFormsCell.objects.create(page=page, placeholder='content', order=0) + context['synchronous'] = True # to get fresh content + with mock.patch('combo.apps.wcs.models.requests.get') as requests_get: + mock_json = mock.Mock(status_code=200) + requests_get.return_value = mock_json + cell.get_data(context) + assert ValidityInfo.objects.exists() is False + + # can not retrieve data, don't set cell as invalid + with mock.patch('combo.apps.wcs.models.requests.get') as requests_get: + mock_json = mock.Mock(status_code=500) + requests_get.return_value = mock_json + cell.get_data(context) + assert ValidityInfo.objects.exists() is False + + with mock.patch('combo.apps.wcs.models.requests.get') as requests_get: + mock_json = mock.Mock(status_code=404) + requests_get.return_value = mock_json + cell.get_data(context) + validity_info = ValidityInfo.objects.latest('pk') + assert validity_info.invalid_reason_code == 'wcs_data_failure' + assert validity_info.invalid_since is not None + + @wcs_present def test_care_forms_cell_render_single_site(context): page = Page(title='xxx', slug='test_care_forms_cell_render', template_name='standard') @@ -579,6 +713,7 @@ def test_forms_of_category_cell_setup(): (u'other:test-3', u'test2 : Test 3'), (u'other:test-9', u'test2 : Test 9')] + @wcs_present def test_forms_of_category_cell_render(context): page = Page(title='xxx', slug='test_forms_of_category_cell_render', template_name='standard') @@ -625,6 +760,48 @@ def test_forms_of_category_cell_render(context): assert cell.render_for_search() == '' assert len(list(cell.get_external_links_data())) == 2 + +@wcs_present +def test_forms_of_category_cell_validity(context): + page = Page.objects.create(title='xxx', slug='test_forms_of_category_cell_render', template_name='standard') + cell = WcsFormsOfCategoryCell.objects.create(page=page, placeholder='content', order=0) + validity_info = ValidityInfo.objects.latest('pk') + assert validity_info.invalid_reason_code == 'wcs_category_not_defined' + assert validity_info.invalid_since is not None + cell.category_reference = 'default:test-9' + cell.save() + assert ValidityInfo.objects.exists() is False + + with mock.patch('combo.apps.wcs.models.requests.get') as requests_get: + mock_json = mock.Mock(status_code=404) + requests_get.return_value = mock_json + cell.render(context) + validity_info = ValidityInfo.objects.latest('pk') + assert validity_info.invalid_reason_code == 'wcs_data_failure' + assert validity_info.invalid_since is not None + cell.mark_as_valid() + + # can not retrieve data, don't set cell as invalid + with mock.patch('combo.apps.wcs.models.requests.get') as requests_get: + mock_json = mock.Mock(status_code=500) + requests_get.return_value = mock_json + cell.render(context) + assert ValidityInfo.objects.exists() is False + + # can not retrieve categories, don't set cell as invalid + with mock.patch('combo.apps.wcs.models.requests.get') as requests_get: + mock_json = mock.Mock(status_code=404) + requests_get.return_value = mock_json + cell.save() + assert ValidityInfo.objects.exists() is False + + cell.category_reference = 'default:foobar' + cell.save() + validity_info = ValidityInfo.objects.latest('pk') + assert validity_info.invalid_reason_code == 'wcs_category_not_found' + assert validity_info.invalid_since is not None + + @wcs_present def test_current_drafts_cell_render_unlogged(context): page = Page(title='xxx', slug='test_current_drafts_cell_render', template_name='standard') @@ -666,6 +843,37 @@ def test_current_drafts_cell_render_logged_in(context): assert len(extra_context['drafts']) == 0 +@wcs_present +def test_current_drafts_cell_check_validity(context): + page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard') + cell = WcsCurrentDraftsCell.objects.create(page=page, placeholder='content', order=0) + + # no category + cell.check_validity() + assert ValidityInfo.objects.exists() is False + + # valid categories + cell.categories = {'data': ['default:test-3', 'default:test-9']} + cell.save() + cell.check_validity() + assert ValidityInfo.objects.exists() is False + + # can not retrieve vategories, don't set cell as invalid + with mock.patch('combo.apps.wcs.models.requests.get') as requests_get: + mock_json = mock.Mock(status_code=404) + requests_get.return_value = mock_json + cell.check_validity() + assert ValidityInfo.objects.exists() is False + + # invalid category + cell.categories = {'data': ['default:foobar', 'default:test-9']} + cell.save() + cell.check_validity() + validity_info = ValidityInfo.objects.latest('pk') + assert validity_info.invalid_reason_code == 'wcs_category_not_found' + assert validity_info.invalid_since is not None + + @wcs_present def test_manager_forms_of_category_cell(app, admin_user): Page.objects.all().delete() @@ -971,6 +1179,7 @@ def test_manager_add_edit_delete_list_link_item(app, admin_user): assert WcsFormCell.objects.count() == 0 +@wcs_present def test_import_export_pages_with_links(): page = Page(title=u'bar', slug='bar', order=1) page.save() @@ -998,3 +1207,19 @@ def test_import_export_pages_with_links(): assert new_item.cached_title == item.cached_title assert new_item.cached_url == item.cached_url assert new_item.cached_json == item.cached_json + + +@wcs_present +def test_hourly(): + appconfig = apps.get_app_config('wcs') + page = Page.objects.create(title='xxx', slug='test_current_forms_cell_render', template_name='standard') + cell_classes = [c for c in appconfig.get_models() if c in get_cell_classes()] + for klass in cell_classes: + klass.objects.create(page=page, placeholder='content', order=0) + for klass in cell_classes: + if klass in [WcsCurrentFormsCell, WcsCurrentDraftsCell]: + with mock.patch('combo.apps.wcs.models.%s.check_validity' % klass.__name__) as check_validity: + appconfig.hourly() + assert check_validity.call_args_list == [mock.call()] + else: + assert hasattr(klass, 'check_validity') is False