entrouvert/wcs
entrouvert
/
wcs
14
1
Fork 0
Code Issues Pull Requests 30 Releases Activity

logged errors: log also standalone errors (#52045)

This commit is contained in:
Lauréline Guérin 2021-03-16 16:21:22 +01:00
parent 8d9430f073
commit 82cb70b26a
No known key found for this signature in database
GPG Key ID: 1FAB9B9B4F93D473
6 changed files with 203 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
tests/admin_pages/test_logged_errors.py
View File

@ -108,6 +108,12 @@ def test_listing_paginations(pub):
error.first_occurence_timestamp = datetime.datetime.now()
error.store()
# standalone error
error = pub.loggederror_class()
error.summary = 'Lonely Logged Error'
error.first_occurence_timestamp = datetime.datetime.now()
error.store()
create_superuser(pub)
app = login(get_app(pub))
@ -115,19 +121,21 @@ def test_listing_paginations(pub):
# default pagination
resp = app.get('/backoffice/studio/logged-errors/')
assert '1-20/66' in resp.text
assert resp.text.count('Logged Error n°') == 20
assert '1-20/67' in resp.text
assert resp.text.count('Lonely Logged Error') == 1
assert resp.text.count('Logged Error n°') == 19
resp = resp.click(href=r'\?offset=60')
assert '61-66/66' in resp.text
assert resp.text.count('Logged Error n°') == 6
assert '61-67/67' in resp.text
assert resp.text.count('Logged Error n°') == 7
# change pagination
resp = app.get('/backoffice/studio/logged-errors/?offset=0&limit=50')
assert '1-50/66' in resp.text
assert resp.text.count('Logged Error n°') == 50
assert '1-50/67' in resp.text
assert resp.text.count('Lonely Logged Error') == 1
assert resp.text.count('Logged Error n°') == 49
resp = resp.click('<!--Next Page-->')
assert '51-66/66' in resp.text
assert resp.text.count('Logged Error n°') == 16
assert '51-67/67' in resp.text
assert resp.text.count('Logged Error n°') == 17
# formdef errors
resp = app.get('/backoffice/forms/%s/logged-errors/' % formdef.id)

13
tests/form_pages/test_all.py
View File

@ -5686,6 +5686,8 @@ def test_items_field_with_disabled_items(http_requests, pub):
def test_item_field_autocomplete_json_source(http_requests, pub, error_email, emails):
if pub.is_using_postgresql():
pub.loggederror_class.wipe()
user = create_user(pub)
formdef = create_formdef()
formdef.data_class().wipe()
@ -5794,10 +5796,19 @@ def test_item_field_autocomplete_json_source(http_requests, pub, error_email, em
assert emails.count() == 0
data_source.notify_on_errors = True
data_source.record_on_errors = True
data_source.store()
resp2 = app.get(select2_url + '?q=hell')
assert emails.count() == 1
assert 'wcs.qommon.errors.ConnectionError: ...' in emails.get_latest('subject')
assert (
emails.get_latest('subject')
== '[ERROR] [DATASOURCE] Exception: Error loading JSON data source (...)'
)
if pub.is_using_postgresql():
assert pub.loggederror_class.count() == 1
logged_error = pub.loggederror_class.select()[0]
assert logged_error.workflow_id is None
assert logged_error.summary == '[DATASOURCE] Exception: Error loading JSON data source (...)'
data_source.notify_on_errors = False
data_source.store()

174
tests/test_datasource.py
View File

@ -4,12 +4,10 @@ import codecs
import io
import json
import os
import shutil
import urllib.parse
import mock
import pytest
from quixote import cleanup
from test_widgets import MockHtmlForm, mock_form_submission
from utilities import clean_temporary_pub, create_temporary_pub
@ -19,9 +17,14 @@ from wcs.qommon.form import Form, get_request
from wcs.qommon.http_request import HTTPRequest
def pytest_generate_tests(metafunc):
if 'pub' in metafunc.fixturenames:
metafunc.parametrize('pub', ['pickle', 'sql'], indirect=True)
@pytest.fixture
def pub():
pub = create_temporary_pub()
def pub(request):
pub = create_temporary_pub(sql_mode=bool('sql' in request.param))
req = HTTPRequest(None, {'SCRIPT_NAME': '/', 'SERVER_NAME': 'example.net'})
pub.set_app_dir(req)
@ -132,15 +135,33 @@ def test_python_datasource(pub):
def test_python_datasource_errors(pub, error_email, http_requests, emails, caplog):
if pub.is_using_postgresql():
pub.loggederror_class.wipe()
# invalid python expression
datasource = {'type': 'formula', 'value': 'foobar', 'notify_on_errors': True}
datasource = {'type': 'formula', 'value': 'foobar', 'notify_on_errors': True, 'record_on_errors': True}
assert data_sources.get_items(datasource) == []
assert 'Failed to eval() Python data source' in emails.get_latest('subject')
if pub.is_using_postgresql():
assert pub.loggederror_class.count() == 1
logged_error = pub.loggederror_class.select()[0]
assert logged_error.workflow_id is None
assert (
logged_error.summary == "[DATASOURCE] Exception: Failed to eval() Python data source ('foobar')"
)
# expression not iterable
datasource = {'type': 'formula', 'value': '2', 'notify_on_errors': True}
datasource = {'type': 'formula', 'value': '2', 'notify_on_errors': True, 'record_on_errors': True}
assert data_sources.get_items(datasource) == []
assert 'gave a non-iterable result' in emails.get_latest('subject')
if pub.is_using_postgresql():
assert pub.loggederror_class.count() == 2
logged_error = pub.loggederror_class.select()[1]
assert logged_error.workflow_id is None
assert (
logged_error.summary
== "[DATASOURCE] Exception: Python data source ('2') gave a non-iterable result"
)
def test_python_datasource_with_evalutils(pub):
@ -158,7 +179,6 @@ def test_python_datasource_with_evalutils(pub):
def test_json_datasource(pub, requests_pub, http_requests):
req = get_request()
get_request().datasources_cache = {}
datasource = {'type': 'json', 'value': ''}
assert data_sources.get_items(datasource) == []
@ -377,53 +397,115 @@ def test_json_datasource(pub, requests_pub, http_requests):
def test_json_datasource_bad_url(pub, error_email, http_requests, emails, caplog):
if pub.is_using_postgresql():
pub.loggederror_class.wipe()
datasource = {'type': 'json', 'value': 'http://remote.example.net/404'}
assert data_sources.get_items(datasource) == []
assert emails.count() == 0
datasource = {'type': 'json', 'value': 'http://remote.example.net/404', 'notify_on_errors': True}
datasource = {
'type': 'json',
'value': 'http://remote.example.net/404',
'notify_on_errors': True,
'record_on_errors': True,
}
assert data_sources.get_items(datasource) == []
assert emails.count() == 1
assert 'error in HTTP request to http://remote.example.net/404 (status: 404)' in emails.get_latest(
'subject'
)
if pub.is_using_postgresql():
assert pub.loggederror_class.count() == 1
logged_error = pub.loggederror_class.select()[0]
assert logged_error.workflow_id is None
assert (
logged_error.summary
== "[DATASOURCE] Exception: Error loading JSON data source (error in HTTP request to http://remote.example.net/404 (status: 404))"
)
datasource = {'type': 'json', 'value': 'http://remote.example.net/xml', 'notify_on_errors': True}
datasource = {
'type': 'json',
'value': 'http://remote.example.net/xml',
'notify_on_errors': True,
'record_on_errors': True,
}
assert data_sources.get_items(datasource) == []
assert emails.count() == 2
assert 'Error reading JSON data source' in emails.get_latest('subject')
if pub.is_using_postgresql():
assert pub.loggederror_class.count() == 2
logged_error = pub.loggederror_class.select()[1]
assert logged_error.workflow_id is None
assert (
logged_error.summary
== "[DATASOURCE] Exception: Error reading JSON data source output (Expecting value: line 1 column 1 (char 0))"
)
datasource = {
'type': 'json',
'value': 'http://remote.example.net/connection-error',
'notify_on_errors': True,
'record_on_errors': True,
}
assert data_sources.get_items(datasource) == []
assert 'Error loading JSON data source' in emails.get_latest('subject')
if pub.is_using_postgresql():
assert pub.loggederror_class.count() == 3
logged_error = pub.loggederror_class.select()[2]
assert logged_error.workflow_id is None
assert logged_error.summary == "[DATASOURCE] Exception: Error loading JSON data source (error)"
datasource = {
'type': 'json',
'value': 'http://remote.example.net/json-list-err1',
'notify_on_errors': True,
'record_on_errors': True,
}
assert data_sources.get_items(datasource) == []
assert 'Error reading JSON data source output (err 1)' in emails.get_latest('subject')
if pub.is_using_postgresql():
assert pub.loggederror_class.count() == 4
logged_error = pub.loggederror_class.select()[3]
assert logged_error.workflow_id is None
assert logged_error.summary == "[DATASOURCE] Exception: Error reading JSON data source output (err 1)"
def test_json_datasource_bad_url_scheme(pub, error_email, emails):
datasource = {'type': 'json', 'value': '', 'notify_on_errors': True}
if pub.is_using_postgresql():
pub.loggederror_class.wipe()
datasource = {'type': 'json', 'value': '', 'notify_on_errors': True, 'record_on_errors': True}
assert data_sources.get_items(datasource) == []
assert emails.count() == 0
if pub.is_using_postgresql():
assert pub.loggederror_class.count() == 0
datasource = {'type': 'json', 'value': 'foo://bar', 'notify_on_errors': True}
datasource = {'type': 'json', 'value': 'foo://bar', 'notify_on_errors': True, 'record_on_errors': True}
assert data_sources.get_items(datasource) == []
assert 'Error loading JSON data source' in emails.get_latest('subject')
assert 'invalid scheme in URL' in emails.get_latest('subject')
if pub.is_using_postgresql():
assert pub.loggederror_class.count() == 1
logged_error = pub.loggederror_class.select()[0]
assert logged_error.workflow_id is None
assert (
logged_error.summary
== "[DATASOURCE] Exception: Error loading JSON data source (invalid scheme in URL foo://bar)"
)
datasource = {'type': 'json', 'value': '/bla/blo', 'notify_on_errors': True}
datasource = {'type': 'json', 'value': '/bla/blo', 'notify_on_errors': True, 'record_on_errors': True}
assert data_sources.get_items(datasource) == []
assert 'Error loading JSON data source' in emails.get_latest('subject')
assert 'invalid scheme in URL' in emails.get_latest('subject')
if pub.is_using_postgresql():
assert pub.loggederror_class.count() == 2
logged_error = pub.loggederror_class.select()[1]
assert logged_error.workflow_id is None
assert (
logged_error.summary
== "[DATASOURCE] Exception: Error loading JSON data source (invalid scheme in URL /bla/blo)"
)
def test_geojson_datasource(pub, requests_pub, http_requests):
@ -766,48 +848,108 @@ def test_geojson_datasource(pub, requests_pub, http_requests):
def test_geojson_datasource_bad_url(pub, http_requests, error_email, emails):
datasource = {'type': 'geojson', 'value': 'http://remote.example.net/404', 'notify_on_errors': True}
if pub.is_using_postgresql():
pub.loggederror_class.wipe()
datasource = {
'type': 'geojson',
'value': 'http://remote.example.net/404',
'notify_on_errors': True,
'record_on_errors': True,
}
assert data_sources.get_items(datasource) == []
assert 'Error loading JSON data source' in emails.get_latest('subject')
assert 'status: 404' in emails.get_latest('subject')
if pub.is_using_postgresql():
assert pub.loggederror_class.count() == 1
logged_error = pub.loggederror_class.select()[0]
assert logged_error.workflow_id is None
assert (
logged_error.summary
== "[DATASOURCE] Exception: Error loading JSON data source (error in HTTP request to http://remote.example.net/404 (status: 404))"
)
datasource = {'type': 'geojson', 'value': 'http://remote.example.net/xml', 'notify_on_errors': True}
datasource = {
'type': 'geojson',
'value': 'http://remote.example.net/xml',
'notify_on_errors': True,
'record_on_errors': True,
}
assert data_sources.get_items(datasource) == []
assert 'Error reading JSON data source output' in emails.get_latest('subject')
assert 'Expecting value:' in emails.get_latest('subject')
if pub.is_using_postgresql():
assert pub.loggederror_class.count() == 2
logged_error = pub.loggederror_class.select()[1]
assert logged_error.workflow_id is None
assert (
logged_error.summary
== "[DATASOURCE] Exception: Error reading JSON data source output (Expecting value: line 1 column 1 (char 0))"
)
datasource = {
'type': 'geojson',
'value': 'http://remote.example.net/connection-error',
'notify_on_errors': True,
'record_on_errors': True,
}
assert data_sources.get_items(datasource) == []
assert 'Error loading JSON data source' in emails.get_latest('subject')
assert 'error' in emails.get_latest('subject')
if pub.is_using_postgresql():
assert pub.loggederror_class.count() == 3
logged_error = pub.loggederror_class.select()[2]
assert logged_error.workflow_id is None
assert logged_error.summary == "[DATASOURCE] Exception: Error loading JSON data source (error)"
datasource = {
'type': 'geojson',
'value': 'http://remote.example.net/json-list-err1',
'notify_on_errors': True,
'record_on_errors': True,
}
assert data_sources.get_items(datasource) == []
assert 'Error reading JSON data source output (err 1)' in emails.get_latest('subject')
if pub.is_using_postgresql():
assert pub.loggederror_class.count() == 4
logged_error = pub.loggederror_class.select()[3]
assert logged_error.workflow_id is None
assert logged_error.summary == "[DATASOURCE] Exception: Error reading JSON data source output (err 1)"
def test_geojson_datasource_bad_url_scheme(pub, error_email, emails):
if pub.is_using_postgresql():
pub.loggederror_class.wipe()
datasource = {'type': 'geojson', 'value': ''}
assert data_sources.get_items(datasource) == []
assert emails.count() == 0
datasource = {'type': 'geojson', 'value': 'foo://bar', 'notify_on_errors': True}
datasource = {'type': 'geojson', 'value': 'foo://bar', 'notify_on_errors': True, 'record_on_errors': True}
assert data_sources.get_items(datasource) == []
assert 'Error loading JSON data source' in emails.get_latest('subject')
assert 'invalid scheme in URL' in emails.get_latest('subject')
if pub.is_using_postgresql():
assert pub.loggederror_class.count() == 1
logged_error = pub.loggederror_class.select()[0]
assert logged_error.workflow_id is None
assert (
logged_error.summary
== "[DATASOURCE] Exception: Error loading JSON data source (invalid scheme in URL foo://bar)"
)
datasource = {'type': 'geojson', 'value': '/bla/blo', 'notify_on_errors': True}
datasource = {'type': 'geojson', 'value': '/bla/blo', 'notify_on_errors': True, 'record_on_errors': True}
assert data_sources.get_items(datasource) == []
assert 'Error loading JSON data source' in emails.get_latest('subject')
assert 'invalid scheme in URL' in emails.get_latest('subject')
if pub.is_using_postgresql():
assert pub.loggederror_class.count() == 2
logged_error = pub.loggederror_class.select()[1]
assert logged_error.workflow_id is None
assert (
logged_error.summary
== "[DATASOURCE] Exception: Error loading JSON data source (invalid scheme in URL /bla/blo)"
)
def test_item_field_named_python_datasource(requests_pub):

8
wcs/api.py
View File

@ -1044,10 +1044,14 @@ class AutocompleteDirectory(Directory):
get_response().set_content_type('application/json')
try:
return misc.urlopen(url).read()
except ConnectionError:
except ConnectionError as e:
if 'data_source' in info:
error_summary = 'Error loading JSON data source (%s)' % str(e)
data_source = NamedDataSource.get(info['data_source'])
exc_info = sys.exc_info()
try:
raise Exception(error_summary) from e
except Exception:
exc_info = sys.exc_info()
get_publisher().notify_of_exception(
exc_info,
context='[DATASOURCE]',

25
wcs/logged_errors.py
View File

@ -78,11 +78,6 @@ class LoggedError:
elif workflow:
error.workflow_id = workflow.id
if not error.formdef_id and not error.workflow_id:
# cannot attach error to formdef or workflow, don't record in journal, it will
# still be sent by email to administrators.
return
if status_item:
error.status_item_id = status_item.id
if getattr(status_item, 'parent', None):
@ -108,17 +103,17 @@ class LoggedError:
return
formdata_id = context.get('form_number_raw')
formdef_urlname = context.get('form_slug')
if not formdef_urlname:
# cannot attach error to formdef, don't record in journal, it will
# still be sent by email to administrators.
return
klass = FormDef
if context.get('form_class_name') == 'CardDef':
klass = CardDef
formdef = klass.get_by_urlname(formdef_urlname)
formdata = formdef.data_class().get(formdata_id, ignore_errors=True)
if formdef_urlname:
klass = FormDef
if context.get('form_class_name') == 'CardDef':
klass = CardDef
formdef = klass.get_by_urlname(formdef_urlname)
formdata = formdef.data_class().get(formdata_id, ignore_errors=True)
workflow = formdef.workflow
else:
formdef = formdata = workflow = None
return cls.record(
error_summary, plain_error_msg, formdata=formdata, formdef=formdef, workflow=formdef.workflow
error_summary, plain_error_msg, formdata=formdata, formdef=formdef, workflow=workflow
)
def build_tech_id(self):

2
wcs/wf/geolocate.py
View File

@ -133,7 +133,7 @@ class GeolocateWorkflowStatusItem(WorkflowStatusItem):
def geolocate_address_string(self, formdata, compute_template=True):
if compute_template:
try:
address = self.compute(self.address_string, raises=True)
address = self.compute(self.address_string, record_errors=False, raises=True)
except Exception as e:
get_publisher().record_error(
_('error in template for address string [%s]') % str(e), formdata=formdata, exception=e