general: record formdef/workflow exceptions (#12566)

tests/test_form_pages.py

@@ -30,6 +30,7 @@ from wcs.tracking_code import TrackingCode
 from wcs.data_sources import NamedDataSource
 from wcs import fields
 from wcs.sessions import BasicSession
+from wcs.logged_errors import LoggedError
 
 from utilities import get_app, login, create_temporary_pub, clean_temporary_pub
 
@@ -3523,3 +3524,48 @@ def test_form_data_keywords(pub):
     resp = resp.follow()
     assert 'The form has been recorded' in resp.body
     assert formdef.data_class().count() == 1
+
+def test_logged_errors(pub):
+    Workflow.wipe()
+    workflow = Workflow.get_default_workflow()
+    workflow.id = '12'
+    jump = JumpWorkflowStatusItem()
+    jump.id = '_jump'
+    jump.status = 'rejected'
+    jump.condition = '1/0' # ZeroDivisionError
+    st1 = workflow.possible_status[0]
+    st1.items.insert(0, jump)
+    jump.parent = st1
+    workflow.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.id = '34'
+    formdef.workflow = workflow
+    formdef.name = 'test'
+    formdef.confirmation = False
+    formdef.fields = []
+    formdef.store()
+
+    LoggedError.wipe()
+
+    app = get_app(pub)
+    resp = app.get('/test/')
+    resp = resp.form.submit('submit').follow()
+    resp = resp.form.submit('submit')
+    assert LoggedError.count() == 1
+
+    resp = app.get('/test/')
+    resp = resp.form.submit('submit').follow()
+    resp = resp.form.submit('submit')
+    assert LoggedError.count() == 1
+
+    error = LoggedError.get_on_index(
+            '34-12-zerodivisionerror-integer-division-or-modulo-by-zero', 'tech_id')
+    assert error.occurences_count == 2
+
+    assert len(LoggedError.get_ids_with_indexed_value('formdef_id', '34')) == 1
+    assert len(LoggedError.get_ids_with_indexed_value('formdef_id', 'X')) == 0
+
+    assert len(LoggedError.get_ids_with_indexed_value('workflow_id', '12')) == 1
+    assert len(LoggedError.get_ids_with_indexed_value('workflow_id', 'X')) == 0

wcs/logged_errors.py

@@ -0,0 +1,94 @@
+# w.c.s. - web application for online forms
+# Copyright (C) 2005-2017  Entr'ouvert
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+import datetime
+
+from qommon.misc import simplify
+from qommon.xml_storage import XmlStorableObject
+from wcs.formdef import FormDef
+from wcs.workflows import Workflow
+
+class LoggedError(XmlStorableObject):
+    _names = 'logged-errors'
+    _xml_tagname = 'error'
+    _indexes = ['tech_id']
+    _hashed_indexes = ['formdef_id', 'workflow_id']
+
+    summary = None
+    formdata_id = None
+    formdef_id = None
+    workflow_id = None
+    traceback = None
+    occurences_count = 0
+    first_occurence_timestamp = None
+    latest_occurence_timestamp = None
+    acked = False
+
+    # declarations for serialization
+    XML_NODES = [
+            ('summary', 'str'), ('traceback', 'str'),
+            ('formdata_id', 'str'), ('formdef_id', 'str'), ('workflow_id', 'str'),
+            ('occurences_count', 'int'),
+            ('first_occurence_timestamp', 'datetime'),
+            ('latest_occurence_timestamp', 'datetime'),
+            ('acked', 'bool')]
+
+    @classmethod
+    def record(cls, error_summary, plain_error_msg, publisher):
+        error = cls()
+        error.summary = error_summary
+        error.traceback = plain_error_msg
+        try:
+            context = publisher.substitutions.get_context_variables()
+        except:
+            return
+
+        formdef_urlname = context.get('form_slug')
+        if not formdef_urlname:
+            # cannot attach error to formdef, don't record in journal, it will
+            # still be sent by email to administrators.
+            return
+
+        error.formdata_id = context.get('form_number_raw')
+        if formdef_urlname:
+            formdef = FormDef.get_by_urlname(formdef_urlname)
+            error.formdef_id = formdef.id
+            error.workflow_id = formdef.workflow_id
+
+        error.first_occurence_timestamp = datetime.datetime.now()
+        error.id = '%s-%s' % (
+                error.first_occurence_timestamp.strftime('%Y%m%d-%H%M%S'),
+                error.tech_id,
+                )
+        existing_error = cls.get_on_index(error.tech_id, 'tech_id', ignore_errors=True)
+        if existing_error:
+            error = existing_error
+        error.occurences_count += 1
+        error.latest_occurence_timestamp = datetime.datetime.now()
+        error.store()
+
+    @property
+    def tech_id(self):
+        return '%s-%s-%s' % (self.formdef_id, self.workflow_id, simplify(self.summary))
+
+    def get_formdef(self):
+        return FormDef.get(self.formdef_id, ignore_errors=True)
+
+    def get_workflow(self):
+        return Workflow.get(self.workflow_id, ignore_errors=True)
+
+    def get_formdata(self):
+        return self.get_formdef().data_class().get(self.formdata_id, ignore_errors=True)

wcs/publisher.py

@@ -45,6 +45,7 @@ from qommon.cron import CronJob
 
 from users import User
 from tracking_code import TrackingCode
+from logged_errors import LoggedError
 
 import pickle
 
@@ -281,6 +282,12 @@ class WcsPublisher(StubWcsPublisher):
         conn.commit()
         cur.close()
 
+    def log_internal_error(self, error_summary, plain_error_msg, record=False):
+        super(WcsPublisher, self).log_internal_error(error_summary,
+                plain_error_msg, record=record)
+        if record:
+            LoggedError.record(error_summary, plain_error_msg, publisher=self)
+
     def apply_global_action_timeouts(self):
         from wcs.workflows import Workflow, WorkflowGlobalActionTimeoutTrigger
         for workflow in Workflow.select():

wcs/qommon/publisher.py

@@ -243,15 +243,18 @@ class QommonPublisher(Publisher, object):
         if context:
             error_summary = '%s %s' % (context, error_summary)
 
-        plain_error_msg = self._generate_plaintext_error(
+        plain_error_msg = str(self._generate_plaintext_error(
                         get_request(),
                         self,
                         exc_type, exc_value,
-                        tb)
+                        tb))
 
         self.notify_sentry(exc_tuple, request=self.get_request(),
                 context=context)
 
+        self.log_internal_error(error_summary, plain_error_msg, record=True)
+
+    def log_internal_error(self, error_summary, plain_error_msg, record=False):
         try:
             self.logger.log_internal_error(error_summary, plain_error_msg)
         except socket.error:

wcs/qommon/xml_storage.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+import datetime
 import xml.etree.ElementTree as ET
 
 from quixote import get_publisher
@@ -57,6 +58,12 @@ class XmlStorableObject(StorableObject):
     def export_int_to_xml(self, element, attribute_name, charset):
         element.text = str(getattr(self, attribute_name))
 
+    def export_bool_to_xml(self, element, attribute_name, charset):
+        element.text = 'true' if getattr(self, attribute_name) else 'false'
+
+    def export_datetime_to_xml(self, element, attribute_name, charset):
+        element.text = getattr(self, attribute_name).isoformat()
+
     def export_to_xml_string(self, include_id=False):
         x = self.export_to_xml(include_id=include_id)
         indent_xml(x)
@@ -100,3 +107,9 @@ class XmlStorableObject(StorableObject):
 
     def import_int_from_xml(self, element, charset):
         return int(element.text)
+
+    def import_bool_from_xml(self, element, charset):
+        return bool(element.text == 'true')
+
+    def import_datetime_from_xml(self, element, charset):
+        return datetime.datetime.strptime(element.text[:19], '%Y-%m-%dT%H:%M:%S')