tests/admin_pages/test_tests.py

@@ -371,6 +371,83 @@ def test_tests_edit_data(pub):
     assert 'test 2' in resp.text
 
 
+def test_tests_edit_data_mark_as_failing(pub):
+    create_superuser(pub)
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.fields = [
+        fields.PageField(
+            id='0',
+            label='1st page',
+            type='page',
+            post_conditions=[
+                {
+                    'condition': {'type': 'django', 'value': 'form_var_text|length > 5'},
+                    'error_message': 'Not enough chars.',
+                }
+            ],
+        ),
+        fields.StringField(id='1', label='Text', varname='text', validation={'type': 'digits'}),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.data['1'] = '12345'
+    formdata.store()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.name = 'First test'
+    testdef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
+    resp = resp.click('Edit data')
+    assert 'Mark as failing' not in resp.text
+
+    resp.form['f1'] = '123456'
+    resp = resp.form.submit('submit').follow()
+    assert '123456' in resp.text
+
+    resp = resp.click('Edit data')
+    assert 'Mark as failing' not in resp.text
+
+    # two errors on page
+    resp.form['f1'] = '123a'
+    resp = resp.form.submit('submit')
+    assert 'Mark as failing' not in resp.text
+
+    # one error
+    resp.form['f1'] = '1234'
+    resp = resp.form.submit('submit')
+    assert 'If test should fail on error "Not enough chars.", click button below.' in resp.text
+    assert 'Mark as failing' in resp.text
+
+    # other error
+    resp.forms[0]['f1'] = 'abcdefg'
+    resp = resp.forms[0].submit('submit')
+    assert 'If test should fail on error "Only digits are allowed", click button below.' in resp.text
+    assert 'Mark as failing' in resp.text
+
+    # click mark as failing button
+    resp = resp.forms[1].submit().follow()
+    assert 'abcdefg' in resp.text
+    assert escape('This test is expected to fail on error "Only digits are allowed".') in resp.text
+
+    resp = resp.click('Edit data')
+    assert 'This test is expected to fail on error "Only digits are allowed".' in resp.text
+
+    resp.form['f1'] = '1234567'
+    resp = resp.form.submit('submit').follow()
+    assert 'This test is expected to fail' not in resp.text
+
+    # only post is allowed
+    app.get('/backoffice/forms/1/tests/%s/edit-data/mark-as-failing' % testdef.id, status=404)
+
+
 def test_tests_manual_run(pub):
     user = create_superuser(pub)
 

tests/test_testdef.py

@@ -876,3 +876,52 @@ def test_computed_field_value_too_long(pub):
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
     testdef.run(formdef)
+
+
+def test_expected_error(pub):
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.fields = [
+        fields.PageField(
+            id='0',
+            label='1st page',
+            type='page',
+            post_conditions=[
+                {
+                    'condition': {'type': 'django', 'value': 'form_var_text|length > 5'},
+                    'error_message': 'Not enough chars.',
+                }
+            ],
+        ),
+        fields.StringField(id='1', label='Text', varname='text', validation={'type': 'digits'}),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.data['1'] = '123456'
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.run(formdef)
+
+    formdata.data['1'] = '1'
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    with pytest.raises(TestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Page 1 post condition was not met (form_var_text|length > 5).'
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.data['expected_error'] = 'Not enough chars.'
+    testdef.run(formdef)
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.data['expected_error'] = 'Other error.'
+    with pytest.raises(TestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Expected error "Other error." but got error "Not enough chars." instead.'
+
+    formdata.data['1'] = 'abcdef'
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.data['expected_error'] = 'Only digits are allowed'
+    testdef.run(formdef)

wcs/admin/tests.py

@@ -34,6 +34,72 @@ from wcs.qommon.storage import Equal, StrictNotEqual
 from wcs.testdef import TestDef, TestError, TestResult
 
 
+class TestEditPage(FormBackofficeEditPage):
+    filling_templates = ['wcs/backoffice/testdata_filling.html']
+
+    def __init__(self, *args, testdef, filled, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.testdef = testdef
+        self.edited_data = filled
+        self.edited_data.data['edited_testdef_id'] = self.testdef.id
+        self._q_exports.append(('mark-as-failing', 'mark_as_failing'))
+
+    def _q_index(self):
+        get_response().breadcrumb.append(('edit-data/', _('Edit data')))
+        return super()._q_index()
+
+    def modify_filling_context(self, context, *args, **kwargs):
+        super().modify_filling_context(context, *args, **kwargs)
+
+        form = context['html_form']
+        if form.get_submit() == 'submit':
+            self.testdef.data['expected_error'] = None
+
+        get_response().filter['sidebar'] = self.get_test_sidebar(form)
+
+    def get_test_sidebar(self, form):
+        context = {'testdef': self.testdef, 'mark_as_failing_form': self.get_mark_as_failing_form(form)}
+        return render_to_string('wcs/backoffice/test_edit_sidebar.html', context=context)
+
+    def get_mark_as_failing_form(self, form):
+        errors = form.global_error_messages or []
+
+        if not errors and not form.has_errors():
+            return
+
+        for widget in form.widgets:
+            widget = TestDef.get_error_widget(widget)
+            if widget:
+                errors.append(widget.error)
+
+        if len(errors) != 1:
+            return
+
+        form = Form(enctype='multipart/form-data', action='mark-as-failing', use_tokens=False)
+        form.add_hidden('error', errors[0])
+        form.test_error = errors[0]
+
+        magictoken = get_request().form.get('magictoken')
+        form.add_hidden('magictoken', magictoken)
+
+        form.add_submit('submit', _('Mark as failing'))
+        return form
+
+    def mark_as_failing(self):
+        if not get_request().get_method() == 'POST':
+            raise TraversalError()
+
+        magictoken = get_request().form.get('magictoken')
+        edited_data = self.get_transient_formdata(magictoken)
+
+        testdef = TestDef.create_from_formdata(self.formdef, edited_data)
+        self.testdef.data = testdef.data
+
+        self.testdef.data['expected_error'] = get_request().form.get('error')
+        self.testdef.store()
+        return redirect('..')
+
+
 class TestPage(FormBackOfficeStatusPage):
     _q_extra_exports = ['delete', 'export', 'edit', ('edit-data', 'edit_data')]
 
@@ -46,6 +112,12 @@ class TestPage(FormBackOfficeStatusPage):
         filled = self.testdef.build_formdata(objectdef, include_fields=True)
         super().__init__(objectdef, filled)
 
+    @property
+    def edit_data(self):
+        return TestEditPage(
+            self.formdef.url_name, update_breadcrumbs=False, testdef=self.testdef, filled=self.filled
+        )
+
     def _q_index(self):
         get_response().add_javascript(['select2.js'])
         return super()._q_index()
@@ -68,9 +140,13 @@ class TestPage(FormBackOfficeStatusPage):
         r += htmltext('<div id="appbar">')
         r += htmltext('<h2>%s</h2>') % self.testdef
         r += htmltext('<span class="actions">')
-        r += htmltext('<a href="edit-data">%s</a>') % _('Edit data')
+        r += htmltext('<a href="edit-data/">%s</a>') % _('Edit data')
         r += htmltext('</span>')
         r += htmltext('</div>')
+        if self.testdef.data.get('expected_error'):
+            r += htmltext('<div class="infonotice"><p>%s</p></div>') % _(
+                'This test is expected to fail on error "%s".' % self.testdef.data['expected_error']
+            )
         r += self.receipt(always_include_user=True, mine=False)
         return r.getvalue()
 
@@ -98,16 +174,6 @@ class TestPage(FormBackOfficeStatusPage):
         )
         return json.dumps(self.testdef.export_to_json())
 
-    def edit_data(self):
-        self.filled.data['edited_testdef_id'] = self.testdef.id
-
-        f = FormBackofficeEditPage(self.formdef.url_name)
-        f.testdef = self.testdef
-        f.edited_data = self.filled
-        f.action_url = 'edit-data'
-
-        return f._q_index()
-
     def edit(self):
         form = Form(enctype='multipart/form-data')
         form.add(StringWidget, 'name', title=_('Name'), required=True, size=50, value=self.testdef.name)

wcs/forms/root.py

@@ -282,7 +282,7 @@ class FormPage(FormdefDirectoryBase, FormTemplateMixin):
     formdef_class = FormDef
     preview_mode = False
 
-    def __init__(self, component, parent_category=None):
+    def __init__(self, component, parent_category=None, update_breadcrumbs=True):
         try:
             self.formdef = self.formdef_class.get_by_urlname(component)
         except KeyError:
@@ -300,7 +300,8 @@ class FormPage(FormdefDirectoryBase, FormTemplateMixin):
         self.on_validation_page = False
         self.current_page = None
         self.user = get_request().user
-        get_response().breadcrumb.append((component + '/', get_publisher().translate(self.formdef.name)))
+        if update_breadcrumbs:
+            get_response().breadcrumb.append((component + '/', get_publisher().translate(self.formdef.name)))
 
     def __call__(self):
         # add missing trailing slash.
@@ -720,12 +721,12 @@ class FormPage(FormdefDirectoryBase, FormTemplateMixin):
             'formdef': LazyFormDef(self.formdef),
             'form_side': self.form_side(data=data, magictoken=magictoken),
             'steps': self.step,
+            'html_form': form,
             # legacy, used in some themes
             'tracking_code_box': lambda: self.tracking_code_box(data, magictoken),
         }
         self.modify_filling_context(context, page, data)
 
-        context['html_form'] = form
         if self.is_popup:
             return template.QommonTemplateResponse(
                 templates=list(self.get_formdef_template_variants(self.popup_filling_templates)),

wcs/templates/wcs/backoffice/test_edit_sidebar.html

@@ -0,0 +1,21 @@
+{% load i18n %}
+
+<div id="mark-as-failing">
+  <h3>{% trans "Mark test as failing" %}</h3>
+  {% if testdef.data.expected_error %}
+    <p>
+      {% blocktrans trimmed with error=testdef.data.expected_error %}
+        This test is expected to fail on error "{{ error }}". Submitting the form will mark it as passing again.
+      {% endblocktrans %}
+    </p>
+  {% elif mark_as_failing_form %}
+    <p>
+      {% blocktrans trimmed with error=mark_as_failing_form.test_error %}
+        If test should fail on error "{{ error }}", click button below.
+      {% endblocktrans %}
+    </p>
+    {{ mark_as_failing_form.render|safe }}
+  {% else %}
+    <p>{% trans "In order to mark test as failing, form must display exactly one error." %}</p>
+  {% endif %}
+</div>

wcs/templates/wcs/backoffice/testdata_filling.html

@@ -0,0 +1,4 @@
+{% extends "wcs/backoffice/formdata_filling.html" %}
+{% load i18n %}
+
+{% block appbar-title %}{% trans "Edit test data" %}{% endblock %}

wcs/testdef.py

@@ -32,7 +32,9 @@ from .qommon.storage import Equal
 
 
 class TestError(Exception):
-    pass
+    def __init__(self, msg, error=None):
+        self.msg = msg
+        self.error = error
 
 
 class TestDef(sql.TestDef):
@@ -121,8 +123,24 @@ class TestDef(sql.TestDef):
             get_publisher()._set_request(true_request)
 
     def run(self, objectdef):
+        expected_error = self.data.get('expected_error')
         with self.fake_request():
-            self._run(objectdef)
+            try:
+                self._run(objectdef)
+            except TestError as e:
+                if not expected_error:
+                    raise e
+
+                if e.error != expected_error:
+                    raise TestError(
+                        _('Expected error "%(expected_error)s" but got error "%(error)s" instead.')
+                        % {'expected_error': expected_error, 'error': e.error}
+                    )
+            else:
+                if expected_error:
+                    raise TestError(
+                        _('Expected error "%s" but test completed with success.') % expected_error
+                    )
 
     def _run(self, objectdef):
         formdata = self.build_formdata(objectdef)
@@ -223,7 +241,8 @@ class TestDef(sql.TestDef):
                 if not Field.evaluate_condition(formdata.data, objectdef, condition):
                     raise TestError(
                         _('Page %(no)d post condition was not met (%(condition)s).')
-                        % {'no': page.index, 'condition': condition.get('value')}
+                        % {'no': page.index, 'condition': condition.get('value')},
+                        error=post_condition.get('error_message'),
                     )
             except RuntimeError:
                 raise TestError(_('Failed to evaluate page %d post condition.') % page.index)
@@ -240,15 +259,11 @@ class TestDef(sql.TestDef):
         widget._parsed = False
         widget.parse()
 
-        if widget.has_error():
+        widget = self.get_error_widget(widget)
+        if widget:
             field_label = _('"%s"') % field.label
-            if field.key == 'block' and (not widget.error or widget.error == widget.REQUIRED_ERROR):
-                widget.error = None
-
-                widget = self.get_error_subwidget(widget)
-                if not widget:
-                    return
 
+            if getattr(widget, 'is_subwidget', False):
                 value = widget.value
                 field = widget.field
                 field_label = _('"%(subfield)s" (of field %(field)s)') % {
@@ -256,9 +271,6 @@ class TestDef(sql.TestDef):
                     'field': field_label,
                 }
 
-            if widget.error == get_selection_error_text():
-                return
-
             if field.convert_value_to_str:
                 value = field.convert_value_to_str(value)
 
@@ -269,7 +281,8 @@ class TestDef(sql.TestDef):
                     'error': error_msg,
                     'label': field_label,
                     'details': widget.error,
-                }
+                },
+                error=widget.error,
             )
 
     def handle_computed_fields(self, fields, formdata, exclude_frozen=False):
@@ -293,13 +306,28 @@ class TestDef(sql.TestDef):
                 formdata.data[field.id] = value
                 get_publisher().substitutions.invalidate_cache()
 
-    def get_error_subwidget(self, widget):
+    @classmethod
+    def get_error_widget(cls, widget):
+        if not widget.has_error():
+            return
+
+        if widget.field.key == 'block' and (not widget.error or widget.error == widget.REQUIRED_ERROR):
+            widget.error = None
+            return cls.get_error_subwidget(widget)
+
+        if widget.error != get_selection_error_text():
+            return widget
+
+    @staticmethod
+    def get_error_subwidget(widget):
         for widget in widget.get_widgets():
+            widget.is_subwidget = True
+
             if widget.error and widget.error != get_selection_error_text():
                 return widget
 
             if hasattr(widget, 'get_widgets'):
-                widget = self.get_error_subwidget(widget)
+                widget = TestDef.get_error_subwidget(widget)
                 if widget:
                     return widget