backoffice: display drafts stats (#72542)

tests/admin_pages/test_form.py

@@ -11,7 +11,7 @@ import responses
 from pyquery import PyQuery
 from webtest import Upload
 
-from wcs import fields
+from wcs import fields, workflow_tests
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.categories import CardDefCategory, Category, DataSourceCategory, WorkflowCategory
@@ -20,7 +20,8 @@ from wcs.formdef import FormDef
 from wcs.qommon.afterjobs import AfterJob
 from wcs.qommon.errors import ConnectionError
 from wcs.qommon.http_request import HTTPRequest
-from wcs.testdef import TestDef, TestResult
+from wcs.testdef import TestDef, TestResult, WebserviceResponse
+from wcs.workflow_tests import WorkflowTests
 from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef, WorkflowVariablesFieldsFormDef
 from wcs.wscalls import NamedWsCall
 
@@ -1495,6 +1496,63 @@ def test_form_duplicate(pub):
     assert FormDef.get(3).name == 'other copy'
 
 
+def test_form_duplicate_with_tests(pub):
+    create_superuser(pub)
+    create_role(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.store()
+
+    TestDef.wipe()
+    testdef = TestDef.create_from_formdata(formdef, formdef.data_class()())
+    testdef.workflow_tests.actions = [
+        workflow_tests.ButtonClick(id='1', button_name='xxx'),
+    ]
+    testdef.name = 'First test'
+    testdef.store()
+
+    response = WebserviceResponse()
+    response.testdef_id = testdef.id
+    response.name = 'First response'
+    response.store()
+
+    testdef = TestDef.create_from_formdata(formdef, formdef.data_class()())
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertStatus(id='1', status_name='yyy'),
+    ]
+    testdef.name = 'Second test'
+    testdef.store()
+
+    response = WebserviceResponse()
+    response.testdef_id = testdef.id
+    response.name = 'Second response'
+    response.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/1/')
+
+    resp = resp.click(href='duplicate')
+    resp = resp.form.submit('submit').follow()
+    assert FormDef.count() == 2
+    assert TestDef.count() == 4
+    assert WorkflowTests.count() == 4
+    assert WebserviceResponse.count() == 4
+
+    new_formdef = FormDef.get(2)
+    assert new_formdef.name == 'form title (copy)'
+
+    testdef1, testdef2 = TestDef.select_for_objectdef(new_formdef)
+    assert testdef1.name == 'First test'
+    assert testdef2.name == 'Second test'
+
+    assert testdef1.workflow_tests.actions[0].button_name == 'xxx'
+    assert testdef2.workflow_tests.actions[0].status_name == 'yyy'
+    assert testdef1.get_webservice_responses()[0].name == 'First response'
+    assert testdef2.get_webservice_responses()[0].name == 'Second response'
+
+
 def test_form_export(pub):
     create_superuser(pub)
     create_role(pub)
@@ -1624,6 +1682,90 @@ def test_form_import_from_url(pub):
     assert FormDef.count() == 1
 
 
+def test_form_import_with_tests(pub):
+    create_superuser(pub)
+    create_role(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.store()
+
+    TestDef.wipe()
+    testdef = TestDef.create_from_formdata(formdef, formdef.data_class()())
+    testdef.workflow_tests.actions = [
+        workflow_tests.ButtonClick(id='1', button_name='xxx'),
+    ]
+    testdef.name = 'First test'
+    testdef.store()
+
+    response = WebserviceResponse()
+    response.testdef_id = testdef.id
+    response.name = 'First response'
+    response.store()
+
+    testdef = TestDef.create_from_formdata(formdef, formdef.data_class()())
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertStatus(id='1', status_name='yyy'),
+    ]
+    testdef.name = 'Second test'
+    testdef.store()
+
+    response = WebserviceResponse()
+    response.testdef_id = testdef.id
+    response.name = 'Second response'
+    response.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get('/backoffice/forms/1/')
+    export_resp = resp.click(href='export')
+
+    FormDef.wipe()
+    TestDef.wipe()
+    WebserviceResponse.wipe()
+
+    resp = app.get('/backoffice/forms/import')
+    resp.forms[0]['file'] = Upload('formdef.wcs', export_resp.body)
+    resp = resp.forms[0].submit()
+
+    assert FormDef.count() == 1
+    formdef = FormDef.get(1)
+
+    testdef1, testdef2 = TestDef.select_for_objectdef(formdef)
+    assert testdef1.name == 'First test'
+    assert testdef2.name == 'Second test'
+
+    # import the same formdef a second time
+    resp = app.get('/backoffice/forms/import')
+    resp.forms[0]['file'] = Upload('formdef.wcs', export_resp.body)
+    resp = resp.forms[0].submit()
+
+    assert FormDef.count() == 2
+    assert TestDef.count() == 4
+    formdef2 = formdef.get(2)
+
+    testdef1, testdef2 = TestDef.select_for_objectdef(formdef2)
+    assert testdef1.name == 'First test'
+    assert testdef2.name == 'Second test'
+
+    assert testdef1.workflow_tests.actions[0].button_name == 'xxx'
+    assert testdef2.workflow_tests.actions[0].status_name == 'yyy'
+    assert testdef1.get_webservice_responses()[0].name == 'First response'
+    assert testdef2.get_webservice_responses()[0].name == 'Second response'
+
+    TestDef.remove_object(testdef1.id)
+    assert TestDef.count() == 3
+
+    # overwrite doesn't impact tests
+    resp = app.get('/backoffice/forms/2/')
+    resp = resp.click(href='overwrite')
+    resp.forms[0]['file'] = Upload('formdef.wcs', export_resp.body)
+    resp = resp.forms[0].submit()
+
+    assert TestDef.count() == 3
+
+
 def test_form_qrcode(pub):
     create_superuser(pub)
     create_role(pub)
@@ -4528,6 +4670,76 @@ def test_admin_form_inspect_validation(pub):
     assert not resp.pyquery('[data-field-id="4"] .parameter-validation').length
 
 
+def test_admin_form_inspect_drafts(pub):
+    create_superuser(pub)
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.enable_tracking_codes = True
+    formdef.fields = [
+        fields.PageField(id='0', label='1st page'),
+        fields.StringField(id='1', label='string 1'),
+        fields.PageField(id='2', label='2nd page'),
+        fields.StringField(id='3', label='string 2'),
+        fields.PageField(id='4', label='3rd page'),
+        fields.StringField(id='5', label='string 3'),
+    ]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/%s/inspect' % formdef.id)
+    assert resp.pyquery('#inspect-drafts p').text() == 'No drafts found for this form'
+
+    data_class = formdef.data_class()
+    formdata = data_class()
+    formdata.status = 'draft'
+    formdata.page_id = '0'
+    formdata.store()
+    formdata = data_class()
+    formdata.status = 'draft'
+    formdata.page_id = '2'
+    formdata.store()
+    formdata = data_class()
+    formdata.status = 'draft'
+    formdata.page_id = '4'
+    formdata.store()
+    formdata = data_class()
+    formdata.status = 'draft'
+    formdata.page_id = '_confirmation_page'
+    formdata.store()
+    formdata = data_class()
+    formdata.status = 'draft'
+    formdata.page_id = 'xxxx'  # unkown page id
+    formdata.store()
+
+    resp = app.get('/backoffice/forms/%s/inspect' % formdef.id)
+    assert resp.pyquery('div#inspect-drafts tr#0').length == 1
+    assert resp.pyquery('div#inspect-drafts tr#0 td.label').text() == '1st page'
+    assert resp.pyquery('div#inspect-drafts tr#0 td.percent').text() == '20.0\xa0%'
+    assert resp.pyquery('div#inspect-drafts tr#0 td.total').text() == '(1/5)'
+
+    assert resp.pyquery('div#inspect-drafts tr#2').length == 1
+    assert resp.pyquery('div#inspect-drafts tr#2 td.label').text() == '2nd page'
+    assert resp.pyquery('div#inspect-drafts tr#2 td.percent').text() == '20.0\xa0%'
+    assert resp.pyquery('div#inspect-drafts tr#2 td.total').text() == '(1/5)'
+
+    assert resp.pyquery('div#inspect-drafts tr#4').length == 1
+    assert resp.pyquery('div#inspect-drafts tr#4 td.label').text() == '3rd page'
+    assert resp.pyquery('div#inspect-drafts tr#4 td.percent').text() == '20.0\xa0%'
+    assert resp.pyquery('div#inspect-drafts tr#4 td.total').text() == '(1/5)'
+
+    assert resp.pyquery('div#inspect-drafts tr#_confirmation_page').length == 1
+    assert resp.pyquery('div#inspect-drafts tr#_confirmation_page td.label').text() == 'Confirmation page'
+    assert resp.pyquery('div#inspect-drafts tr#_confirmation_page td.percent').text() == '20.0\xa0%'
+    assert resp.pyquery('div#inspect-drafts tr#_confirmation_page td.total').text() == '(1/5)'
+
+    assert resp.pyquery('div#inspect-drafts tr#_unkown').length == 1
+    assert resp.pyquery('div#inspect-drafts tr#_unkown td.label').text() == 'Unkown'
+    assert resp.pyquery('div#inspect-drafts tr#_unkown td.percent').text() == '20.0\xa0%'
+    assert resp.pyquery('div#inspect-drafts tr#_unkown td.total').text() == '(1/5)'
+
+
 def test_form_import_fields(pub):
     create_superuser(pub)
     create_role(pub)

tests/admin_pages/test_tests.py

@@ -1,18 +1,20 @@
 import datetime
-import json
 import os
 
 import pytest
 from django.utils.html import escape
+from django.utils.timezone import make_aware
 from webtest import Upload
 
-from wcs import fields
+from wcs import fields, workflow_tests
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.formdef import FormDef
 from wcs.qommon.http_request import HTTPRequest
 from wcs.qommon.upload_storage import PicklableUpload
-from wcs.testdef import TestDef, TestResult
+from wcs.testdef import TestDef, TestResult, WebserviceResponse
+from wcs.workflow_tests import WorkflowTests
+from wcs.wscalls import NamedWsCall
 
 from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
 from .test_all import create_superuser
@@ -30,6 +32,8 @@ def pub():
     FormDef.wipe()
     TestDef.wipe()
     TestResult.wipe()
+    WorkflowTests.wipe()
+    WebserviceResponse.wipe()
     return pub
 
 
@@ -38,7 +42,7 @@ def teardown_module(module):
 
 
 def test_tests_page(pub):
-    create_superuser(pub)
+    user = create_superuser(pub)
 
     formdef = FormDef()
     formdef.name = 'test title'
@@ -54,7 +58,12 @@ def test_tests_page(pub):
 
     resp = resp.click('New')
     resp.form['name'] = 'First test'
-    resp = resp.form.submit().follow()
+    resp = resp.form.submit()
+
+    testdef = TestDef.select()[0]
+    assert testdef.agent_id == str(user.id)
+
+    resp = resp.follow()
     assert 'Edit test data' in resp.text
 
     resp.form['f1'] = 'abcdefg'
@@ -103,7 +112,7 @@ def test_tests_page_creation_from_formdata(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = 'abcdefg'
     formdata.user_id = user.id
     formdata.store()
@@ -118,11 +127,12 @@ def test_tests_page_creation_from_formdata(pub):
 
     testdef = TestDef.select()[0]
     assert testdef.data['user']['id'] == 1
+    assert testdef.agent_id == str(user.id)
     assert not testdef.is_in_backoffice
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2022, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2022, 1, 1, 0, 0))
     formdata.data['1'] = 'hijklmn'
     formdata.backoffice_submission = True
     formdata.store()
@@ -167,51 +177,74 @@ def test_tests_import_export(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = 'a'
     formdata.user_id = user.id
     formdata.store()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.workflow_tests.actions = [
+        workflow_tests.ButtonClick(id='1', button_name='Go to end status'),
+        workflow_tests.AssertStatus(id='2', status_name='End status'),
+    ]
     testdef.name = 'First test'
     testdef.store()
 
+    response = WebserviceResponse()
+    response.testdef_id = testdef.id
+    response.name = 'Response xxx'
+    response.store()
+
     app = login(get_app(pub))
 
     resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
     export_resp = resp.click('Export')
+    assert 'filename=test-first-test.wcs' in export_resp.headers['content-disposition']
 
     resp = resp.click('Delete')
     resp = resp.form.submit().follow()
     assert 'First test' not in resp.text
+    assert WorkflowTests.count() == 0
+    assert WebserviceResponse.count() == 0
 
     resp = resp.click('Import')
-    resp.form['file'] = Upload('export.json', export_resp.body, 'application/json')
+    resp.form['file'] = Upload('export.wcs', export_resp.body)
     resp = resp.form.submit().follow()
     assert TestDef.count() == 1
+    assert WorkflowTests.count() == 1
+    assert WebserviceResponse.count() == 1
     assert 'First test' in resp.text
     assert escape('Test "First test" has been successfully imported.') in resp.text
 
     imported_testdef = TestDef.select()[0]
-    assert imported_testdef.export_to_json() == testdef.export_to_json()
-
-    export_json = json.loads(export_resp.body)
-    export_json['data']['fields']['1'] = 'b'
+    assert imported_testdef.name == testdef.name
+    assert imported_testdef.data == testdef.data
 
     resp = resp.click('Import')
-    resp.form['file'] = Upload('export.json', json.dumps(export_json).encode(), 'application/json')
+    resp.form['file'] = Upload('export.wcs', export_resp.body)
     resp = resp.form.submit().follow()
-    assert TestDef.count() == 1
-    assert 'First test' in resp.text
+    assert TestDef.count() == 2
+    assert WorkflowTests.count() == 2
+    assert WebserviceResponse.count() == 2
+    assert len(resp.pyquery('li a:contains("First test")')) == 2
     assert escape('Test "First test" has been successfully imported.') in resp.text
 
-    imported_testdef = TestDef.get(imported_testdef.id)
-    assert imported_testdef.data['fields']['1'] == 'b'
-
     resp = resp.click('Import')
-    resp.form['file'] = Upload('export.json', b'invalid', 'application/json')
+    resp.form['file'] = Upload('export.wcs', b'invalid')
     resp = resp.form.submit()
-    assert 'Expecting value: line 1 column 1' in resp.text
+    assert 'Invalid File' in resp.text
+
+    formdef2 = FormDef()
+    formdef2.name = 'test title'
+    formdef2.store()
+
+    resp = app.get('/backoffice/forms/%s/tests/' % formdef2.id)
+    resp = resp.click('Import')
+
+    resp.form['file'] = Upload('export.wcs', export_resp.body)
+    resp = resp.form.submit().follow()
+    assert len(TestDef.select_for_objectdef(formdef2)) == 1
+    assert len(resp.pyquery('li a:contains("First test")')) == 1
 
 
 def test_tests_status_page(pub):
@@ -224,7 +257,7 @@ def test_tests_status_page(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = 'This is a test'
     formdata.user_id = user.id
     formdata.store()
@@ -264,7 +297,7 @@ def test_tests_status_page_block_field(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = {'data': [{'1': 'foo'}]}
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
@@ -288,7 +321,7 @@ def test_tests_status_page_image_field(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
 
     upload = PicklableUpload('test.jpeg', 'image/jpeg')
     with open(os.path.join(os.path.dirname(__file__), '..', 'image-with-gps-data.jpeg'), 'rb') as jpg:
@@ -325,7 +358,7 @@ def test_tests_edit(pub):
     formdef.store()
 
     formdata = formdef.data_class()()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.user_id = user.id
     formdata.data = {'1': 'xxx'}
     formdata.store()
@@ -375,7 +408,7 @@ def test_tests_edit_data(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = 'test 1'
     formdata.data['3'] = 'test 2'
     formdata.user_id = user.id
@@ -434,7 +467,7 @@ def test_tests_edit_data_mark_as_failing(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = '12345'
     formdata.store()
 
@@ -516,7 +549,7 @@ def test_tests_edit_data_mark_as_failing_hidden_error(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['0'] = 'not-digits'
     formdata.data['1'] = 'also-not-digits'
     formdata.store()
@@ -548,7 +581,7 @@ def test_tests_edit_data_mark_as_failing_required_field(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.store()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
@@ -599,7 +632,7 @@ def test_tests_edit_data_is_in_backoffice(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = '12345'
     formdata.store()
 
@@ -690,7 +723,7 @@ def test_tests_manual_run(pub):
     # create test
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = 'a'
     formdata.user_id = user.id
     formdata.store()
@@ -787,7 +820,7 @@ def test_tests_result_recorded_errors(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.store()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
@@ -803,6 +836,106 @@ def test_tests_result_recorded_errors(pub):
     assert escape('Invalid filter "unknown"') in resp.text
 
 
+def test_tests_result_sent_requests(pub, http_requests):
+    create_superuser(pub)
+
+    wscall = NamedWsCall()
+    wscall.name = 'Hello world'
+    wscall.request = {'url': 'http://remote.example.net/json'}
+    wscall.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.fields = [
+        fields.PageField(
+            id='0',
+            label='1st page',
+            post_conditions=[
+                {
+                    'condition': {'type': 'django', 'value': 'form_var_computed_foo == "bar"'},
+                    'error_message': '',
+                }
+            ],
+        ),
+        fields.ComputedField(
+            id='1',
+            label='Computed',
+            varname='computed',
+            value_template='{{ webservice.hello_world }}',
+            freeze_on_initial_value=True,
+        ),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.name = 'First test'
+    testdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/1/tests/results/run').follow()
+
+    assert 'Success!' in resp.text
+    assert http_requests.count() == 1
+    http_requests.empty()
+
+    resp = resp.click('Display details')
+
+    assert 'Sent requests:' in resp.text
+    assert 'GET http://remote.example.net/json' in resp.text
+    assert 'Used webservice response:' not in resp.text
+
+    response = WebserviceResponse()
+    response.testdef_id = testdef.id
+    response.name = 'Response xxx'
+    response.url = 'http://remote.example.net/json'
+    response.payload = '{"foo": "wrong"}'
+    response.store()
+
+    resp = app.get('/backoffice/forms/1/tests/results/run').follow()
+
+    assert 'Success!' not in resp.text
+    assert http_requests.count() == 0
+
+    resp = resp.click('Display details')
+    result_url = resp.request.url
+
+    assert 'Sent requests:' in resp.text
+    assert 'GET http://remote.example.net/json' in resp.text
+    assert 'Used webservice response:' in resp.text
+
+    resp = resp.click('Response xxx')
+    assert 'Edit webservice response' in resp.text
+
+    response.remove_self()
+    resp = app.get(result_url)
+
+    assert 'Used webservice response:' in resp.text
+    assert 'Response xxx' not in resp.text
+    assert 'deleted' in resp.text
+
+    wscall.request['method'] = 'POST'
+    wscall.store()
+
+    resp = app.get('/backoffice/forms/1/tests/results/run').follow()
+
+    assert 'Success!' not in resp.text
+    assert http_requests.count() == 0
+
+    resp = resp.click('Display details')
+
+    assert 'Sent requests:' in resp.text
+    assert 'POST http://remote.example.net/json' in resp.text
+    assert 'Request was blocked since it is not a GET request.' in resp.text
+    assert 'Recorded errors:' not in resp.text
+
+    resp = resp.click('You can create corresponding webservice response here.')
+    assert 'Webservice responses' in resp.text
+
+
 def test_tests_run_order(pub):
     create_superuser(pub)
 
@@ -845,7 +978,73 @@ def test_tests_duplicate(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
+    formdata.data['1'] = 'abcdefg'
+    formdata.user_id = user.id
+    formdata.store()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.name = 'First test'
+    testdef.workflow_tests.actions = [
+        workflow_tests.ButtonClick(id='1', button_name='Go to end status'),
+        workflow_tests.AssertStatus(id='2', status_name='End status'),
+    ]
+    testdef.store()
+
+    response = WebserviceResponse()
+    response.testdef_id = testdef.id
+    response.name = 'Response xxx'
+    response.store()
+
+    app = login(get_app(pub))
+
+    assert TestDef.count() == 1
+    assert WorkflowTests.count() == 1
+    assert WebserviceResponse.count() == 1
+
+    resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
+    resp = resp.click('Duplicate')
+    resp = resp.form.submit().follow()
+
+    assert 'First test (copy)' in resp.text
+    assert 'abcdefg' in resp.text
+    assert TestDef.count() == 2
+    assert WorkflowTests.count() == 2
+    assert WebserviceResponse.count() == 2
+
+    testdef1, testdef2 = TestDef.select(order_by='id')
+    testdef1.workflow_tests.actions[0].button_name = 'Changed'
+    testdef1.store()
+
+    response = testdef1.get_webservice_responses()[0]
+    response.name = 'Changed'
+    response.store()
+
+    testdef1, testdef2 = TestDef.select(order_by='id')
+    assert testdef1.workflow_tests.actions[0].button_name == 'Changed'
+    assert testdef2.workflow_tests.actions[0].button_name == 'Go to end status'
+    assert testdef1.get_webservice_responses()[0].name == 'Changed'
+    assert testdef2.get_webservice_responses()[0].name == 'Response xxx'
+
+    resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
+    resp = resp.click('Duplicate')
+    resp = resp.form.submit().follow()
+
+    assert 'First test (copy 2)' in resp.text
+    assert 'abcdefg' in resp.text
+    assert TestDef.count() == 3
+
+
+def test_form_with_test_duplicate(pub):
+    user = create_superuser(pub)
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.fields = [fields.StringField(id='1', varname='test field', label='Test')]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
     formdata.data['1'] = 'abcdefg'
     formdata.user_id = user.id
     formdata.store()
@@ -855,24 +1054,10 @@ def test_tests_duplicate(pub):
     testdef.store()
 
     app = login(get_app(pub))
-
-    assert TestDef.count() == 1
-
-    resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
+    resp = app.get('/backoffice/forms/1/')
     resp = resp.click('Duplicate')
     resp = resp.form.submit().follow()
-
-    assert 'First test (copy)' in resp.text
-    assert 'abcdefg' in resp.text
-    assert TestDef.count() == 2
-
-    resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
-    resp = resp.click('Duplicate')
-    resp = resp.form.submit().follow()
-
-    assert 'First test (copy 2)' in resp.text
-    assert 'abcdefg' in resp.text
-    assert TestDef.count() == 3
+    assert resp.pyquery('#appbar h2').text() == 'test title (copy)'
 
 
 def test_tests_page_with_empty_map_field(pub):
@@ -885,7 +1070,7 @@ def test_tests_page_with_empty_map_field(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = None
     formdata.store()
 
@@ -970,12 +1155,12 @@ def test_tests_exclude_self(pub):
 
     submitted_formdata = formdef.data_class()()
     submitted_formdata.just_created()
-    submitted_formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    submitted_formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     submitted_formdata.store()
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
 
     app = login(get_app(pub))
     resp = app.get('/backoffice/forms/1/tests/new')
@@ -985,3 +1170,77 @@ def test_tests_exclude_self(pub):
 
     resp = resp.form.submit('submit').follow()
     assert 'First test' in resp.text
+
+
+def test_tests_webservice_response(pub):
+    create_superuser(pub)
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.name = 'First test'
+    testdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
+
+    resp = resp.click('Webservice response')
+    assert 'There are no webservice responses yet.' in resp.text
+
+    resp = resp.click('New')
+    resp.form['name'] = 'Test response'
+    resp = resp.form.submit().follow()
+
+    resp.form['url'] = 'http://example.com/'
+    resp = resp.form.submit('submit').follow()
+
+    assert 'There are no webservice responses yet.' not in resp.text
+
+    resp = resp.click('Test response')
+    resp.form['payload'] = '{"a": "b"}'
+    resp.form['qs_data$element0key'] = 'foo'
+    resp.form['method'] = 'POST (JSON)'
+    resp.form['post_data$element0key'] = 'bar'
+    resp = resp.form.submit('submit').follow()
+
+    response = testdef.get_webservice_responses()[0]
+    assert response.name == 'Test response'
+    assert response.url == 'http://example.com/'
+    assert response.payload == '{"a": "b"}'
+    assert response.qs_data == {'foo': ''}
+    assert response.method == 'POST'
+    assert response.post_data == {'bar': ''}
+
+    resp = resp.click('Duplicate').follow()
+    assert 'Test response' in resp.text
+    assert 'not configured' not in resp.text
+    assert 'Test response (copy)' in resp.text
+
+    response = testdef.get_webservice_responses()[1]
+    assert response.name == 'Test response (copy)'
+    assert response.url == 'http://example.com/'
+    assert response.payload == '{"a": "b"}'
+
+    resp = resp.click('Remove', href=response.id)
+    resp = resp.form.submit().follow()
+
+    assert 'Test response (copy)' not in resp.text
+
+    resp = resp.click('Test response')
+    resp.form['payload'] = ''
+    resp = resp.form.submit('submit').follow()
+
+    assert 'Test response' in resp.text
+    assert '(not configured)' in resp.text
+
+    resp = resp.click('Test response')
+    resp.form['payload'] = '{"a"}'
+    resp = resp.form.submit()
+
+    assert "Invalid JSON: Expecting ':' delimiter: line 1 column 5 (char 4)" in resp.text

tests/admin_pages/test_workflow_tests.py

@@ -0,0 +1,508 @@
+import os
+
+import pytest
+from django.utils.html import escape
+
+from wcs import workflow_tests
+from wcs.formdef import FormDef, fields
+from wcs.qommon.http_request import HTTPRequest
+from wcs.testdef import TestDef
+from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef
+
+from ..utilities import create_temporary_pub, get_app, login
+from .test_all import create_superuser
+
+
+@pytest.fixture
+def pub():
+    pub = create_temporary_pub()
+
+    req = HTTPRequest(None, {'SCRIPT_NAME': '/', 'SERVER_NAME': 'example.net'})
+    pub.set_app_dir(req)
+    pub.cfg['identification'] = {'methods': ['password']}
+    pub.write_cfg()
+
+    if not pub.site_options.has_section('options'):
+        pub.site_options.add_section('options')
+    pub.site_options.set('options', 'enable-workflow-tests', 'true')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+
+    FormDef.wipe()
+    TestDef.wipe()
+    return pub
+
+
+def test_workflow_tests_link_feature_flag(pub):
+    create_superuser(pub)
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.name = 'First test'
+    testdef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
+    assert 'Workflow tests' in resp.text
+
+    pub.site_options.set('options', 'enable-workflow-tests', 'false')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+
+    resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
+    assert 'Workflow tests' not in resp.text
+
+
+def test_workflow_tests_options(pub):
+    create_superuser(pub)
+    user = pub.user_class(name='test user')
+    user.email = 'test@example.com'
+    user.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.name = 'First test'
+    testdef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
+    resp = resp.click('Options')
+
+    resp.form['agent'] = user.id
+    resp = resp.form.submit('submit').follow()
+
+    testdef = TestDef.get(testdef.id)
+    assert testdef.agent_id == str(user.id)
+
+
+def test_workflow_tests_disabled_no_agent(pub):
+    user = create_superuser(pub)
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.name = 'First test'
+    testdef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
+    assert 'Backoffice user is not defined, workflow tests will not be executed.' in resp.text
+
+    resp = resp.click('Open test options')
+    resp.form['agent'] = user.id
+    resp.form.submit().follow()
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
+    assert 'Backoffice user is not defined' not in resp.text
+    assert 'Open test options' not in resp.text
+
+
+def test_workflow_tests_edit_actions(pub):
+    user = create_superuser(pub)
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.name = 'First test'
+    testdef.agent_id = user.id
+    testdef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
+    resp = resp.click('Workflow tests')
+
+    assert 'There are no workflow test actions yet.' in resp.text
+    assert len(resp.pyquery('.biglist li')) == 0
+
+    # add workflow test action through sidebar form
+    resp.form['type'] = 'button-click'
+    resp = resp.form.submit().follow()
+
+    assert 'There are no workflow test actions yet.' not in resp.text
+    assert len(resp.pyquery('.biglist li')) == 1
+    assert resp.pyquery('.biglist li .label').text() == 'Simulate click on action button'
+    assert 'not configured' in resp.text
+
+    resp = resp.click('Edit')
+    resp.form['button_name'] = 'Accept'
+    resp = resp.form.submit().follow()
+
+    assert 'not configured' not in resp.text
+    assert resp.text.count(escape('Click on "Accept"')) == 1
+
+    resp = resp.click('Duplicate').follow()
+    assert resp.text.count(escape('Click on "Accept"')) == 2
+
+    resp = resp.click('Edit', index=0)
+    resp.form['button_name'] = 'Reject'
+    resp = resp.form.submit().follow()
+
+    assert resp.text.count(escape('Click on "Accept"')) == 1
+    assert resp.text.count(escape('Click on "Reject"')) == 1
+
+    resp = resp.click('Delete', index=0)
+    resp = resp.form.submit().follow()
+    assert resp.text.count(escape('Click on "Accept"')) == 1
+    assert resp.text.count(escape('Click on "Reject"')) == 0
+
+    # simulate invalid action
+    testdef = TestDef.get(testdef.id)
+    testdef.workflow_tests.actions[0].key = 'xxx'
+    testdef.store()
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
+    assert 'There are no workflow test actions yet.' in resp.text
+
+
+def test_workflow_tests_action_button_click(pub):
+    create_superuser(pub)
+
+    workflow = Workflow(name='Workflow One')
+    new_status = workflow.add_status(name='New status')
+
+    jump = new_status.add_action('choice')
+    jump.label = 'Button 1'
+    jump.status = new_status.id
+
+    jump = new_status.add_action('choice')
+    jump.label = 'Button 2'
+    jump.status = new_status.id
+
+    jump = new_status.add_action('choice')
+    jump.label = 'Button no target status'
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.workflow_tests.actions = [
+        workflow_tests.ButtonClick(id='1', button_name='Button 4'),
+    ]
+    testdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
+
+    assert resp.form['button_name'].options == [
+        ('Button 1', False, 'Button 1'),
+        ('Button 2', False, 'Button 2'),
+        ('Button 4 (not available)', True, 'Button 4 (not available)'),
+    ]
+
+
+def test_workflow_tests_action_assert_status(pub):
+    create_superuser(pub)
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertStatus(id='1', status_name='Deleted status'),
+    ]
+    testdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
+
+    assert resp.form['status_name'].options == [
+        ('Just Submitted', False, 'Just Submitted'),
+        ('New', False, 'New'),
+        ('Rejected', False, 'Rejected'),
+        ('Accepted', False, 'Accepted'),
+        ('Finished', False, 'Finished'),
+        ('Deleted status (not available)', False, 'Deleted status (not available)'),
+    ]
+
+
+def test_workflow_tests_action_skip_time(pub):
+    create_superuser(pub)
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.workflow_tests.actions = [
+        workflow_tests.SkipTime(id='1'),
+    ]
+    testdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
+
+    resp.form['seconds'] = '1 day 1 hour 1 minute'
+    resp = resp.form.submit().follow()
+
+    assert TestDef.get(testdef.id).workflow_tests.actions[0].seconds == 25 * 60 * 60 + 60
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
+    assert resp.form['seconds'].value == '1 day, 1 hour and 1 minute'
+
+    resp = resp.form.submit().follow()
+    assert TestDef.get(testdef.id).workflow_tests.actions[0].seconds == 25 * 60 * 60 + 60
+
+
+def test_workflow_tests_action_assert_email(pub):
+    create_superuser(pub)
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertEmail(id='1'),
+    ]
+    testdef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
+    assert 'not configured' not in resp.text
+
+    # empty configuration is allowed
+    resp = resp.click('Edit')
+    resp = resp.form.submit().follow()
+
+    resp = resp.click('Edit')
+    resp.form['subject_strings$element0'] = 'abc'
+    resp.form['body_strings$element0'] = 'def'
+    resp = resp.form.submit().follow()
+
+    assert_email = TestDef.get(testdef.id).workflow_tests.actions[0]
+    assert assert_email.subject_strings == ['abc']
+    assert assert_email.body_strings == ['def']
+
+
+def test_workflow_tests_action_assert_backoffice_field(pub):
+    create_superuser(pub)
+
+    workflow = Workflow(name='Workflow One')
+    workflow.add_status(name='New status')
+
+    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
+    workflow.backoffice_fields_formdef.fields = [
+        fields.StringField(id='bo1', label='Text'),
+        fields.StringField(id='bo2', label='Text 2'),
+    ]
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.workflow = workflow
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertBackofficeFieldValues(id='1'),
+    ]
+    testdef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
+    assert resp.form['fields$element0$field_id'].options == [
+        ('', False, ''),
+        ('bo1', False, 'Text - Text (line)'),
+        ('bo2', False, 'Text 2 - Text (line)'),
+    ]
+
+    resp.form['fields$element0$field_id'] = 'bo2'
+    resp.form['fields$element0$value'] = 'xxx'
+    resp = resp.form.submit().follow()
+
+    assert_bakoffice_field_values = TestDef.get(testdef.id).workflow_tests.actions[0]
+    assert assert_bakoffice_field_values.fields == [
+        {'field_id': 'bo2', 'value': 'xxx'},
+    ]
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
+    assert resp.form['fields$element0$field_id'].value == 'bo2'
+    assert resp.form['fields$element0$value'].value == 'xxx'
+
+
+def test_workflow_tests_actions_reorder(pub):
+    create_superuser(pub)
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.name = 'First test'
+    testdef.workflow_tests.actions = [
+        workflow_tests.ButtonClick(id='0', button_name='First'),
+        workflow_tests.ButtonClick(id='1', button_name='Second'),
+        workflow_tests.ButtonClick(id='2', button_name='Third'),
+        workflow_tests.ButtonClick(id='3', button_name='Fourth'),
+    ]
+    testdef.store()
+
+    app = login(get_app(pub))
+    url = '/backoffice/forms/%s/tests/%s/workflow/update_order' % (formdef.id, testdef.id)
+
+    # missing element in params: do nothing
+    resp = app.get(url + '?order=0;3;1;2;')
+    assert resp.json == {'success': 'ko'}
+
+    # missing order in params: do nothing
+    resp = app.get(url + '?element=0')
+    assert resp.json == {'success': 'ko'}
+
+    resp = app.get(url + '?order=0;3;1;2;&element=3')
+    assert resp.json == {'success': 'ok'}
+    testdef = TestDef.get(testdef.id)
+    assert [x.id for x in testdef.workflow_tests.actions] == ['0', '3', '1', '2']
+
+    # unknown id: ignored
+    resp = app.get(url + '?order=0;1;2;3;4;&element=3')
+    assert resp.json == {'success': 'ok'}
+    testdef = TestDef.get(testdef.id)
+    assert [x.id for x in testdef.workflow_tests.actions] == ['0', '1', '2', '3']
+
+    # missing id: do nothing
+    resp = app.get(url + '?order=0;3;1;&element=3')
+    assert resp.json == {'success': 'ko'}
+    testdef = TestDef.get(testdef.id)
+    assert [x.id for x in testdef.workflow_tests.actions] == ['0', '1', '2', '3']
+
+
+def test_workflow_tests_run(pub):
+    user = create_superuser(pub)
+
+    role = pub.role_class(name='test role')
+    role.store()
+    user.roles = [role.id]
+    user.store()
+
+    workflow = Workflow(name='Workflow One')
+    new_status = workflow.add_status(name='New status')
+
+    sendmail = new_status.add_action('sendmail')
+    sendmail.to = ['test@example.org']
+    sendmail.subject = 'Hello'
+    sendmail.body = 'abc'
+
+    jump = new_status.add_action('choice')
+    jump.label = 'Loop on status'
+    jump.status = new_status.id
+    jump.by = [role.id]
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.agent_id = user.id
+    testdef.workflow_tests.actions = [
+        workflow_tests.ButtonClick(id='1', button_name='Loop on status'),
+    ]
+    testdef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get('/backoffice/forms/1/tests/results/')
+    resp = resp.click('Run tests').follow()
+
+    assert len(resp.pyquery('tr')) == 1
+    assert 'Success!' in resp.text
+
+    # change button label
+    jump.label = 'xxx'
+    workflow.store()
+
+    resp = app.get('/backoffice/forms/1/tests/results/')
+    resp = resp.click('Run tests').follow()
+
+    assert escape('Workflow error: Button "Loop on status" is not displayed.') in resp.text
+
+    resp = resp.click('Display details')
+
+    assert 'Form status when error occured: New status' in resp.text
+    assert resp.pyquery('li#test-action').text() == 'Test action: Simulate click on action button'
+    assert (
+        resp.pyquery('li#test-action a').attr('href')
+        == 'http://example.net/backoffice/forms/1/tests/%s/workflow/#1' % testdef.id
+    )
+
+    testdef.workflow_tests.actions = []
+    testdef.store()
+
+    resp = app.get(resp.request.url)
+    assert 'Form status when error occured: New status' in resp.text
+    assert resp.pyquery('li#test-action').text() == 'Test action: deleted'
+
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertEmail(id='1', body_strings=['def']),
+    ]
+    testdef.store()
+
+    resp = app.get('/backoffice/forms/1/tests/results/')
+    resp = resp.click('Run tests').follow()
+    assert escape('Email body does not contain "def".') in resp.text
+
+    resp = resp.click('Display details')
+    assert 'Form status when error occured: New status' in resp.text
+    assert 'Email body: \nabc' in resp.text
+    assert resp.pyquery('li#test-action').text() == 'Test action: Assert email is sent'

tests/api/test_all.py

@@ -125,57 +125,24 @@ def test_tracking_code(pub, auth, admin_user):
     resp = get_url('/api/code/%s' % code.id, status=404)
 
 
-def test_validate_expression(pub):
-    resp = get_app(pub).get('/api/validate-expression?expression=hello')
-    assert resp.json == {'klass': None, 'msg': ''}
-    resp = get_app(pub).get('/api/validate-expression?expression=[hello]')
-    assert resp.json == {'klass': None, 'msg': ''}
-    resp = get_app(pub).get('/api/validate-expression?expression==[hello')
-    assert resp.json['klass'] == 'error'
-    assert resp.json['msg'].startswith('syntax error')
-    resp = get_app(pub).get('/api/validate-expression?expression==[hello]')
-    assert resp.json['klass'] == 'warning'
-    assert resp.json['msg'].startswith('Make sure you want a Python expression,')
-    resp = get_app(pub).get('/api/validate-expression?expression==hello[0]')
-    assert resp.json == {'klass': None, 'msg': ''}
-    resp = get_app(pub).get('/api/validate-expression?expression==hello[\'plop\']')
-    assert resp.json == {'klass': None, 'msg': ''}
-    # django with unicode
-    resp = get_app(pub).get('/api/validate-expression?expression={{hello+%C3%A9l%C3%A9phant}}')
-    assert resp.json['klass'] == 'error'
-    assert resp.json['msg'].startswith('syntax error in Django template: Could not parse the remainder')
-    # broken ezt
-    resp = get_app(pub).get('/api/validate-expression?expression=[for]')
-    assert resp.json == {
-        'klass': 'error',
-        'msg': 'syntax error in ezt template: wrong number of arguments at line 1 and column 1',
-    }
-
-
 def test_validate_condition(pub):
     resp = get_app(pub).get('/api/validate-condition?type=python&value_python=hello')
-    assert resp.json == {'klass': None, 'msg': ''}
+    assert resp.json == {'msg': ''}
     resp = get_app(pub).get('/api/validate-condition?type=python&value_python=~2')
-    assert resp.json == {'klass': None, 'msg': ''}
+    assert resp.json == {'msg': ''}
     resp = get_app(pub).get('/api/validate-condition?type=python&value_python=hello -')
-    assert resp.json['klass'] == 'error'
     assert resp.json['msg'].startswith('syntax error')
     resp = get_app(pub).get('/api/validate-condition?type=python&value_python={{form_number}}==3')
-    assert resp.json['klass'] == 'error'
     assert 'Python condition cannot contain {{' in resp.json['msg']
 
     resp = get_app(pub).get('/api/validate-condition?type=django&value_django=un+%C3%A9l%C3%A9phant')
-    assert resp.json['klass'] == 'error'
     assert resp.json['msg'].startswith("syntax error: Unused 'éléphant'")
     resp = get_app(pub).get('/api/validate-condition?type=django&value_django=~2')
-    assert resp.json['klass'] == 'error'
     assert resp.json['msg'].startswith('syntax error')
     resp = get_app(pub).get('/api/validate-condition?type=django&value_django=%22...%22+inf')  # "..." + inf
-    assert resp.json['klass'] == 'error'
     assert resp.json['msg'].startswith('syntax error')
 
     resp = get_app(pub).get('/api/validate-condition?type=unknown&value_unknown=2')
-    assert resp.json['klass'] == 'error'
     assert resp.json['msg'] == 'unknown condition type'
 
 

tests/api/test_export_import.py

@@ -7,6 +7,7 @@ import xml.etree.ElementTree as ET
 
 import pytest
 
+from wcs.api_export_import import klass_to_slug
 from wcs.applications import Application, ApplicationElement
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
@@ -940,6 +941,195 @@ def test_export_import_bundle_import(pub):
     assert formdef.workflow_roles == {'_receiver': extra_role.id}
 
 
+@pytest.mark.parametrize(
+    'category_class',
+    [
+        Category,
+        CardDefCategory,
+        BlockCategory,
+        WorkflowCategory,
+        MailTemplateCategory,
+        CommentTemplateCategory,
+        DataSourceCategory,
+    ],
+)
+def test_export_import_bundle_import_categories_ordering(pub, category_class):
+    category_class.wipe()
+    category = category_class(name='cat 1')
+    category.position = 1
+    category.store()
+    category = category_class(name='cat 2')
+    category.position = 2
+    category.store()
+    category = category_class(name='cat 3')
+    category.position = 3
+    category.store()
+    bundle = create_bundle(
+        [
+            {'type': klass_to_slug[category_class], 'slug': 'cat-1', 'name': 'cat 1'},
+            {'type': klass_to_slug[category_class], 'slug': 'cat-2', 'name': 'cat 2'},
+            {'type': klass_to_slug[category_class], 'slug': 'cat-3', 'name': 'cat 3'},
+        ],
+        ('%s/cat-1' % klass_to_slug[category_class], category_class.get(1)),
+        ('%s/cat-2' % klass_to_slug[category_class], category_class.get(2)),
+        ('%s/cat-3' % klass_to_slug[category_class], category_class.get(3)),
+    )
+
+    # delete categories
+    category_class.wipe()
+    # and recreate only cat 4 and 5 in first positions
+    category = category_class(name='cat 4')
+    category.position = 1
+    category.store()
+    category = category_class(name='cat 5')
+    category.position = 2
+    category.store()
+
+    # import bundle
+    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    afterjob_url = resp.json['url']
+    resp = get_app(pub).put(sign_uri(afterjob_url))
+    assert resp.json['data']['status'] == 'completed'
+
+    # cat 1, 2, 3 are placed at the end
+    assert category_class.get_by_slug('cat-4').position == 1
+    assert category_class.get_by_slug('cat-5').position == 2
+    assert category_class.get_by_slug('cat-1').position == 3
+    assert category_class.get_by_slug('cat-2').position == 4
+    assert category_class.get_by_slug('cat-3').position == 5
+
+    # delete categories
+    category_class.wipe()
+    # recreate only cat 2, cat 4, cat 5 in this order
+    category = category_class(name='cat 2')
+    category.position = 1
+    category.store()
+    category = category_class(name='cat 4')
+    category.position = 2
+    category.store()
+    category = category_class(name='cat 5')
+    category.position = 3
+    category.store()
+
+    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    afterjob_url = resp.json['url']
+    resp = get_app(pub).put(sign_uri(afterjob_url))
+    assert resp.json['data']['status'] == 'completed'
+
+    # cat 1, 2, 3 are placed after cat 4
+    assert category_class.get_by_slug('cat-1').position == 1
+    assert category_class.get_by_slug('cat-2').position == 2
+    assert category_class.get_by_slug('cat-3').position == 3
+    assert category_class.get_by_slug('cat-4').position == 4
+    assert category_class.get_by_slug('cat-5').position == 5
+
+    # delete categories
+    category_class.wipe()
+    # recreate only cat 4, cat 2, cat 5 in this order
+    category = category_class(name='cat 4')
+    category.position = 1
+    category.store()
+    category = category_class(name='cat 2')
+    category.position = 2
+    category.store()
+    category = category_class(name='cat 5')
+    category.position = 3
+    category.store()
+
+    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    afterjob_url = resp.json['url']
+    resp = get_app(pub).put(sign_uri(afterjob_url))
+    assert resp.json['data']['status'] == 'completed'
+
+    # cat 1, 2, 3 are placed after cat 4
+    assert category_class.get_by_slug('cat-4').position == 1
+    assert category_class.get_by_slug('cat-1').position == 2
+    assert category_class.get_by_slug('cat-2').position == 3
+    assert category_class.get_by_slug('cat-3').position == 4
+    assert category_class.get_by_slug('cat-5').position == 5
+
+    # delete categories
+    category_class.wipe()
+    # recreate only cat 4, cat 5, cat 2 in this order
+    category = category_class(name='cat 4')
+    category.position = 1
+    category.store()
+    category = category_class(name='cat 5')
+    category.position = 2
+    category.store()
+    category = category_class(name='cat 2')
+    category.position = 3
+    category.store()
+
+    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    afterjob_url = resp.json['url']
+    resp = get_app(pub).put(sign_uri(afterjob_url))
+    assert resp.json['data']['status'] == 'completed'
+
+    # cat 1, 2, 3 are placed after cat 4
+    assert category_class.get_by_slug('cat-4').position == 1
+    assert category_class.get_by_slug('cat-5').position == 2
+    assert category_class.get_by_slug('cat-1').position == 3
+    assert category_class.get_by_slug('cat-2').position == 4
+    assert category_class.get_by_slug('cat-3').position == 5
+
+    # delete categories
+    category_class.wipe()
+    # recreate only cat 4, cat 2, cat1 cat 5 in this order but with weird positions
+    category = category_class(name='cat 4')
+    category.position = 4
+    category.store()
+    category = category_class(name='cat 2')
+    category.position = 12
+    category.store()
+    category = category_class(name='cat 1')
+    category.position = 13
+    category.store()
+    category = category_class(name='cat 5')
+    category.position = 20
+    category.store()
+
+    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    afterjob_url = resp.json['url']
+    resp = get_app(pub).put(sign_uri(afterjob_url))
+    assert resp.json['data']['status'] == 'completed'
+
+    # cat 1, 2, 3 are placed after cat 4
+    assert category_class.get_by_slug('cat-4').position == 1
+    assert category_class.get_by_slug('cat-1').position == 2
+    assert category_class.get_by_slug('cat-2').position == 3
+    assert category_class.get_by_slug('cat-3').position == 4
+    assert category_class.get_by_slug('cat-5').position == 5
+
+    # delete categories
+    category_class.wipe()
+    # recreate only cat 4, cat 2, cat1 cat 5 in this order but with weird positions
+    category = category_class(name='cat 4')
+    category.position = 1
+    category.store()
+    category = category_class(name='cat 2')
+    category.position = 2
+    category.store()
+    category = category_class(name='cat 1')
+    category.position = 2
+    category.store()
+    category = category_class(name='cat 5')
+    category.position = None  # no position
+    category.store()
+
+    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    afterjob_url = resp.json['url']
+    resp = get_app(pub).put(sign_uri(afterjob_url))
+    assert resp.json['data']['status'] == 'completed'
+
+    # cat 1, 2, 3 are placed after cat 4
+    assert category_class.get_by_slug('cat-4').position == 1
+    assert category_class.get_by_slug('cat-1').position == 2
+    assert category_class.get_by_slug('cat-2').position == 3
+    assert category_class.get_by_slug('cat-3').position == 4
+    assert category_class.get_by_slug('cat-5').position == 5
+
+
 def test_export_import_formdef_do_not_overwrite_table_name(pub):
     formdef = FormDef()
     formdef.name = 'Test2'

tests/api/test_formdata.py

@@ -10,6 +10,7 @@ import zipfile
 
 import pytest
 from django.utils.encoding import force_bytes
+from django.utils.timezone import localtime, make_aware
 from quixote import get_publisher
 from webtest import Upload
 
@@ -979,9 +980,14 @@ def test_api_list_formdata(pub, local_user):
         if i % 7 == 0:
             formdata.backoffice_submission = True
             formdata.submission_channel = 'mail'
-        formdata.evolution[-1].time = (
+        formdata.receipt_time = make_aware(datetime.datetime(2018, 1, 2, 3, 4) + datetime.timedelta(hours=i))
+        formdata.evolution[0].time = make_aware(
+            datetime.datetime(2019, 1, 2, 3, 4) + datetime.timedelta(hours=i)
+        )
+        formdata.evolution[-1].time = make_aware(
             datetime.datetime(2020, 1, 2, 3, 4) + datetime.timedelta(hours=i)
-        ).timetuple()
+        )
+        formdata._store_all_evolution = True
         formdata.store()
     # a draft by user
     formdata = data_class()
@@ -2554,7 +2560,7 @@ def test_api_anonymized_formdata(pub, local_user, admin_user):
         else:
             evo = Evolution(formdata=formdata)
             evo.who = admin_user.id
-            evo.time = time.localtime()
+            evo.time = localtime()
             evo.status = 'wf-%s' % 'st2'
             formdata.evolution.append(evo)
             formdata.status = evo.status

tests/api/test_formdef.py

@@ -10,6 +10,7 @@ from functools import partial
 import pytest
 import responses
 from django.utils.encoding import force_str
+from django.utils.timezone import localtime
 from quixote import get_publisher
 
 from wcs import fields, qommon
@@ -163,7 +164,7 @@ def test_formdef_list(pub):
         formdata = formdef.data_class()()
         formdata.data = {}
         formdata.just_created()
-        formdata.receipt_time = (datetime.datetime.now() - datetime.timedelta(days=days)).timetuple()
+        formdata.receipt_time = localtime() - datetime.timedelta(days=days)
         formdata.store()
 
     resp = get_app(pub).get(sign_uri('/api/formdefs/?include-count=on'))
@@ -1222,6 +1223,8 @@ def test_formdef_import_export_unnamed_block(pub, admin_user):
     formdata_export = formdata.get_json_export_dict(include_unnamed_fields=True, include_evolution=False)
     del formdata_export['receipt_time']
     del formdata_export['last_update_time']
+    del formdata_export['workflow']['real_status']['first_arrival_datetime']
+    del formdata_export['workflow']['real_status']['latest_arrival_datetime']
     formdef.data_class().wipe()
 
     app = login(get_app(pub))

tests/api/test_statistics.py

@@ -3,6 +3,7 @@ import json
 import os
 
 import pytest
+from django.utils.timezone import make_aware
 
 from wcs import fields
 from wcs.backoffice.management import format_time
@@ -313,7 +314,7 @@ def test_statistics_forms_count(pub):
     for i in range(20):
         formdata = formdef.data_class()()
         formdata.just_created()
-        formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
         # "Web" channel has three equivalent values
         if i == 0:
             formdata.submission_channel = 'web'
@@ -327,14 +328,14 @@ def test_statistics_forms_count(pub):
     for i in range(30):
         formdata = formdef2.data_class()()
         formdata.just_created()
-        formdata.receipt_time = datetime.datetime(2021, 3, 1, 2, 0).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2021, 3, 1, 2, 0))
         formdata.backoffice_submission = bool(i % 3)
         formdata.submission_channel = 'mail'
         formdata.store()
 
     # draft should not be counted
     formdata = formdef.data_class()()
-    formdata.receipt_time = datetime.datetime(2021, 3, 1, 2, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 3, 1, 2, 0))
     formdata.status = 'draft'
     formdata.store()
 
@@ -414,7 +415,7 @@ def test_statistics_forms_count_subfilters(pub, formdef):
         formdata.data['3_display'] = 'Foo' if i % 2 else 'Bar, Baz'
         formdata.data['4'] = {'data': [{'2': ['foo', 'bar'], '2_display': 'Foo, Bar'}]}
         formdata.just_created()
-        formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
         formdata.store()
 
     url = '/api/statistics/forms/count/?form=%s&time_interval=year' % formdef.url_name
@@ -593,7 +594,7 @@ def test_statistics_forms_count_subfilters_empty_block_items_field(pub, formdef)
     formdata = formdef.data_class()()
     formdata.data['4'] = {'data': [{'1': 'a', '1_display': 'B'}]}
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.store()
 
     resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?form=%s' % formdef.url_name))
@@ -619,7 +620,7 @@ def test_statistics_forms_count_subfilters_empty_item_field_no_datasource(pub, f
     formdata.just_created()
     formdata.data['10'] = 'extra-option'
     formdata.data['10_display'] = 'Extra option'
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.store()
 
     resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?form=%s' % formdef.url_name))
@@ -647,7 +648,7 @@ def test_statistics_forms_count_subfilters_query(pub, formdef):
             formdata.data['3'] = ['baz']
             formdata.data['4'] = {'data': [{'1': False, '2': ['foo', 'bar'], '2_display': 'Foo, Bar'}]}
             formdata.jump_status('2')
-        formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
         formdata.store()
 
     # query all formdata
@@ -769,7 +770,7 @@ def test_statistics_forms_count_subfilters_query_same_varname(pub, formdef):
     for i in range(5):
         formdata = formdef.data_class()()
         formdata.just_created()
-        formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
         if i == 0:
             formdata.data['1'] = 'foo'
         if i == 1:
@@ -803,7 +804,7 @@ def test_statistics_forms_count_subfilters_query_integer_items(pub, formdef):
             formdata.data['3'] = ['1', '2']
         else:
             formdata.data['3'] = ['1']
-        formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
         formdata.store()
 
     url = '/api/statistics/forms/count/?form=%s' % formdef.url_name
@@ -819,7 +820,7 @@ def test_statistics_forms_count_group_by(pub, formdef, anonymise):
     for i in range(20):
         formdata = formdef.data_class()()
         formdata.just_created()
-        formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
         if i % 3:
             formdata.data['1'] = True
             formdata.data['2'] = 'foo'
@@ -855,7 +856,7 @@ def test_statistics_forms_count_group_by(pub, formdef, anonymise):
             formdata.submission_channel = 'mail'
             formdata.backoffice_submission = bool(i % 3)
         else:
-            formdata.receipt_time = datetime.datetime(2021, 3, 1, 2, 0).timetuple()
+            formdata.receipt_time = make_aware(datetime.datetime(2021, 3, 1, 2, 0))
         formdata.store()
         if anonymise:
             formdata.anonymise()
@@ -1017,7 +1018,7 @@ def test_statistics_forms_count_group_by_same_varname(pub, formdef):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = 'foo'
     formdata.data['2'] = 'bar'
     formdata.store()
@@ -1036,7 +1037,7 @@ def test_statistics_forms_count_group_by_same_varname(pub, formdef):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['2'] = 'foo'
     formdata.store()
 
@@ -1052,7 +1053,7 @@ def test_statistics_forms_count_group_by_form(pub):
     for i in range(10):
         formdata = formdef.data_class()()
         formdata.just_created()
-        formdata.receipt_time = datetime.datetime(2022, 1, 1, 0, 0).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2022, 1, 1, 0, 0))
         formdata.store()
 
     formdef = FormDef()
@@ -1062,7 +1063,7 @@ def test_statistics_forms_count_group_by_form(pub):
     for i in range(5):
         formdata = formdef.data_class()()
         formdata.just_created()
-        formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
         formdata.store()
 
     resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/'))
@@ -1095,7 +1096,7 @@ def test_statistics_forms_count_months_to_show(pub, formdef):
         formdata.data['2'] = 'foo' if i % 2 else 'baz'
         formdata.data['2_display'] = 'Foo' if i % 2 else 'Baz'
         formdata.just_created()
-        formdata.receipt_time = datetime.datetime(2022 + i // 12, i % 12 + 1, 1, 0, 0).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2022 + i // 12, i % 12 + 1, 1, 0, 0))
         formdata.store()
 
     url = '/api/statistics/forms/count/'
@@ -1140,7 +1141,7 @@ def test_statistics_cards_count(pub):
     for _i in range(20):
         carddata = carddef.data_class()()
         carddata.just_created()
-        carddata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+        carddata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
         carddata.store()
 
     # apply (required) card filter
@@ -1399,8 +1400,8 @@ def test_statistics_resolution_time_median(pub, freezer):
     resp = get_app(pub).get(sign_uri('/api/statistics/resolution-time/?form=test'))
     assert get_humanized_duration_serie(resp.json) == [
         '1 day(s) and 0 hour(s)',  # min
-        '89 day(s) and 23 hour(s)',  # max
-        '13 day(s) and 23 hour(s)',  # mean
+        '90 day(s) and 0 hour(s)',  # max
+        '14 day(s) and 0 hour(s)',  # mean
         '5 day(s) and 0 hour(s)',  # median
     ]
 
@@ -1511,7 +1512,7 @@ def test_statistics_multiple_forms_count(pub, formdef):
         formdata.data['3'] = ['foo']
         formdata.data['3_display'] = 'Foo'
         formdata.just_created()
-        formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
         formdata.store()
 
     for _i in range(30):
@@ -1522,7 +1523,7 @@ def test_statistics_multiple_forms_count(pub, formdef):
         formdata.data['3_display'] = 'Bar, Baz'
         formdata.data['4'] = {'data': [{'1': True}]}
         formdata.just_created()
-        formdata.receipt_time = datetime.datetime(2021, 3, 1, 2, 0).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2021, 3, 1, 2, 0))
         formdata.jump_status('2')
         formdata.store()
 
@@ -1622,14 +1623,14 @@ def test_statistics_multiple_forms_count_different_ids(pub):
     formdata.data['1'] = 'foo'
     formdata.data['1_display'] = 'Foo'
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.store()
 
     formdata = formdef2.data_class()()
     formdata.data['2'] = 'baz'
     formdata.data['2_display'] = 'Baz'
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 3, 1, 2, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 3, 1, 2, 0))
     formdata.store()
 
     url = '/api/statistics/forms/count/?form=%s&form=%s' % (formdef1.url_name, formdef2.url_name)
@@ -1664,7 +1665,7 @@ def test_statistics_multiple_forms_count_subfilters(pub, formdef):
         formdata.data['2'] = 'foo'
         formdata.data['2_display'] = 'Foo'
         formdata.just_created()
-        formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
         formdata.jump_status('2')
         formdata.store()
 
@@ -1673,7 +1674,7 @@ def test_statistics_multiple_forms_count_subfilters(pub, formdef):
         formdata.data['2'] = 'baz'
         formdata.data['2_display'] = 'Baz'
         formdata.just_created()
-        formdata.receipt_time = datetime.datetime(2021, 3, 1, 2, 0).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2021, 3, 1, 2, 0))
         formdata.store()
 
     resp = get_app(pub).get(

tests/api/test_user.py

@@ -3,6 +3,7 @@ import os
 from functools import partial
 
 import pytest
+from django.utils.timezone import make_aware
 from quixote import get_publisher
 
 from wcs import fields
@@ -377,7 +378,7 @@ def test_user_forms(pub, local_user, access):
     formdata = formdef.data_class()()
     formdata.user_id = local_user.id
     formdata.status = 'draft'
-    formdata.receipt_time = datetime.datetime(2015, 1, 1).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2015, 1, 1))
     formdata.store()
 
     resp = get_app(pub).get(sign_uri('/api/user/forms', user=local_user))
@@ -399,7 +400,7 @@ def test_user_forms(pub, local_user, access):
     formdata.data = {'0': 'foo@localhost', '1': 'xyy'}
     formdata.user_id = local_user.id
     formdata.just_created()
-    formdata.receipt_time = (datetime.datetime.now() + datetime.timedelta(days=1)).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime.now() + datetime.timedelta(days=1))
     formdata.jump_status('new')
     formdata.store()
 
@@ -408,9 +409,9 @@ def test_user_forms(pub, local_user, access):
     resp2 = get_app(pub).get(sign_uri('/api/user/forms?sort=desc', user=local_user))
     assert len(resp2.json['data']) == 4
     assert resp2.json['data'][0] == resp.json['data'][3]
-    assert resp2.json['data'][1] == resp.json['data'][0]
+    assert resp2.json['data'][1] == resp.json['data'][2]
     assert resp2.json['data'][2] == resp.json['data'][1]
-    assert resp2.json['data'][3] == resp.json['data'][2]
+    assert resp2.json['data'][3] == resp.json['data'][0]
 
     # check there is no access with roles-limited API users
     role = pub.role_class(name='test')
@@ -443,7 +444,7 @@ def test_user_forms_limit_offset(pub, local_user):
         formdata.data = {'0': 'foo@localhost', '1': str(i)}
         formdata.user_id = local_user.id
         formdata.just_created()
-        formdata.receipt_time = (datetime.datetime.now() + datetime.timedelta(days=i)).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime.now() + datetime.timedelta(days=i))
         formdata.jump_status('new')
         formdata.store()
 
@@ -452,7 +453,7 @@ def test_user_forms_limit_offset(pub, local_user):
         formdata.data = {'0': 'foo@localhost', '1': str(i)}
         formdata.user_id = local_user.id
         formdata.status = 'draft'
-        formdata.receipt_time = (datetime.datetime.now() - datetime.timedelta(days=i)).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime.now() - datetime.timedelta(days=i))
         formdata.store()
 
     resp = get_app(pub).get(sign_uri('/api/users/%s/forms' % local_user.id))
@@ -772,7 +773,7 @@ def test_user_drafts(pub, local_user):
     formdata.user_id = local_user.id
     formdata.page_no = 1
     formdata.status = 'draft'
-    formdata.receipt_time = datetime.datetime(2015, 1, 1).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2015, 1, 1))
     formdata.store()
 
     resp = get_app(pub).get(sign_uri('/api/user/drafts', user=local_user))

tests/backoffice_pages/test_all.py

@@ -9,6 +9,7 @@ import zipfile
 
 import pytest
 import responses
+from django.utils.timezone import make_aware
 from webtest import Upload
 
 import wcs.qommon.storage as st
@@ -132,7 +133,7 @@ def create_environment(pub, set_receiver=True):
     for i in range(50):
         formdata = formdef.data_class()()
         formdata.just_created()
-        formdata.receipt_time = datetime.datetime(2015, 1, 1, 0, i).timetuple()
+        formdata.receipt_time = datetime.datetime(2015, 1, 1, 0, i)
         formdata.data = {'1': 'FOO BAR %d' % i}
         if i % 4 == 0:
             formdata.data[formdef.fields[1].id] = 'foo'
@@ -170,7 +171,7 @@ def create_environment(pub, set_receiver=True):
     for i in range(20):
         formdata = formdef.data_class()()
         formdata.just_created()
-        formdata.receipt_time = datetime.datetime(2014, 1, 1).timetuple()
+        formdata.receipt_time = datetime.datetime(2014, 1, 1)
         formdata.jump_status('new')
         formdata.store()
 
@@ -1259,7 +1260,9 @@ def test_backoffice_multi_actions_interactive(pub):
     for i in range(10):
         formdata = formdef.data_class()()
         formdata.just_created()
+        formdata.receipt_time = make_aware(datetime.datetime(2015, 1, 1, 0, i))
         formdata.jump_status('new')
+        formdata.evolution[-1].time = make_aware(datetime.datetime(2015, 1, 1, 0, i))
         formdata.store()
 
     app = login(get_app(pub))
@@ -3196,6 +3199,57 @@ def test_backoffice_wfedit_single_page(pub):
     assert formdata.data == {'2': 'a', '4': 'changed', '5': None, '7': 'c'}
 
 
+def test_backoffice_wfedit_partial_pages(pub):
+    user = create_user(pub)
+
+    workflow = Workflow(name='test')
+    st1 = workflow.add_status('Status1', 'st1')
+    editable = st1.add_action('editable', id='_editable')
+    editable.by = ['_receiver']
+    editable.operation_mode = 'partial'
+    editable.page_identifier = 'plop'
+    workflow.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = []
+    formdef.store()
+
+    formdef.data_class().wipe()
+    formdef.fields = [
+        fields.PageField(id='1', label='1st page'),
+        fields.StringField(id='2', label='field1'),
+        fields.PageField(id='3', label='2nd page', varname='plop'),
+        fields.StringField(id='4', label='field2'),
+        fields.StringField(
+            id='5', label='field2b', condition={'type': 'django', 'value': 'not is_in_backoffice'}
+        ),
+        fields.PageField(id='6', label='3rd page'),
+        fields.StringField(id='7', label='field3'),
+    ]
+    formdef.workflow_id = workflow.id
+    formdef.workflow_roles = {'_receiver': user.roles[0]}
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {'2': 'a', '4': 'b', '5': 'b2', '7': 'c'}
+    formdata.just_created()
+    formdata.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get(formdata.get_backoffice_url())
+    resp = resp.form.submit('button_editable').follow()
+    assert [x.text for x in resp.pyquery('#steps .wcs-step--label-text')] == ['2nd page', '3rd page']
+    resp.form['f4'] = 'changed'
+    resp = resp.form.submit('submit')
+    resp.form['f7'] = 'changed'
+    resp = resp.form.submit('submit')
+    formdata.refresh_from_storage()
+    assert formdata.data == {'2': 'a', '4': 'changed', '5': None, '7': 'changed'}
+
+
 def test_global_listing(pub):
     create_user(pub)
     create_environment(pub)
@@ -3229,7 +3283,7 @@ def test_global_listing(pub):
     last_update_time = formdef.data_class().select(lambda x: not x.is_draft())[0].last_update_time
     # check created and last modified columns
     assert '>2014-01-01 00:00<' in resp.text
-    assert time.strftime('>%Y-%m-%d', last_update_time) in resp.text
+    assert f'>{last_update_time.strftime("%Y-%m-%d")}' in resp.text
 
     # check digest is included
     formdata = formdef.data_class().get(
@@ -6193,6 +6247,7 @@ def test_anonymise_action_intermediate(pub):
 
     formdata = formdef.data_class()()
     formdata.data = {'0': 'Foo', '1': 'Bar'}
+    formdata.user_id = user.id
     formdata.just_created()
     formdata.store()
     formdata.perform_workflow()
@@ -6200,6 +6255,7 @@ def test_anonymise_action_intermediate(pub):
     formdata = formdef.data_class().select()[0]
     assert formdata.status == 'wf-anonymise_intermediate'
     assert formdata.data == {'0': None, '1': 'Bar'}
+    assert formdata.user_id
 
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/foo/%s/' % formdata.id)
@@ -6212,6 +6268,7 @@ def test_anonymise_action_intermediate(pub):
     formdata = formdef.data_class().select()[0]
     assert formdata.status == 'wf-anonymise_final'
     assert formdata.data == {'0': None, '1': None}
+    assert not formdata.user_id
 
 
 def test_anonymise_action_final_also_deletes_fields_with_intermediate(pub):

tests/backoffice_pages/test_export.py

@@ -8,6 +8,7 @@ import xml.etree.ElementTree as ET
 import zipfile
 
 import pytest
+from django.utils.timezone import make_aware
 
 from wcs import fields
 from wcs.blocks import BlockDef
@@ -75,7 +76,7 @@ def test_backoffice_csv(pub):
     formdef.data_class().wipe()
     for i in range(3):
         formdata = formdef.data_class()()
-        formdata.receipt_time = datetime.datetime(2015, 1, 1).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2015, 1, 1))
         formdata.data = {'1': 'FOO BAR %d' % i}
         if i == 0:
             formdata.data['2'] = 'foo'
@@ -247,7 +248,7 @@ def test_backoffice_export_long_listings(pub, threshold):
     formdef.data_class().wipe()
     for i in range(2):
         formdata = formdef.data_class()()
-        formdata.receipt_time = datetime.datetime(2015, 1, 1).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2015, 1, 1))
         formdata.data = {'1': 'BAZ BAZ %d' % i}
         formdata.jump_status('new')
         formdata.store()
@@ -264,8 +265,8 @@ def test_backoffice_export_long_listings(pub, threshold):
     resp_lines = resp.text.splitlines()
     assert resp_lines[0] == '"Number","Created","Last Modified","User Label","1st field","Status"'
     assert len(resp_lines) == 3
-    assert resp_lines[1].split(',')[1].startswith('"' + time.strftime('%Y-%m-%d', formdata.receipt_time))
-    assert resp_lines[1].split(',')[2].startswith('"' + time.strftime('%Y-%m-%d', formdata.last_update_time))
+    assert resp_lines[1].split(',')[1].startswith('"' + formdata.receipt_time.strftime('%Y-%m-%d'))
+    assert resp_lines[1].split(',')[2].startswith('"' + formdata.last_update_time.strftime('%Y-%m-%d'))
 
     resp = app.get('/backoffice/management/form-title/')
     resp = resp.click('Export a Spreadsheet')
@@ -1117,7 +1118,7 @@ def test_backoffice_header_line(pub):
     formdef.data_class().wipe()
     for i in range(3):
         formdata = formdef.data_class()()
-        formdata.receipt_time = datetime.datetime(2015, 1, 1).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2015, 1, 1))
         formdata.data = {'1': 'FOO BAR %d' % i}
         formdata.jump_status('new')
         formdata.store()

tests/backoffice_pages/test_form_inspect.py

@@ -1,10 +1,11 @@
+import datetime
 import io
 import json
 import os
 import re
-import time
 
 import pytest
+from django.utils.timezone import localtime
 from pyquery import PyQuery
 
 from wcs import fields
@@ -821,7 +822,7 @@ def test_inspect_page_actions_traces(pub):
     assert formdata.status == 'wf-accepted'
 
     # change receipt time to get global timeout to run
-    formdata.receipt_time = time.localtime(time.time() - 3 * 86400)
+    formdata.receipt_time = localtime() - datetime.timedelta(days=3)
     formdata.store()
     pub.apply_global_action_timeouts()
     formdata.refresh_from_storage()

tests/backoffice_pages/test_sort.py

@@ -3,6 +3,7 @@ import os
 import re
 
 import pytest
+from django.utils.timezone import make_aware
 
 from wcs import fields
 from wcs.blocks import BlockDef
@@ -64,9 +65,9 @@ def test_backoffice_listing_order(pub):
         formdata.jump_status('new')
         formdata.store()
         ids.append(formdata.id)
-        formdata.receipt_time = datetime.datetime(2015, 1, 1, 10, i).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2015, 1, 1, 10, i))
         # ordered with odd-numbered ids then even-numbered ids
-        formdata.evolution[-1].time = datetime.datetime(2015, 2, 1, 10 + i % 2, i).timetuple()
+        formdata.evolution[-1].time = make_aware(datetime.datetime(2015, 2, 1, 10 + i % 2, i))
         formdata.store()
 
     inversed_receipt_time_order = list(reversed([str(x) for x in sorted(ids)]))
@@ -142,7 +143,7 @@ def test_backoffice_criticality_in_formdef_listing_order(pub):
         formdata.data = {}
         formdata.just_created()
         formdata.jump_status('new')
-        formdata.receipt_time = datetime.datetime(2015, 1, 1, 10, i).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2015, 1, 1, 10, i))
         if i < 8:
             if i % 3 == 0:
                 formdata.set_criticality_level(1)

tests/backoffice_pages/test_submission.py

@@ -1,10 +1,10 @@
 import datetime
 import os
 import re
-import time
 
 import pytest
 import responses
+from django.utils.timezone import localtime, make_aware
 
 from wcs import fields
 from wcs.carddef import CardDef
@@ -213,7 +213,7 @@ def test_backoffice_submission_with_tracking_code(pub):
     assert formdata.tracking_code in resp.text
 
     # check access at a later time
-    formdata.receipt_time = time.localtime(time.time() - 3600)
+    formdata.receipt_time = localtime() - datetime.timedelta(hours=1)
     formdata.store()
     resp = app.get(formdata_location)
     assert formdata.tracking_code not in resp.text
@@ -977,7 +977,7 @@ def test_backoffice_submission_sections(pub):
     formdata.data = {}
     formdata.status = 'draft'
     formdata.backoffice_submission = True
-    formdata.receipt_time = datetime.datetime(2015, 1, 1).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2015, 1, 1))
     formdata.store()
 
     resp = app.get('/backoffice/submission/')
@@ -1016,7 +1016,7 @@ def test_backoffice_submission_drafts_order(pub):
         formdata.data = {}
         formdata.status = 'draft'
         formdata.backoffice_submission = True
-        formdata.receipt_time = datetime.datetime(2023, 11, 20 - i).timetuple()
+        formdata.receipt_time = make_aware(datetime.datetime(2023, 11, 20 - i))
         formdata.store()
         formdata_ids.append(formdata.id)
 
@@ -2073,6 +2073,19 @@ def test_backoffice_submission_sidebar_lateral_block(pub):
         == "Could not render submission lateral template ('datetime.date' object is not iterable)"
     )
 
+    formdef.submission_lateral_template = 'XX{{ "a"|add:bar }}XX'
+    formdef.store()
+
+    pub.loggederror_class.wipe()
+    resp = app.get('/backoffice/submission/form-title/')
+    partial_resp = get_lateral_block_url(resp)
+    assert partial_resp.text == ''
+    assert pub.loggederror_class.count() == 1
+    assert (
+        pub.loggederror_class.select()[0].summary
+        == "Could not render submission lateral template (missing variable \"bar\" in template)"
+    )
+
 
 def test_backoffice_submission_computed_field(pub):
     user = create_user(pub)

tests/form_pages/test_all.py

@@ -5650,7 +5650,7 @@ def test_form_edit_with_category(pub):
     assert 'f1' in resp.form.fields
 
 
-def test_form_edit_single_page(pub):
+def test_form_edit_single_or_partial_pages(pub):
     user = create_user(pub)
 
     workflow = Workflow(name='test')
@@ -5737,6 +5737,34 @@ def test_form_edit_single_page(pub):
     formdata.refresh_from_storage()
     assert formdata.data == {'2': 'a', '4': 'other change', '6': 'last change'}
 
+    # make page 2 hidden
+    formdef.fields[2].condition = {'type': 'django', 'value': 'false'}
+    formdef.store()
+
+    resp = app.get(formdata.get_url())
+    resp = resp.form.submit('button_editable').follow()
+    assert [x.text for x in resp.pyquery('#steps .wcs-step--label-text')] == ['3rd page']
+    assert [x.text for x in resp.pyquery('.buttons button')] == ['Save Changes', 'Previous', 'Cancel']
+    assert resp.pyquery('.buttons button.form-previous[hidden]')
+    assert resp.pyquery('.buttons button.form-previous[disabled]')
+    resp.form['f6'] = 'another last change'
+    resp = resp.form.submit('submit')
+    formdata.refresh_from_storage()
+    assert formdata.data == {'2': 'a', '4': 'other change', '6': 'another last change'}
+
+    # also make page 3 hidden -> 404
+    formdef.fields[4].condition = {'type': 'django', 'value': 'false'}
+    formdef.store()
+    resp = app.get(formdata.get_url())
+    resp = resp.form.submit('button_editable').follow(status=404)
+
+    # check single page mode with hidden page (also 404)
+    editable.operation_mode = 'partial'
+    workflow.store()
+    formdef.store()
+    resp = app.get(formdata.get_url())
+    resp = resp.form.submit('button_editable').follow(status=404)
+
 
 def test_form_edit_and_jump_on_submit(pub):
     wf = Workflow(name='edit and jump on submit')

tests/form_pages/test_block.py

@@ -715,6 +715,58 @@ def test_block_multi_string_modify_prefill(pub):
     assert resp.form['f3$element2$f123'].value == 'Bye World'  # updated
 
 
+def test_block_string_prefill_and_items(pub):
+    FormDef.wipe()
+    BlockDef.wipe()
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [
+        fields.StringField(
+            id='123',
+            required=True,
+            label='Test',
+            prefill={'type': 'string', 'value': '{{ form_var_foo }} World'},
+        ),
+        fields.ItemsField(
+            id='234',
+            required=False,
+            label='Items',
+            items=['Pomme', 'Poire', 'Pêche', 'Abricot'],
+        ),
+    ]
+    block.store()
+
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [
+        fields.PageField(id='0', label='1st page'),
+        fields.StringField(id='1', label='string', varname='foo'),
+        fields.PageField(id='2', label='2nd page'),
+        fields.BlockField(id='3', label='test', block_slug='foobar', max_items=5),
+    ]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    app = get_app(pub)
+    resp = app.get(formdef.get_url())
+    resp.form['f1'] = 'Hello'
+    resp = resp.form.submit('submit')  # -> 2nd page
+    assert resp.form['f3$element0$f123'].value == 'Hello World'
+    resp = resp.form.submit('f3$add_element')  # add second row
+    assert resp.form['f3$element1$f123'].value == 'Hello World'
+    resp = resp.form.submit('f3$add_element')  # add third row
+    assert resp.form['f3$element2$f123'].value == 'Hello World'
+    resp = resp.form.submit('submit')  # -> validation page
+    resp = resp.form.submit('submit')  # -> end page
+    resp = resp.follow()
+
+    formdata = formdef.data_class().select()[0]
+    assert formdata.data['3']['data'][0]['123'] == 'Hello World'
+    assert formdata.data['3']['data'][1]['123'] == 'Hello World'
+    assert formdata.data['3']['data'][2]['123'] == 'Hello World'
+
+
 def test_workflow_form_block_prefill(pub):
     FormDef.wipe()
     BlockDef.wipe()

tests/form_pages/test_draft.py

@@ -3,6 +3,7 @@ import time
 from unittest import mock
 
 import pytest
+from django.utils.timezone import make_aware
 from webtest import Upload
 
 from wcs import fields
@@ -386,7 +387,7 @@ def test_form_max_drafts(pub):
     # create another draft, not linked to user, to check it's not deleted
     another_draft = formdef.data_class()()
     another_draft.status = 'draft'
-    another_draft.receipt_time = datetime.datetime(2023, 11, 23, 0, 0).timetuple()
+    another_draft.receipt_time = make_aware(datetime.datetime(2023, 11, 23, 0, 0))
     another_draft.store()
 
     drafts = []
@@ -394,7 +395,7 @@ def test_form_max_drafts(pub):
         draft = formdef.data_class()()
         draft.user_id = user.id
         draft.status = 'draft'
-        draft.receipt_time = datetime.datetime(2023, 11, 23, 0, i).timetuple()
+        draft.receipt_time = make_aware(datetime.datetime(2023, 11, 23, 0, i))
         draft.store()
         drafts.append(draft)
 

tests/test_categories.py

@@ -7,10 +7,28 @@ import pytest
 from django.utils.encoding import force_bytes
 from quixote import cleanup
 
-from wcs.categories import CardDefCategory, Category
+from wcs.categories import (
+    BlockCategory,
+    CardDefCategory,
+    Category,
+    CommentTemplateCategory,
+    DataSourceCategory,
+    MailTemplateCategory,
+    WorkflowCategory,
+)
 
 from .utilities import clean_temporary_pub, create_temporary_pub
 
+category_classes = [
+    Category,
+    CardDefCategory,
+    BlockCategory,
+    WorkflowCategory,
+    MailTemplateCategory,
+    CommentTemplateCategory,
+    DataSourceCategory,
+]
+
 
 def setup_module(module):
     cleanup()
@@ -24,7 +42,7 @@ def teardown_module(module):
     clean_temporary_pub()
 
 
-@pytest.mark.parametrize('category_class', [Category, CardDefCategory])
+@pytest.mark.parametrize('category_class', category_classes)
 def test_store(category_class):
     category_class.wipe()
     test = category_class()
@@ -37,7 +55,7 @@ def test_store(category_class):
     assert test.description == test2.description
 
 
-@pytest.mark.parametrize('category_class', [Category, CardDefCategory])
+@pytest.mark.parametrize('category_class', category_classes)
 def test_urlname(category_class):
     category_class.wipe()
     test = category_class()
@@ -48,7 +66,7 @@ def test_urlname(category_class):
     assert test.url_name == 'test'
 
 
-@pytest.mark.parametrize('category_class', [Category, CardDefCategory])
+@pytest.mark.parametrize('category_class', category_classes)
 def test_duplicate_urlname(category_class):
     category_class.wipe()
     test = category_class()
@@ -64,7 +82,7 @@ def test_duplicate_urlname(category_class):
     assert test2.url_name == 'test-2'
 
 
-@pytest.mark.parametrize('category_class', [Category, CardDefCategory])
+@pytest.mark.parametrize('category_class', category_classes)
 def test_name_giving_a_forbidden_slug(category_class):
     category_class.wipe()
     test = category_class()
@@ -74,7 +92,7 @@ def test_name_giving_a_forbidden_slug(category_class):
     assert test.url_name == 'cat-api'
 
 
-@pytest.mark.parametrize('category_class', [Category, CardDefCategory])
+@pytest.mark.parametrize('category_class', category_classes)
 def test_sort_positions(category_class):
     category_class.wipe()
 
@@ -94,7 +112,7 @@ def test_sort_positions(category_class):
     assert categories[-1].name in ('Test 8', 'Test 9')
 
 
-@pytest.mark.parametrize('category_class', [Category, CardDefCategory])
+@pytest.mark.parametrize('category_class', category_classes)
 def test_xml_export(category_class):
     category_class.wipe()
     test = category_class()
@@ -108,7 +126,7 @@ def test_xml_export(category_class):
     assert b' id="1"' not in test.export_to_xml_string(include_id=False)
 
 
-@pytest.mark.parametrize('category_class', [Category, CardDefCategory])
+@pytest.mark.parametrize('category_class', category_classes)
 def test_xml_import(category_class):
     category_class.wipe()
     test = category_class()
@@ -190,7 +208,7 @@ def test_load_old_python2_pickle():
     assert test2.description == 'Hello world'
 
 
-@pytest.mark.parametrize('category_class', [Category, CardDefCategory])
+@pytest.mark.parametrize('category_class', category_classes)
 def test_get_by_urlname(category_class):
     category_class.wipe()
     test = category_class()
@@ -202,7 +220,7 @@ def test_get_by_urlname(category_class):
     assert test.id == test2.id
 
 
-@pytest.mark.parametrize('category_class', [Category, CardDefCategory])
+@pytest.mark.parametrize('category_class', category_classes)
 def test_has_urlname(category_class):
     category_class.wipe()
     test = category_class()
@@ -215,7 +233,7 @@ def test_has_urlname(category_class):
     assert not category_class.has_urlname('foobar')
 
 
-@pytest.mark.parametrize('category_class', [Category, CardDefCategory])
+@pytest.mark.parametrize('category_class', category_classes)
 def test_remove_self(category_class):
     category_class.wipe()
     test = category_class()

tests/test_content_snapshots.py

@@ -1357,7 +1357,7 @@ def test_api_formdata_at(pub, user, access, role):
     evo.add_part(part)
 
     evo = Evolution(formdata=formdata)
-    evo.time = time.localtime()
+    evo.time = localtime()
     evo.status = formdata.status
     part = ContentSnapshotPart(formdata=formdata, old_data={'0': 'bar', 'bo1': 'foo'})
     part.new_data = {'0': 'baz', 'bo1': 'foo'}
@@ -1366,7 +1366,7 @@ def test_api_formdata_at(pub, user, access, role):
     formdata.evolution.append(evo)
 
     evo = Evolution(formdata=formdata)
-    evo.time = time.localtime()
+    evo.time = localtime()
     evo.status = formdata.status
     part = ContentSnapshotPart(formdata=formdata, old_data={'0': 'baz', 'bo1': 'foo'})
     part.new_data = {'0': 'foooo', '1': 'unknown', 'bo1': 'foo'}
@@ -1375,7 +1375,7 @@ def test_api_formdata_at(pub, user, access, role):
     formdata.evolution.append(evo)
 
     evo = Evolution(formdata=formdata)
-    evo.time = time.localtime()
+    evo.time = localtime()
     evo.status = formdata.status
     part = ContentSnapshotPart(formdata=formdata, old_data={'0': 'foooo', '1': 'unknown', 'bo1': 'foo'})
     part.new_data = {'0': 'fooo', 'bo1': 'foo'}

tests/test_ctl.py

@@ -1,4 +1,3 @@
-import collections
 import json
 import os
 import pickle
@@ -17,10 +16,7 @@ from django.core.management import CommandError, call_command
 import wcs.qommon.ctl
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
-from wcs.ctl.delete_tenant import CmdDeleteTenant
 from wcs.ctl.management.commands.trigger_jumps import select_and_jump_formdata
-from wcs.ctl.rebuild_indexes import rebuild_vhost_indexes
-from wcs.ctl.wipe_data import CmdWipeData
 from wcs.fields import EmailField, ItemField, PageField, StringField
 from wcs.formdef import FormDef
 from wcs.mail_templates import MailTemplate
@@ -124,35 +120,32 @@ def test_wipe_formdata(pub):
     formdata_2.store()
     assert form_2.data_class().count() == 1
 
-    wipe_cmd = CmdWipeData()
-
-    # check command options
-    options, args = wipe_cmd.parse_args(['--all'])
-    assert options.all
-
-    options, args = wipe_cmd.parse_args([form_1.url_name, form_2.url_name])
-    assert form_1.url_name in args
-    assert form_2.url_name in args
-
-    sub_options_class = collections.namedtuple('Options', ['all'])
-    sub_options = sub_options_class(False)
+    # no support for --all-tenants
+    with pytest.raises(CommandError):
+        call_command('wipe_data', '--all-tenants')
 
     # test with no options
-    wipe_cmd.wipe(pub, sub_options, [])
+    call_command('wipe_data', '--domain=example.net')
     assert form_1.data_class().count() == 1
     assert form_2.data_class().count() == 1
 
     # wipe one form formdatas
-    wipe_cmd.wipe(pub, sub_options, [form_1.url_name])
+    call_command('wipe_data', '--domain=example.net', '--forms=%s' % form_1.url_name)
     assert form_1.data_class().count() == 0
     assert form_2.data_class().count() == 1
 
     # wipe all formdatas
-    sub_options = sub_options_class(True)
-    wipe_cmd.wipe(pub, sub_options, [])
+    call_command('wipe_data', '--domain=example.net', '--all')
     assert form_1.data_class().count() == 0
     assert form_2.data_class().count() == 0
 
+    # exclude some forms
+    formdata_1.store()
+    formdata_2.store()
+    call_command('wipe_data', '--domain=example.net', '--all', '--exclude-forms=%s' % form_2.url_name)
+    assert form_1.data_class().count() == 0
+    assert form_2.data_class().count() == 1
+
 
 def test_trigger_jumps(pub):
     Workflow.wipe()
@@ -232,15 +225,11 @@ def test_trigger_jumps(pub):
 
 def test_delete_tenant_with_sql(freezer):
     pub = create_temporary_pub()
-    delete_cmd = CmdDeleteTenant()
 
     assert os.path.isdir(pub.app_dir)
 
-    sub_options_class = collections.namedtuple('Options', ['force_drop'])
-    sub_options = sub_options_class(False)
-
     freezer.move_to('2018-12-01T00:00:00')
-    delete_cmd.delete_tenant(pub, sub_options, [])
+    call_command('delete_tenant', '--vhost=example.net')
 
     assert not os.path.isdir(pub.app_dir)
     parent_dir = os.path.dirname(pub.app_dir)
@@ -260,8 +249,7 @@ def test_delete_tenant_with_sql(freezer):
     clean_temporary_pub()
     pub = create_temporary_pub()
 
-    sub_options = sub_options_class(True)
-    delete_cmd.delete_tenant(pub, sub_options, [])
+    call_command('delete_tenant', '--vhost=example.net', '--force-drop')
 
     conn, cur = get_connection_and_cursor(new=True)
 
@@ -287,13 +275,13 @@ def test_delete_tenant_with_sql(freezer):
     clean_temporary_pub()
     pub = create_temporary_pub()
 
-    cleanup_connection()
-    sub_options = sub_options_class(True)
     pub.cfg['postgresql']['createdb-connection-params'] = {
         'user': pub.cfg['postgresql']['user'],
         'database': 'postgres',
     }
-    delete_cmd.delete_tenant(pub, sub_options, [])
+    pub.write_cfg()
+    pub.cleanup()
+    call_command('delete_tenant', '--vhost=example.net', '--force-drop')
 
     connect_kwargs = {'dbname': 'postgres', 'user': pub.cfg['postgresql']['user']}
     pgconn = psycopg2.connect(**connect_kwargs)
@@ -313,12 +301,13 @@ def test_delete_tenant_with_sql(freezer):
     pub = create_temporary_pub()
     cleanup_connection()
 
-    sub_options = sub_options_class(False)
     pub.cfg['postgresql']['createdb-connection-params'] = {
         'user': pub.cfg['postgresql']['user'],
         'database': 'postgres',
     }
-    delete_cmd.delete_tenant(pub, sub_options, [])
+    pub.write_cfg()
+    call_command('delete_tenant', '--vhost=example.net')
+    cleanup_connection()
 
     pgconn = psycopg2.connect(**connect_kwargs)
     pgconn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
@@ -351,39 +340,6 @@ def test_delete_tenant_with_sql(freezer):
     clean_temporary_pub()
 
 
-def test_delete_tenant_without_sql():
-    pub = create_temporary_pub()
-    delete_cmd = CmdDeleteTenant()
-
-    assert os.path.isdir(pub.app_dir)
-
-    sub_options_class = collections.namedtuple('Options', ['force_drop'])
-    sub_options = sub_options_class(False)
-
-    delete_cmd.delete_tenant(pub, sub_options, [])
-
-    assert not os.path.isdir(pub.app_dir)
-    parent_dir = os.path.dirname(pub.app_dir)
-    if not [filename for filename in os.listdir(parent_dir) if 'removed' in filename]:
-        assert False
-
-    clean_temporary_pub()
-
-    pub = create_temporary_pub()
-    assert os.path.isdir(pub.app_dir)
-
-    sub_options = sub_options_class(True)
-
-    delete_cmd.delete_tenant(pub, sub_options, [])
-
-    assert not os.path.isdir(pub.app_dir)
-    parent_dir = os.path.dirname(pub.app_dir)
-    if [filename for filename in os.listdir(parent_dir) if 'removed' in filename]:
-        assert False
-
-    clean_temporary_pub()
-
-
 def test_rebuild_indexes(pub):
     form = FormDef()
     form.name = 'example'
@@ -393,11 +349,11 @@ def test_rebuild_indexes(pub):
 
     os.unlink(os.path.join(pub.app_dir, 'formdefs-url_name', 'example'))
     os.symlink('../formdefs/1', os.path.join(pub.app_dir, 'formdefs-url_name', 'XXX'))
-    rebuild_vhost_indexes(pub, destroy=False)
+    call_command('rebuild_indexes', '--all-tenants')
     assert 'example' in os.listdir(os.path.join(pub.app_dir, 'formdefs-url_name'))
     assert 'XXX' in os.listdir(os.path.join(pub.app_dir, 'formdefs-url_name'))
 
-    rebuild_vhost_indexes(pub, destroy=True)
+    call_command('rebuild_indexes', '--all-tenants', '--destroy')
     assert os.listdir(os.path.join(pub.app_dir, 'formdefs-url_name')) == ['example']
 
 
@@ -1045,6 +1001,7 @@ def test_configdb(pub):
 def test_replace_python(pub, alt_tempdir, source_type):
     FormDef.wipe()
     Workflow.wipe()
+    MailTemplate.wipe()
 
     formdef = FormDef()
     formdef.name = 'Foo'
@@ -1055,6 +1012,12 @@ def test_replace_python(pub, alt_tempdir, source_type):
             size='40',
             required=True,
             condition={'type': 'python', 'value': 'python condition'},
+            post_conditions=[
+                {
+                    'condition': {'type': 'python', 'value': 'python condition'},
+                    'error_message': 'You shall not pass.',
+                },
+            ],
         ),
         StringField(id='2', label='Foo', prefill={'type': 'formula', 'value': 'form_var_foo'}),
         StringField(id='3', label='Bar', prefill={'type': 'formula', 'value': 'form_var_bar'}),
@@ -1084,6 +1047,7 @@ def test_replace_python(pub, alt_tempdir, source_type):
     item = st1.add_action('sendmail')
     item.to = ['=form_var_foo']
     item.subject = '=form_var_foo'
+    item.attachments = ['{{ form_var_xxx }}', 'getattr(form_attachments, "form_var_bar", None)']
 
     item = st1.add_action('webservice_call')
     item.varname = 'xxx'
@@ -1100,6 +1064,11 @@ def test_replace_python(pub, alt_tempdir, source_type):
 
     workflow.store()
 
+    mail_template = MailTemplate(name='test mail template')
+    mail_template.subject = '=form_var_foo'
+    mail_template.attachments = ['{{ form_var_xxx }}', 'getattr(form_attachments, "form_var_bar", None)']
+    mail_template.store()
+
     with open(os.path.join(alt_tempdir, 'replacements.json'), 'w') as fp:
         replacements = {
             'conditions': {
@@ -1108,6 +1077,7 @@ def test_replace_python(pub, alt_tempdir, source_type):
             'templates': {
                 'form_var_foo': '{{ form_var_foo }}',
                 'datetime.date(2023, 12, 27)': '2023-12-27',
+                'getattr(form_attachments, "form_var_bar", None)': '{{ form_var_bar }}',
             },
         }
         json.dump(replacements, fp)
@@ -1128,6 +1098,12 @@ def test_replace_python(pub, alt_tempdir, source_type):
 
     formdef.refresh_from_storage()
     assert formdef.fields[0].condition == {'type': 'django', 'value': 'django condition'}
+    assert formdef.fields[0].post_conditions == [
+        {
+            'condition': {'type': 'django', 'value': 'django condition'},
+            'error_message': 'You shall not pass.',
+        },
+    ]
     assert formdef.fields[1].prefill == {'type': 'string', 'value': '{{ form_var_foo }}'}
     assert formdef.fields[2].prefill == {'type': 'formula', 'value': 'form_var_bar'}  # no replacement
     workflow.refresh_from_storage()
@@ -1137,6 +1113,7 @@ def test_replace_python(pub, alt_tempdir, source_type):
     assert workflow.possible_status[0].items[3].mappings[0].expression == '{{ form_var_foo }}'
     assert workflow.possible_status[0].items[4].subject == '{{ form_var_foo }}'
     assert workflow.possible_status[0].items[4].to == ['{{ form_var_foo }}']
+    assert workflow.possible_status[0].items[4].attachments == ['{{ form_var_xxx }}', '{{ form_var_bar }}']
     assert workflow.possible_status[0].items[5].post_data == {'str': 'abcd', 'expr': '{{ form_var_foo }}'}
     assert workflow.possible_status[0].items[5].qs_data == {
         'str': 'abcd',
@@ -1148,3 +1125,7 @@ def test_replace_python(pub, alt_tempdir, source_type):
     assert workflow.global_actions[0].triggers[1].anchor_template == '2023-12-27'
     assert workflow.global_actions[0].triggers[2].anchor == 'python'
     assert workflow.global_actions[0].triggers[2].anchor_expression == 'nope'
+
+    mail_template.refresh_from_storage()
+    assert mail_template.subject == '{{ form_var_foo }}'
+    assert mail_template.attachments == ['{{ form_var_xxx }}', '{{ form_var_bar }}']

tests/test_formdata.py

@@ -9,6 +9,7 @@ from unittest import mock
 
 import pytest
 from django.utils import formats
+from django.utils.timezone import localtime, make_aware
 from quixote import get_publisher, get_request
 from quixote.http_request import Upload
 
@@ -392,7 +393,7 @@ def test_get_last_update_time(pub, formdef):
 
     time.sleep(1)
     evo = Evolution(formdata=formdata)
-    evo.time = time.localtime()
+    evo.time = localtime()
     evo.status = formdata.status
     evo.comment = 'hello world'
     formdata.evolution.append(evo)
@@ -442,13 +443,13 @@ def test_clean_drafts(pub):
 
     d = formdef.data_class()()
     d.status = 'draft'
-    d.receipt_time = time.localtime()
+    d.receipt_time = localtime()
     d.store()
     d_id1 = d.id
 
     d = formdef.data_class()()
     d.status = 'draft'
-    d.receipt_time = time.localtime(0)  # epoch, 1970-01-01
+    d.receipt_time = make_aware(datetime.datetime(1970, 1, 1))
     d.store()
 
     assert formdef.data_class().count() == 2
@@ -460,7 +461,7 @@ def test_clean_drafts(pub):
 
     d = formdef.data_class()()
     d.status = 'draft'
-    d.receipt_time = time.localtime(time.time() - 86400 * 5)
+    d.receipt_time = localtime() - datetime.timedelta(days=5)
     d.store()
     clean_drafts(pub)
     assert formdef.data_class().count() == 2
@@ -575,9 +576,9 @@ def test_get_json_export_dict_evolution(pub, local_user):
     d = formdef.data_class()()
     d.status = 'wf-%s' % st_new.id
     d.user_id = local_user.id
-    d.receipt_time = time.localtime()
+    d.receipt_time = localtime()
     evo = Evolution(formdata=d)
-    evo.time = time.localtime()
+    evo.time = localtime()
     evo.status = 'wf-%s' % st_new.id
     evo.who = '_submitter'
     d.evolution = [evo]
@@ -587,7 +588,7 @@ def test_get_json_export_dict_evolution(pub, local_user):
     evo.add_part(JournalAssignationErrorPart('summary', 'label'))
     d.store()
     evo = Evolution(formdata=d)
-    evo.time = time.localtime()
+    evo.time = localtime()
     evo.status = 'wf-%s' % st_finished.id
     evo.who = '_submitter'
     d.evolution.append(evo)
@@ -731,25 +732,25 @@ def test_evolution_get_status(pub):
     d.evolution = []
 
     evo = Evolution(formdata=d)
-    evo.time = time.localtime()
+    evo.time = localtime()
     evo.status = 'wf-%s' % st_new.id
     d.evolution.append(evo)
 
     evo = Evolution(formdata=d)
-    evo.time = time.localtime()
+    evo.time = localtime()
     d.evolution.append(evo)
 
     evo = Evolution(formdata=d)
-    evo.time = time.localtime()
+    evo.time = localtime()
     d.evolution.append(evo)
 
     evo = Evolution(formdata=d)
-    evo.time = time.localtime()
+    evo.time = localtime()
     evo.status = 'wf-%s' % st_finished.id
     d.evolution.append(evo)
 
     evo = Evolution(formdata=d)
-    evo.time = time.localtime()
+    evo.time = localtime()
     d.evolution.append(evo)
 
     d.store()
@@ -833,9 +834,9 @@ def test_lazy_formdata(pub, variable_test_data):
     formdef = FormDef.select()[0]
     formdata = formdef.data_class().select()[0]
     lazy_formdata = LazyFormData(formdata)
-    assert lazy_formdata.receipt_date == time.strftime('%Y-%m-%d', formdata.receipt_time)
-    assert lazy_formdata.receipt_time == formats.time_format(datetime.datetime(*formdata.receipt_time[:6]))
-    assert lazy_formdata.last_update_datetime.timetuple()[:6] == formdata.last_update_time[:6]
+    assert lazy_formdata.receipt_date == formdata.receipt_time.strftime('%Y-%m-%d')
+    assert lazy_formdata.receipt_time == formats.time_format(formdata.receipt_time)
+    assert lazy_formdata.last_update_datetime.timetuple()[:6] == formdata.last_update_time.timetuple()[:6]
     assert lazy_formdata.internal_id == formdata.id
     assert lazy_formdata.name == 'foobarlazy'
     assert lazy_formdata.display_name == 'foobarlazy #%s' % formdata.get_display_id()
@@ -5369,7 +5370,7 @@ def test_get_visible_status(pub, local_user):
     # create evolution [new, empty, finished, empty]
     for status in (st_new, None, st_finished, None):
         evo = Evolution(formdata=formdata)
-        evo.time = time.localtime()
+        evo.time = localtime()
         if status:
             evo.status = 'wf-%s' % status.id
         formdata.evolution.append(evo)
@@ -5630,7 +5631,7 @@ def test_get_status_datetime(pub, freezer):
 
     freezer.move_to(datetime.datetime(2023, 10, 31, 12, 0))
     evo = Evolution(formdata=formdata)
-    evo.time = time.localtime()
+    evo.time = localtime()
     formdata.evolution.append(evo)
     assert formdata.get_status_datetime(status=st_next) == formdata.evolution[1].time
     assert formdata.get_status_datetime(status=st_next, latest=True) == formdata.evolution[1].time

tests/test_hobo.py

@@ -2,12 +2,14 @@ import copy
 import json
 import os
 import pickle
+import random
 import shutil
 import tempfile
 import urllib.parse
 import zipfile
 from unittest import mock
 
+import psycopg2
 import pytest
 from django.core.management import call_command
 from quixote import cleanup
@@ -257,18 +259,28 @@ def deploy_setup(alt_tempdir):
         del hobo_json['services'][1]  # authentic
         fd.write(json.dumps(hobo_json))
     skeleton_dir = os.path.join(CompatWcsPublisher.APP_DIR, 'skeletons')
-    if not os.path.exists(skeleton_dir):
-        os.mkdir(skeleton_dir)
-        with open(os.path.join(skeleton_dir, 'export-test.wcs'), 'wb') as f:
-            with zipfile.ZipFile(f, 'w') as z:
-                CONFIG = {
-                    'postgresql': {
-                        'createdb-connection-params': {'database': 'postgres', 'user': os.environ['USER']},
-                        'database-template-name': 'wcstests_hobo_%s',
-                        'user': os.environ['USER'],
-                    }
+    os.mkdir(skeleton_dir)
+    db_template_name = 'wcstests_hobo_%d_%%s' % random.randint(0, 100000)
+    with open(os.path.join(skeleton_dir, 'export-test.wcs'), 'wb') as f:
+        with zipfile.ZipFile(f, 'w') as z:
+            CONFIG = {
+                'postgresql': {
+                    'createdb-connection-params': {'database': 'postgres', 'user': os.environ['USER']},
+                    'database-template-name': db_template_name,
+                    'user': os.environ['USER'],
                 }
-                z.writestr('config.json', json.dumps(CONFIG))
+            }
+            z.writestr('config.json', json.dumps(CONFIG))
+    yield True
+    shutil.rmtree(skeleton_dir)
+
+    conn = psycopg2.connect(user=os.environ['USER'], dbname='postgres')
+    conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    cur = conn.cursor()
+    cur.execute('DROP DATABASE IF EXISTS %s' % db_template_name % 'wcs_example_net')
+    cur.execute('DROP DATABASE IF EXISTS %s' % db_template_name % 'wcs2_example_net')
+    cur.close()
+    conn.commit()
 
 
 def test_configure_site_options(setuptest, alt_tempdir):
@@ -519,6 +531,8 @@ def test_deploy(setuptest, alt_tempdir, deploy_setup, settings):
     with open(os.path.join(alt_tempdir, 'tenants', 'wcs.example.net', 'config.pck'), 'rb') as fd:
         pub_cfg = pickle.load(fd)
     assert pub_cfg['language'] == {'language': 'fr'}
+    cleanup_connection()
+    cleanup()
 
 
 def test_configure_postgresql(setuptest, alt_tempdir, deploy_setup, settings):
@@ -634,6 +648,8 @@ def test_redeploy(setuptest, alt_tempdir, deploy_setup, settings):
             'http://wcs.example.net/',
             'http://wcs2.example.net/',
         }
+    cleanup_connection()
+    cleanup()
 
 
 def test_configure_site_options_legacy_urls(setuptest, alt_tempdir):

tests/test_misc.py

@@ -691,11 +691,12 @@ def test_http_request_url_switch(mock_request, pub):
     mock_request.reset_mock()
 
 
-def test_validate_phone_fr():
+def test_validate_phone_fr(pub):
     valid = [
         '0123456789',
         '+33123456789',
         '+590690000102',
+        '06 92 32 00 00',  # valid number in (+262) but not in (+33)
     ]
     invalid = [
         '1234559',

tests/test_snapshots.py

@@ -1435,25 +1435,49 @@ def test_category_snapshot_browse(pub):
 
     Category.wipe()
     category = Category(name='test')
+    category.position = 42
     category.store()
     assert pub.snapshot_class.count() == 1
     # check calling .store() without changes doesn't create snapshots
     category.store()
     assert pub.snapshot_class.count() == 1
+    category.name = 'foobar'
+    category.store()
+    assert pub.snapshot_class.count() == 2
 
     app = login(get_app(pub))
 
     resp = app.get('/backoffice/forms/categories/%s/' % category.id)
     resp = resp.click('History')
-    snapshot = pub.snapshot_class.select_object_history(category)[0]
+    snapshot = pub.snapshot_class.select_object_history(category)[1]
+    snapshot = snapshot.get_latest(
+        snapshot.object_type, snapshot.object_id, complete=True, max_timestamp=snapshot.timestamp
+    )
+    assert snapshot.patch is None
+    assert 'position' not in snapshot.serialization
     resp = resp.click(href='%s/view/' % snapshot.id)
     assert 'This category is readonly' in resp.text
+    assert 'inspect' not in resp
     assert '<p>%s</p>' % localstrftime(snapshot.timestamp) in resp.text
     with pytest.raises(IndexError):
-        resp = resp.click('Edit')
+        resp.click('Edit')
+    resp = app.get('/backoffice/forms/categories/%s/' % category.id)
+    resp = resp.click('History')
+    resp = resp.click(href='%s/restore' % snapshot.id)
+    assert resp.form['action'].value == 'as-new'
+    resp = resp.form.submit('submit')
+    assert Category.count() == 2
+    new_category = Category.get(resp.location.split('/')[-2])
+    assert new_category.position == 43
 
-    resp = app.get('/backoffice/forms/categories/%s/history/%s/view/' % (category.id, snapshot.id))
-    assert 'inspect' not in resp
+    resp = app.get('/backoffice/forms/categories/%s/' % category.id)
+    resp = resp.click('History')
+    resp = resp.click(href='%s/restore' % snapshot.id)
+    resp.form['action'].value = 'overwrite'
+    resp = resp.form.submit('submit')
+    assert Category.count() == 2
+    category.refresh_from_storage()
+    assert category.position == 42
 
 
 def test_snapshots_test_results(pub):

tests/test_sql.py

@@ -10,6 +10,7 @@ import zipfile
 
 import psycopg2
 import pytest
+from django.utils.timezone import localtime, make_aware
 from django.utils.timezone import now as tz_now
 
 import wcs.sql_criterias as st
@@ -20,7 +21,6 @@ from wcs.data_sources import NamedDataSource
 from wcs.formdata import Evolution
 from wcs.formdef import FormDef
 from wcs.qommon import force_str
-from wcs.testdef import TestDef
 from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem
 from wcs.workflows import (
     ActionsTracingEvolutionPart,
@@ -390,7 +390,7 @@ def test_sql_evolution(formdef):
     assert len(formdata.evolution) == 1
 
     evo = Evolution(formdata=formdata)
-    evo.time = time.localtime()
+    evo.time = localtime()
     evo.status = formdata.status
     evo.comment = 'hello world'
     formdata.evolution.append(evo)
@@ -412,7 +412,7 @@ def test_sql_evolution_change(formdef):
     assert len(formdata.evolution) == 1
 
     evo = Evolution(formdata=formdata)
-    evo.time = time.localtime()
+    evo.time = localtime()
     evo.status = formdata.status
     evo.comment = 'hello world'
     formdata.evolution.append(evo)
@@ -441,7 +441,7 @@ def test_sql_multiple_evolutions(formdef):
         formdata = data_class.get(id)
 
         evo = Evolution(formdata=formdata)
-        evo.time = time.localtime()
+        evo.time = localtime()
         evo.status = formdata.status
         evo.comment = 'hello world %d' % i
         formdata.evolution.append(evo)
@@ -930,24 +930,19 @@ def test_sql_table_select_datetime(pub):
     data_class = test_formdef.data_class(mode='sql')
     assert data_class.count() == 0
 
-    d = datetime.datetime(2014, 1, 1)
+    d = make_aware(datetime.datetime(2014, 1, 1))
     for i in range(50):
         t = data_class()
-        t.receipt_time = (d + datetime.timedelta(days=i)).timetuple()
+        t.receipt_time = d + datetime.timedelta(days=i)
         t.store()
 
     assert data_class.count() == 50
     assert len(data_class.select()) == 50
 
-    assert len(data_class.select(lambda x: x.receipt_time == d.timetuple())) == 1
-    assert len(data_class.select([st.Equal('receipt_time', d.timetuple())])) == 1
-    assert (
-        len(data_class.select([st.Less('receipt_time', (d + datetime.timedelta(days=20)).timetuple())])) == 20
-    )
-    assert (
-        len(data_class.select([st.Greater('receipt_time', (d + datetime.timedelta(days=20)).timetuple())]))
-        == 29
-    )
+    assert len(data_class.select(lambda x: x.receipt_time == d)) == 1
+    assert len(data_class.select([st.Equal('receipt_time', d)])) == 1
+    assert len(data_class.select([st.Less('receipt_time', d + datetime.timedelta(days=20))])) == 20
+    assert len(data_class.select([st.Greater('receipt_time', d + datetime.timedelta(days=20))])) == 29
     assert len(data_class.select([st.Equal('receipt_time', datetime.date(1900, 1, 1).timetuple())])) == 0
     assert len(data_class.select([st.Equal('receipt_time', datetime.date(1, 1, 1))])) == 0
     assert len(data_class.select([st.Greater('receipt_time', datetime.date(1, 1, 1))])) == 50
@@ -1611,7 +1606,7 @@ def test_views_fts(pub):
 def test_select_any_formdata(pub):
     drop_formdef_tables()
 
-    now = datetime.datetime.now()
+    now = localtime()
 
     cnt = 0
     for i in range(5):
@@ -1626,7 +1621,7 @@ def test_select_any_formdata(pub):
             formdata.just_created()
             formdata.user_id = '%s' % ((i + j) % 11)
             # set receipt_time to make sure all entries are unique.
-            formdata.receipt_time = (now + datetime.timedelta(seconds=cnt)).timetuple()
+            formdata.receipt_time = now + datetime.timedelta(seconds=cnt)
             formdata.status = ['wf-new', 'wf-accepted', 'wf-rejected', 'wf-finished'][(i + j) % 4]
             if j < 5:
                 formdata.submission_channel = 'mail'
@@ -1670,7 +1665,7 @@ def test_select_any_formdata(pub):
 def test_load_all_evolutions_on_any_formdata(pub):
     drop_formdef_tables()
 
-    now = datetime.datetime.now()
+    now = localtime()
 
     cnt = 0
     for i in range(5):
@@ -1685,7 +1680,7 @@ def test_load_all_evolutions_on_any_formdata(pub):
             formdata.just_created()
             formdata.user_id = '%s' % ((i + j) % 11)
             # set receipt_time to make sure all entries are unique.
-            formdata.receipt_time = (now + datetime.timedelta(seconds=cnt)).timetuple()
+            formdata.receipt_time = now + datetime.timedelta(seconds=cnt)
             formdata.status = ['wf-new', 'wf-accepted', 'wf-rejected', 'wf-finished'][(i + j) % 4]
             formdata.store()
             cnt += 1
@@ -1849,8 +1844,8 @@ def test_last_update_time(pub):
     formdata1.just_created()
     formdata1.evolution[0].comment = 'comment'
     formdata1.jump_status('st1')  # will add another evolution entry
-    formdata1.evolution[0].time = datetime.datetime(2015, 1, 1, 0, 0, 0).timetuple()
-    formdata1.evolution[1].time = datetime.datetime(2015, 1, 2, 0, 0, 0).timetuple()
+    formdata1.evolution[0].time = make_aware(datetime.datetime(2015, 1, 1, 0, 0, 0))
+    formdata1.evolution[1].time = make_aware(datetime.datetime(2015, 1, 2, 0, 0, 0))
     formdata1.store()
 
     formdata2 = data_class()
@@ -1858,8 +1853,8 @@ def test_last_update_time(pub):
     formdata2.just_created()
     formdata2.evolution[0].comment = 'comment'
     formdata2.jump_status('st1')  # will add another evolution entry
-    formdata2.evolution[0].time = datetime.datetime(2015, 1, 3, 0, 0, 0).timetuple()
-    formdata2.evolution[1].time = datetime.datetime(2015, 1, 4, 0, 0, 0).timetuple()
+    formdata2.evolution[0].time = make_aware(datetime.datetime(2015, 1, 3, 0, 0, 0))
+    formdata2.evolution[1].time = make_aware(datetime.datetime(2015, 1, 4, 0, 0, 0))
     formdata2.store()
 
     cur.execute('''SELECT COUNT(*) FROM wcs_all_forms''')
@@ -2167,7 +2162,7 @@ def test_migration_30_anonymize_evo_who(pub):
     formdata.anonymised = datetime.datetime.now()
     evo = Evolution(formdata)
     evo.who = user.id
-    evo.time = time.localtime()
+    evo.time = localtime()
     formdata.evolution.append(evo)
     formdata.store()
 
@@ -2727,24 +2722,6 @@ def test_python_datasource_migration(pub):
     assert sql.is_reindex_needed('python_ds_migration', conn=conn, cur=cur) is False
 
 
-def test_sql_testdef_unicity(pub):
-    testdef = TestDef()
-    testdef.slug = 'test-1'
-    testdef.object_type = 'formdef'
-    testdef.object_id = '1'
-    testdef.store()
-
-    # same slug, different object_id
-    testdef.id = None
-    testdef.object_id = '2'
-    testdef.store()
-
-    # same slug, object_id and object_type
-    testdef.id = None
-    with pytest.raises(psycopg2.errors.UniqueViolation):
-        testdef.store()
-
-
 def test_form_tokens_migration(pub):
     conn, cur = sql.get_connection_and_cursor()
     cur.execute('UPDATE wcs_meta SET value = 70 WHERE key = %s', ('sql_level',))
@@ -2789,7 +2766,7 @@ def test_workflow_traces_initial_migration(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.evolution[-1].time = (datetime.datetime.now() - datetime.timedelta(seconds=11)).timetuple()
+    formdata.evolution[-1].time = localtime() - datetime.timedelta(seconds=11)
     action_part = ActionsTracingEvolutionPart()
     action_part.event = 'frontoffice-created'
     action_part.actions = [
@@ -2798,7 +2775,7 @@ def test_workflow_traces_initial_migration(pub):
     ]
     formdata.evolution[-1].add_part(action_part)
     formdata.evolution.append(Evolution(formdata))
-    formdata.evolution[-1].time = (datetime.datetime.now() - datetime.timedelta(seconds=8)).timetuple()
+    formdata.evolution[-1].time = localtime() - datetime.timedelta(seconds=8)
     action_part = ActionsTracingEvolutionPart()
     action_part.event = 'timeout-jump'
     action_part.event_args = ('xxx',)
@@ -2807,7 +2784,7 @@ def test_workflow_traces_initial_migration(pub):
     ]
     formdata.evolution[-1].add_part(action_part)
     formdata.evolution.append(Evolution(formdata))
-    formdata.evolution[-1].time = (datetime.datetime.now() - datetime.timedelta(seconds=6)).timetuple()
+    formdata.evolution[-1].time = localtime() - datetime.timedelta(seconds=6)
     action_part = ActionsTracingEvolutionPart()
     action_part.event = 'global-action-timeout'
     action_part.event_args = ('xxx2', 'xxx3')
@@ -2816,7 +2793,7 @@ def test_workflow_traces_initial_migration(pub):
     ]
     formdata.evolution[-1].add_part(action_part)
     formdata.evolution.append(Evolution(formdata))
-    formdata.evolution[-1].time = (datetime.datetime.now() - datetime.timedelta(seconds=4)).timetuple()
+    formdata.evolution[-1].time = localtime() - datetime.timedelta(seconds=4)
     action_part = ActionsTracingEvolutionPart()
     action_part.event = 'global-api-trigger'
     action_part.event_args = ('xxx2',)
@@ -2829,7 +2806,7 @@ def test_workflow_traces_initial_migration(pub):
 
     formdata2 = formdef.data_class()()
     formdata2.just_created()
-    formdata2.evolution[-1].time = (datetime.datetime.now() - datetime.timedelta(seconds=2)).timetuple()
+    formdata2.evolution[-1].time = localtime() - datetime.timedelta(seconds=2)
     action_part = ActionsTracingEvolutionPart()
     action_part.event = 'workflow-created'
     action_part.external_workflow_id = '1'

tests/test_templates.py

@@ -449,6 +449,8 @@ def test_decimal_templatetag(pub):
     assert tmpl.render({'plop': 12345.678}) == '12345.678'
     assert tmpl.render({'plop': None}) == '0'
     assert tmpl.render({'plop': 0}) == '0'
+    assert tmpl.render({'plop': ['foo', 'bar']}) == '0'
+    assert tmpl.render({'plop': ['a', 'b', 'c']}) == '0'
 
     tmpl = Template('{{ plop|decimal:3 }}')
     assert tmpl.render({'plop': '3.14'}) == '3.140'
@@ -1649,3 +1651,16 @@ def test_with_auth(pub):
         Template('{{ service_url|with_auth:"username:password" }}').render(context)
         == 'https://username:password@www.example.net/api/whatever?x=y'
     )
+
+
+def test_check_no_duplicates(pub):
+    pub.loggederror_class.wipe()
+    context = {'value1': ['a', 'b', 'c'], 'value2': ['a', 'a', 'b', 'c'], 'value3': None, 'value4': '12'}
+    assert Template('{% if value1|check_no_duplicates %}ok{% else %}nok{% endif %}').render(context) == 'ok'
+    assert Template('{% if value2|check_no_duplicates %}ok{% else %}nok{% endif %}').render(context) == 'nok'
+    assert Template('{% if value3|check_no_duplicates %}ok{% else %}nok{% endif %}').render(context) == 'ok'
+    assert pub.loggederror_class.count() == 0
+    assert Template('{% if value4|check_no_duplicates %}ok{% else %}nok{% endif %}').render(context) == 'nok'
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.summary == '|check_no_duplicates not used on a list (12)'

tests/test_testdef.py

@@ -1,18 +1,22 @@
 import datetime
 import decimal
+import io
 import json
 import time
+import xml.etree.ElementTree as ET
+from unittest import mock
 
 import pytest
 import responses
+from django.utils.timezone import make_aware
 
-from wcs import fields
+from wcs import fields, workflow_tests
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.formdef import FormDef
 from wcs.qommon.http_request import HTTPRequest
 from wcs.qommon.upload_storage import PicklableUpload
-from wcs.testdef import TestDef, TestError
+from wcs.testdef import TestDef, TestDefXmlProxy, TestError, TestResult, WebserviceResponse
 from wcs.wscalls import NamedWsCall
 
 from .utilities import clean_temporary_pub, create_temporary_pub
@@ -28,6 +32,8 @@ def pub():
 
     FormDef.wipe()
     BlockDef.wipe()
+    WebserviceResponse.wipe()
+    NamedWsCall.wipe()
     return pub
 
 
@@ -35,22 +41,109 @@ def teardown_module(module):
     clean_temporary_pub()
 
 
-def test_testdef_slug_generation(pub):
-    testdef = TestDef()
+def test_testdef_export_to_xml(pub):
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.fields = [
+        fields.ItemsField(id='1', label='Test', items=['foo', 'bar', 'baz']),
+        fields.BoolField(id='2', label='Check', varname='check'),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.data['1'] = ['foo', 'baz']
+    formdata.data['2'] = True
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertStatus(status_name='End status'),
+    ]
     testdef.name = 'test'
-    testdef.object_type = 'formdef'
-    testdef.object_id = '1'
+    testdef.expected_error = 'xxx'
     testdef.store()
-    assert testdef.slug == 'test'
 
-    testdef.slug = testdef.id = None
-    testdef.store()
-    assert testdef.slug == 'test-2'
+    response = WebserviceResponse()
+    response.testdef_id = testdef.id
+    response.name = 'Fake response'
+    response.store()
 
-    testdef.slug = testdef.id = None
-    testdef.object_id = '2'
-    testdef.store()
-    assert testdef.slug == 'test'
+    testdef_xml = ET.tostring(testdef.export_to_xml())
+    TestDef.wipe()
+    workflow_tests.WorkflowTests.wipe()
+    WebserviceResponse.wipe()
+
+    testdef2 = TestDef.import_from_xml(io.BytesIO(testdef_xml), formdef)
+    assert testdef2.name == 'test'
+    assert testdef2.object_type == 'formdefs'
+    assert testdef2.object_id == str(formdef.id)
+    assert testdef2.data == {'fields': {'1': ['foo', 'baz'], '2': True}, 'user': None}
+    assert testdef2.expected_error == 'xxx'
+    assert testdef2.is_in_backoffice is False
+
+    assert len(testdef2.workflow_tests.actions) == 1
+    assert testdef2.workflow_tests.actions[0].status_name == 'End status'
+
+    assert len(testdef2.get_webservice_responses()) == 1
+    assert testdef2.get_webservice_responses()[0].name == 'Fake response'
+
+    # check storage of temporary object used during import is forbidden
+    testdef_xml = TestDefXmlProxy()
+    with pytest.raises(AssertionError):
+        testdef_xml.store()
+
+
+def test_testdef_result_migrate_legacy_json(pub):
+    test_result = TestResult()
+    test_result.object_type = 'formdef'
+    test_result.object_id = '1'
+    test_result.timestamp = datetime.datetime(2021, 1, 1, 0, 0)
+    test_result.success = False
+    test_result.reason = 'xxx'
+    test_result.results = [
+        {
+            'id': '1',
+            'name': 'xxx',
+            'error': 'xxx',
+            'recorded_errors': ['a', 'b'],
+            'missing_required_fields': ['c', 'd'],
+        },
+        {
+            'id': '2',
+            'name': 'xxx',
+            'error': 'xxx',
+        },
+    ]
+    test_result.store()
+    test_result.store()
+
+    TestResult.migrate_legacy()
+    test_result = TestResult.get(test_result.id)
+    assert test_result.results == [
+        {
+            'id': '1',
+            'name': 'xxx',
+            'error': 'xxx',
+            'details': {
+                'form_status': None,
+                'recorded_errors': ['a', 'b'],
+                'missing_required_fields': ['c', 'd'],
+                'workflow_test_action_uuid': None,
+            },
+        },
+        {
+            'id': '2',
+            'name': 'xxx',
+            'error': 'xxx',
+            'details': {
+                'form_status': None,
+                'recorded_errors': [],
+                'missing_required_fields': [],
+                'workflow_test_action_uuid': None,
+            },
+        },
+    ]
 
 
 def test_testdef_create_from_formdata_boolean(pub):
@@ -122,7 +215,7 @@ def test_page_post_conditions(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['2'] = 'a'
     formdata.data['4'] = 'a'
 
@@ -156,7 +249,7 @@ def test_page_post_condition_invalid(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
     with pytest.raises(TestError) as excinfo:
@@ -181,7 +274,7 @@ def test_field_conditions(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = 'a'
     formdata.data['2'] = 'xxx'
 
@@ -219,7 +312,7 @@ def test_field_conditions_boolean(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = False
     formdata.data['2'] = None
 
@@ -259,7 +352,7 @@ def test_multi_page_condition(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = 'a'
     formdata.data['3'] = 'xxx'
     formdata.data['5'] = 'yyy'
@@ -294,7 +387,7 @@ def test_validation_string_field(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = '1'
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
@@ -320,7 +413,7 @@ def test_validation_required_field(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
     testdef.run(formdef)
@@ -339,7 +432,7 @@ def test_validation_item_field(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = 'foo'
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
@@ -371,7 +464,7 @@ def test_validation_item_field_inside_block(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = {'data': [{'1': 'foo'}]}
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
@@ -404,7 +497,7 @@ def test_validation_optional_field_inside_required_block(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = {'data': [{'1': 'foo'}]}
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
@@ -434,7 +527,7 @@ def test_item_field_display_value(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = 'foo'
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
@@ -465,7 +558,7 @@ def test_item_field_structured_value(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data = {
         '1': '2',
         '1_raw': '2',
@@ -526,7 +619,7 @@ def test_item_field_structured_value_inside_block(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = {
         'data': [
             {
@@ -593,7 +686,7 @@ def test_item_field_card_data_source_live(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = str(carddata.id)
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
@@ -616,7 +709,7 @@ def test_validation_items_field(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = ['foo', 'baz']
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
@@ -643,7 +736,7 @@ def test_validation_email_field(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = 'test@entrouvert.com'
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
@@ -669,7 +762,7 @@ def test_validation_boolean_field(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = False
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
@@ -694,7 +787,7 @@ def test_validation_date_field(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = time.strptime('2022-07-19', '%Y-%m-%d')
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
@@ -727,7 +820,7 @@ def test_validation_map_field(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = '1.0;2.0'
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
@@ -750,7 +843,7 @@ def test_validation_file_field(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
 
     upload = PicklableUpload('test.pdf', 'application/pdf', 'ascii')
     upload.receive([b'first line', b'second line'])
@@ -813,7 +906,7 @@ def test_validation_block_field(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = {'data': [{'1': 'b'}, {'1': 'a'}]}
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
@@ -864,7 +957,7 @@ def test_computed_field_support(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = 'zzz'
     formdata.data['3'] = 'hop'
 
@@ -904,13 +997,13 @@ def test_computed_field_support_complex_data(pub):
 
     submitted_formdata = formdef.data_class()()
     submitted_formdata.just_created()
-    submitted_formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    submitted_formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     submitted_formdata.data['2'] = ['a', 'bc']
     submitted_formdata.store()
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
     with pytest.raises(TestError) as excinfo:
@@ -954,11 +1047,65 @@ def test_computed_field_support_webservice(pub, http_requests):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.store()
     testdef.run(formdef)
 
+    assert len(testdef.sent_requests) == 1
+    assert testdef.sent_requests[0]['method'] == 'GET'
+    assert testdef.sent_requests[0]['url'] == 'http://remote.example.net/json'
+
+    response = WebserviceResponse()
+    response.testdef_id = testdef.id
+    response.name = 'Fake response'
+    response.url = 'http://remote.example.net/json'
+    response.payload = '{"foo": "bar"}'
+    response.store()
+
+    testdef.run(formdef)
+
+    assert len(testdef.sent_requests) == 1
+    assert testdef.sent_requests[0]['url'] == 'http://remote.example.net/json'
+    assert testdef.sent_requests[0]['webservice_response_id'] == response.id
+
+    response.payload = '{"foo": "baz"}'
+    response.store()
+
+    with pytest.raises(TestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Page 1 post condition was not met (form_var_computed_foo == "bar").'
+    assert len(testdef.sent_requests) == 1
+    assert testdef.sent_requests[0]['url'] == 'http://remote.example.net/json'
+    assert testdef.sent_requests[0]['webservice_response_id'] == response.id
+
+    response.url = 'http://example.com/json'
+    response.store()
+
+    testdef.run(formdef)
+    assert len(testdef.sent_requests) == 1
+    assert testdef.sent_requests[0]['url'] == 'http://remote.example.net/json'
+    assert testdef.sent_requests[0]['webservice_response_id'] is None
+
+    response.url = None
+    response.store()
+
+    testdef.run(formdef)
+    assert len(testdef.sent_requests) == 1
+    assert testdef.sent_requests[0]['url'] == 'http://remote.example.net/json'
+    assert testdef.sent_requests[0]['webservice_response_id'] is None
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    with mock.patch('wcs.testdef.MockWebserviceResponseAdapter._send', side_effect=KeyError('missing key')):
+        with pytest.raises(TestError):
+            testdef.run(formdef)
+
+    assert len(testdef.sent_requests) == 0
+    assert testdef.recorded_errors == [
+        "Unexpected error when mocking webservice call for url http://remote.example.net/json: 'missing key'."
+    ]
+
 
 def test_computed_field_value_too_long(pub):
     formdef = FormDef()
@@ -985,7 +1132,7 @@ def test_computed_field_value_too_long(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
     with pytest.raises(TestError):
@@ -1024,7 +1171,7 @@ def test_computed_field_forms_template_access(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.store()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
@@ -1062,7 +1209,7 @@ def test_expected_error(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = '123456'
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
@@ -1107,7 +1254,7 @@ def test_expected_error_conditional_field(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
     formdata.data['1'] = 'a'
     formdata.data['2'] = 'b'
 
@@ -1153,7 +1300,7 @@ def test_is_in_backoffice(pub):
 
     formdata = formdef.data_class()()
     formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
     testdef.run(formdef)
@@ -1166,3 +1313,89 @@ def test_is_in_backoffice(pub):
 
     testdef.is_in_backoffice = False
     testdef.run(formdef)
+
+
+def test_webservice_response_match_request(pub, http_requests):
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.fields = [
+        fields.ComputedField(
+            id='1',
+            label='Computed',
+            varname='computed',
+            value_template='{{ webservice.hello_world }}',
+            freeze_on_initial_value=True,
+        ),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.store()
+
+    wscall = NamedWsCall()
+    wscall.name = 'Hello world'
+    wscall.request = {
+        'url': 'http://remote.example.net/json',
+        'method': 'POST',
+        'qs_data': {'foo': 'bar'},
+        'post_data': {'foo2': 'bar2'},
+    }
+    wscall.store()
+
+    response = WebserviceResponse()
+    response.testdef_id = testdef.id
+    response.name = 'Fake response'
+    response.url = 'http://remote.example.net/json'
+    response.payload = '{}'
+    response.store()
+
+    testdef.run(formdef)
+    assert testdef.sent_requests[0]['webservice_response_id'] == response.id
+
+    # method restriction
+    response.method = 'GET'
+    response.store()
+
+    testdef.run(formdef)
+    assert testdef.sent_requests[0]['webservice_response_id'] is None
+
+    response.method = 'POST'
+    response.store()
+
+    testdef.run(formdef)
+    assert testdef.sent_requests[0]['webservice_response_id'] == response.id
+
+    # query string restriction
+    response.qs_data = {
+        'foo': 'bar',
+        'xxx': 'yyy',
+    }
+    response.store()
+
+    testdef.run(formdef)
+    assert testdef.sent_requests[0]['webservice_response_id'] is None
+
+    del response.qs_data['xxx']
+    response.store()
+
+    testdef.run(formdef)
+    assert testdef.sent_requests[0]['webservice_response_id'] == response.id
+
+    # post data restriction
+    response.post_data = {
+        'foo2': 'bar2',
+        'xxx': 'yyy',
+    }
+    response.store()
+
+    testdef.run(formdef)
+    assert testdef.sent_requests[0]['webservice_response_id'] is None
+
+    del response.post_data['xxx']
+    response.store()
+
+    testdef.run(formdef)
+    assert testdef.sent_requests[0]['webservice_response_id'] == response.id

tests/test_users.py

@@ -1,10 +1,10 @@
 import datetime
 import json
-import time
 
 import pytest
 import responses
 from django.core.management import call_command
+from django.utils.timezone import localtime
 
 from wcs import fields
 from wcs.carddef import CardDef
@@ -305,10 +305,10 @@ def test_clean_deleted_users(pub):
     formdata1 = data_class()
     formdata1.user_id = user1.id
     evo = Evolution(formdata=formdata1)
-    evo.time = time.localtime()
+    evo.time = localtime()
     evo.who = user4.id
     evo2 = Evolution(formdata=formdata1)
-    evo2.time = time.localtime()
+    evo2.time = localtime()
     evo2.who = '_submitter'
     formdata1.evolution = [evo, evo2]
     formdata1.workflow_roles = {'_received': '_user:%s' % user5.id}

tests/test_workflow_tests.py

@@ -0,0 +1,395 @@
+from unittest import mock
+
+import pytest
+
+from wcs import workflow_tests
+from wcs.formdef import FormDef, fields
+from wcs.qommon.http_request import HTTPRequest
+from wcs.testdef import TestDef
+from wcs.wf.jump import JumpWorkflowStatusItem
+from wcs.workflow_tests import WorkflowTestError
+from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef, WorkflowStatusItem
+
+from .utilities import create_temporary_pub
+
+
+@pytest.fixture
+def pub():
+    pub = create_temporary_pub()
+
+    req = HTTPRequest(None, {'SCRIPT_NAME': '/', 'SERVER_NAME': 'example.net'})
+    pub.set_app_dir(req)
+    pub.write_cfg()
+
+    pub.user_class.wipe()
+    pub.role_class.wipe()
+    FormDef.wipe()
+    Workflow.wipe()
+    return pub
+
+
+def test_workflow_tests_ignore_unsupported_items(pub, monkeypatch):
+    user = pub.user_class(name='test user')
+    user.store()
+
+    workflow = Workflow(name='Workflow One')
+    new_status = workflow.add_status(name='New status')
+    end_status = workflow.add_status(name='End status')
+
+    jump = new_status.add_action('jump')
+    jump.status = end_status.id
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.agent_id = user.id
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertStatus(status_name='End status'),
+    ]
+    testdef.run(formdef)
+
+    monkeypatch.setattr(
+        JumpWorkflowStatusItem, 'get_workflow_test_action', WorkflowStatusItem.get_workflow_test_action
+    )
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Form should be in status "End status" but is in status "New status".'
+
+
+def test_workflow_tests_button_click(pub):
+    role = pub.role_class(name='test role')
+    role.store()
+    user = pub.user_class(name='test user')
+    user.roles = [role.id]
+    user.store()
+
+    workflow = Workflow(name='Workflow One')
+    new_status = workflow.add_status(name='New status')
+    end_status = workflow.add_status(name='End status')
+
+    jump = new_status.add_action('choice')
+    jump.label = 'Go to end status'
+    jump.status = end_status.id
+    jump.by = [role.id]
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.agent_id = user.id
+    testdef.workflow_tests.actions = [
+        workflow_tests.ButtonClick(button_name='Go to end status'),
+        workflow_tests.AssertStatus(status_name='End status'),
+    ]
+    testdef.run(formdef)
+
+    # change jump target status
+    other_status = workflow.add_status(name='Other status')
+    jump.status = other_status.id
+    workflow.store()
+    formdef.refresh_from_storage()
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Form should be in status "End status" but is in status "Other status".'
+
+    # hide button from test user
+    other_role = pub.role_class(name='test role 2')
+    other_role.store()
+    jump.by = [other_role.id]
+    workflow.store()
+    formdef.refresh_from_storage()
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Button "Go to end status" is not displayed.'
+
+    # change button label
+    jump.by = [role.id]
+    jump.label = 'Go to xxx'
+    workflow.store()
+    formdef.refresh_from_storage()
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Button "Go to end status" is not displayed.'
+
+
+def test_workflow_tests_automatic_jump(pub):
+    user = pub.user_class(name='test user')
+    user.store()
+
+    workflow = Workflow(name='Workflow One')
+    new_status = workflow.add_status(name='New status')
+    end_status = workflow.add_status(name='End status')
+
+    jump = new_status.add_action('jump')
+    jump.status = end_status.id
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.agent_id = user.id
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertStatus(status_name='End status'),
+    ]
+    testdef.run(formdef)
+
+    new_end_status = workflow.add_status(name='New end status')
+
+    jump = end_status.add_action('jump')
+    jump.status = new_end_status.id
+
+    workflow.store()
+    formdef.refresh_from_storage()
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Form should be in status "End status" but is in status "New end status".'
+
+
+def test_workflow_tests_automatic_jump_condition(pub):
+    user = pub.user_class(name='test user')
+    user.store()
+
+    workflow = Workflow(name='Workflow One')
+    new_status = workflow.add_status(name='New status')
+    frog_status = workflow.add_status(name='Frog status')
+    bear_status = workflow.add_status(name='Bear status')
+
+    jump = new_status.add_action('jump')
+    jump.status = frog_status.id
+    jump.condition = {'type': 'django', 'value': 'form_var_animal == "frog"'}
+
+    jump = new_status.add_action('jump')
+    jump.status = bear_status.id
+    jump.condition = {'type': 'django', 'value': 'form_var_animal == "bear"'}
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.workflow_id = workflow.id
+    formdef.fields = [
+        fields.StringField(id='1', label='Text', varname='animal'),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.data['1'] = 'frog'
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.agent_id = user.id
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertStatus(status_name='Frog status'),
+    ]
+    testdef.run(formdef)
+
+    testdef.data['fields']['1'] = 'bear'
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Form should be in status "Frog status" but is in status "Bear status".'
+
+
+@pytest.mark.freeze_time('2024-02-19 12:00')
+def test_workflow_tests_automatic_jump_timeout(pub):
+    user = pub.user_class(name='test user')
+    user.store()
+
+    workflow = Workflow(name='Workflow One')
+    new_status = workflow.add_status(name='New status')
+    stalled_status = workflow.add_status(name='Stalled')
+
+    jump = new_status.add_action('jump')
+    jump.status = stalled_status.id
+    jump.timeout = 120 * 60  # 2 hours
+    jump.condition = {'type': 'django', 'value': 'form_receipt_datetime|age_in_days >= 1'}
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.agent_id = user.id
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertStatus(status_name='New status'),
+        workflow_tests.SkipTime(seconds=119 * 60),
+        workflow_tests.AssertStatus(status_name='New status'),
+        workflow_tests.SkipTime(seconds=60),
+        workflow_tests.AssertStatus(status_name='New status'),
+        workflow_tests.SkipTime(seconds=24 * 60 * 60),
+        workflow_tests.AssertStatus(status_name='Stalled'),
+    ]
+    testdef.run(formdef)
+
+    jump.condition = {'type': 'django', 'value': 'form_receipt_datetime|age_in_hours >= 1'}
+    workflow.store()
+    formdef.refresh_from_storage()
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Form should be in status "New status" but is in status "Stalled".'
+
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertStatus(status_name='New status'),
+        workflow_tests.SkipTime(seconds=119 * 60),
+        workflow_tests.AssertStatus(status_name='New status'),
+        workflow_tests.SkipTime(seconds=60),
+        workflow_tests.AssertStatus(status_name='Stalled'),
+    ]
+    testdef.run(formdef)
+
+
+@mock.patch('wcs.qommon.emails.send_email')
+def test_workflow_tests_sendmail(mocked_send_email, pub):
+    role = pub.role_class(name='test role')
+    role.store()
+    user = pub.user_class(name='test user')
+    user.roles = [role.id]
+    user.store()
+
+    workflow = Workflow(name='Workflow One')
+    new_status = workflow.add_status(name='New status')
+    end_status = workflow.add_status(name='End status')
+
+    sendmail = new_status.add_action('sendmail')
+    sendmail.to = ['test@example.org']
+    sendmail.subject = 'In new status'
+    sendmail.body = 'xxx'
+
+    jump = new_status.add_action('choice')
+    jump.label = 'Go to end status'
+    jump.status = end_status.id
+    jump.by = [role.id]
+
+    sendmail = end_status.add_action('sendmail')
+    sendmail.to = ['test@example.org']
+    sendmail.subject = 'In end status'
+    sendmail.body = 'yyy'
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.agent_id = user.id
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertEmail(subject_strings=['In new status'], body_strings=['xxx']),
+        workflow_tests.ButtonClick(button_name='Go to end status'),
+        workflow_tests.AssertStatus(status_name='End status'),
+        workflow_tests.AssertEmail(subject_strings=['end status'], body_strings=['yyy']),
+    ]
+
+    testdef.run(formdef)
+    mocked_send_email.assert_not_called()
+
+    testdef.workflow_tests.actions.append(workflow_tests.AssertEmail())
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'No email was sent.'
+
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertEmail(subject_strings=['bla'], body_strings=['xxx']),
+    ]
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Email subject does not contain "bla".'
+
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertEmail(body_strings=['xxx', 'bli']),
+    ]
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Email body does not contain "bli".'
+
+
+def test_workflow_tests_backoffice_fields(pub):
+    user = pub.user_class(name='test user')
+    user.store()
+
+    workflow = Workflow(name='Workflow One')
+    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
+    workflow.backoffice_fields_formdef.fields = [
+        fields.StringField(id='bo1', label='Text'),
+        fields.StringField(id='bo2', label='Text 2'),
+    ]
+
+    new_status = workflow.add_status(name='New status')
+    set_backoffice_fields = new_status.add_action('set-backoffice-fields')
+    set_backoffice_fields.fields = [{'field_id': 'bo2', 'value': '{{ form_var_text }}'}]
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.workflow_id = workflow.id
+    formdef.fields = [
+        fields.StringField(id='1', label='Text', varname='text'),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.data['1'] = 'abc'
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.agent_id = user.id
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertBackofficeFieldValues(id='1', fields=[{'field_id': 'bo2', 'value': 'abc'}]),
+    ]
+
+    testdef.run(formdef)
+
+    testdef.data['fields']['1'] = 'def'
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Wrong value for backoffice field "Text 2" (expected "abc", got "def").'
+
+    workflow.backoffice_fields_formdef.fields = []
+    workflow.store()
+    formdef.refresh_from_storage()
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Field bo2 not found (expected value "abc").'

tests/workflow/test_all.py

@@ -6,6 +6,7 @@ from unittest import mock
 
 import pytest
 import responses
+from django.utils.timezone import localtime
 from quixote import cleanup, get_publisher, get_response
 
 from wcs import sessions, sql
@@ -616,7 +617,7 @@ def test_anonymise(pub):
     formdata.submission_context = {'foo': 'bar'}
     formdata.store()
     evo = Evolution(formdata)  # add a new evolution
-    evo.time = time.localtime()
+    evo.time = localtime()
     evo.status = formdata.status
     evo.who = 42
     evo.parts = [AttachmentEvolutionPart('hello.txt', fp=io.BytesIO(b'hello world'), varname='testfile')]
@@ -1331,7 +1332,7 @@ def test_global_timeouts(pub, formdef_class):
     pub.apply_global_action_timeouts()
     assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'
 
-    formdata1.receipt_time = time.localtime(time.time() - 3 * 86400)
+    formdata1.receipt_time = localtime() - datetime.timedelta(days=3)
     formdata1.store()
     pub.apply_global_action_timeouts()
     assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'
@@ -1358,7 +1359,7 @@ def test_global_timeouts(pub, formdef_class):
     pub.apply_global_action_timeouts()
     assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'
 
-    formdata1.evolution[-1].time = time.localtime(time.time() - 3 * 86400)
+    formdata1.evolution[-1].time = localtime() - datetime.timedelta(days=3)
     formdata1.store()
     pub.apply_global_action_timeouts()
     assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'
@@ -1368,7 +1369,7 @@ def test_global_timeouts(pub, formdef_class):
     # bad (obsolete) status: do nothing
     trigger.anchor_status_first = 'wf-foobar'
     workflow.store()
-    formdata1.evolution[-1].time = time.localtime(time.time() - 3 * 86400)
+    formdata1.evolution[-1].time = localtime() - datetime.timedelta(days=3)
     formdata1.store()
     pub.apply_global_action_timeouts()
     assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'
@@ -1378,18 +1379,18 @@ def test_global_timeouts(pub, formdef_class):
     trigger.anchor_status_latest = None
     workflow.store()
 
-    formdata1.evolution[-1].time = time.localtime()
+    formdata1.evolution[-1].time = localtime()
     formdata1.store()
     formdata1.jump_status('new')
-    formdata1.evolution[-1].time = time.localtime(time.time() - 7 * 86400)
+    formdata1.evolution[-1].time = localtime() - datetime.timedelta(days=7)
     formdata1.jump_status('accepted')
     formdata1.jump_status('new')
-    formdata1.evolution[-1].time = time.localtime(time.time() - 1 * 86400)
+    formdata1.evolution[-1].time = localtime() - datetime.timedelta(days=1)
 
     pub.apply_global_action_timeouts()
     assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'
 
-    formdata1.evolution[-1].time = time.localtime(time.time() - 4 * 86400)
+    formdata1.evolution[-1].time = localtime() - datetime.timedelta(days=4)
     formdata1.store()
     pub.apply_global_action_timeouts()
     assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'
@@ -1418,7 +1419,7 @@ def test_global_timeouts(pub, formdef_class):
 
     # check trigger is not run on finalized formdata
     formdata1.jump_status('finished')
-    formdata1.evolution[-1].time = time.localtime(time.time() - 4 * 86400)
+    formdata1.evolution[-1].time = localtime() - datetime.timedelta(days=4)
     formdata1.store()
     trigger.anchor = 'creation'
     workflow.store()
@@ -1430,7 +1431,7 @@ def test_global_timeouts(pub, formdef_class):
     # endpoint
     formdata1.jump_status('finished')
     formdata1.evolution[-1].last_jump_datetime = None
-    formdata1.evolution[-1].time = time.localtime(time.time() - 4 * 86400)
+    formdata1.evolution[-1].time = localtime() - datetime.timedelta(days=4)
     formdata1.store()
     trigger.anchor = 'latest-arrival'
     trigger.anchor_status_latest = 'wf-finished'
@@ -1451,7 +1452,7 @@ def test_global_timeouts(pub, formdef_class):
     # use python expression as anchor
     #  timestamp
     formdata1.jump_status('new')
-    formdata1.evolution[-1].time = time.localtime(time.time() - 4 * 86400)
+    formdata1.evolution[-1].time = localtime() - datetime.timedelta(days=4)
     formdata1.evolution[-1].last_jump_datetime = None
     formdata1.store()
 
@@ -1572,7 +1573,7 @@ def test_global_timeouts(pub, formdef_class):
     # * invalid value
     pub.loggederror_class.wipe()
     formdata1.jump_status('accepted')
-    formdata1.evolution[-1].time = time.localtime(time.time() - 1 * 86400)
+    formdata1.evolution[-1].time = localtime() - datetime.timedelta(days=1)
     formdata1.store()
     pub.apply_global_action_timeouts()
     assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'
@@ -1597,7 +1598,7 @@ def test_global_timeouts(pub, formdef_class):
     assert pub.loggederror_class.count() == 0
 
     # * ok value, and timeout is triggered
-    formdata1.evolution[-1].time = time.localtime(time.time() - 4 * 86400)
+    formdata1.evolution[-1].time = localtime() - datetime.timedelta(days=4)
     formdata1.store()
     pub.apply_global_action_timeouts()
     assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'
@@ -1651,7 +1652,7 @@ def test_global_timeouts_finalized(pub, sql_queries, timeout):
     formdata1.just_created()
     formdata1.store()
     formdata1.jump_status('finished')
-    formdata1.evolution[-1].time = time.localtime(time.time() - 4 * 86400)
+    formdata1.evolution[-1].time = localtime() - datetime.timedelta(days=4)
     formdata1.store()
 
     formdata2 = formdef.data_class()()
@@ -1659,7 +1660,7 @@ def test_global_timeouts_finalized(pub, sql_queries, timeout):
     formdata2.just_created()
     formdata2.store()
     formdata2.jump_status('finished')
-    formdata2.evolution[-1].time = time.localtime(time.time() - 1 * 86400)
+    formdata2.evolution[-1].time = localtime() - datetime.timedelta(days=1)
     formdata2.store()
 
     formdef2 = FormDef()
@@ -1675,7 +1676,7 @@ def test_global_timeouts_finalized(pub, sql_queries, timeout):
     formdata3.just_created()
     formdata3.store()
     formdata3.jump_status('finished')
-    formdata3.evolution[-1].time = time.localtime(time.time() - 6 * 86400)
+    formdata3.evolution[-1].time = localtime() - datetime.timedelta(days=6)
     formdata3.store()
 
     formdata4 = formdef2.data_class()()
@@ -1683,7 +1684,7 @@ def test_global_timeouts_finalized(pub, sql_queries, timeout):
     formdata4.just_created()
     formdata4.store()
     formdata4.jump_status('finished')
-    formdata4.evolution[-1].time = time.localtime(time.time() - 4 * 86400)
+    formdata4.evolution[-1].time = localtime() - datetime.timedelta(days=4)
     formdata4.store()
 
     pub.apply_global_action_timeouts()
@@ -1735,18 +1736,18 @@ def test_global_timeouts_latest_arrival(pub):
 
     formdata1.jump_status('new')
     # enter in status 8 days ago
-    formdata1.evolution[-1].time = time.localtime(time.time() - 8 * 86400)
+    formdata1.evolution[-1].time = localtime() - datetime.timedelta(days=8)
     formdata1.store()
     # but get a new comment 1 day ago
     formdata1.evolution.append(Evolution(formdata1))
-    formdata1.evolution[-1].time = time.localtime(time.time() - 1 * 86400)
+    formdata1.evolution[-1].time = localtime() - datetime.timedelta(days=1)
     formdata1.evolution[-1].comment = 'plop'
     formdata1.store()
     pub.apply_global_action_timeouts()
     # no change
     assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'
 
-    formdata1.evolution[-1].time = time.localtime(time.time() - 5 * 86400)
+    formdata1.evolution[-1].time = localtime() - datetime.timedelta(days=5)
     formdata1.store()
     pub.apply_global_action_timeouts()
     # change
@@ -1757,7 +1758,7 @@ def test_global_timeouts_latest_arrival(pub):
     formdata1.just_created()
     formdata1.store()
     formdata1.jump_status('new')
-    formdata1.evolution[-1].time = time.localtime(time.time() - 5 * 86400)
+    formdata1.evolution[-1].time = localtime() - datetime.timedelta(days=5)
     formdata1.store()
     formdata1.jump_status('accepted')
     formdata1.store()
@@ -1769,7 +1770,7 @@ def test_global_timeouts_latest_arrival(pub):
     formdata1.just_created()
     formdata1.store()
     formdata1.jump_status('new')
-    formdata1.evolution[-1].time = time.localtime(time.time() - 5 * 86400)
+    formdata1.evolution[-1].time = localtime() - datetime.timedelta(days=5)
     formdata1.store()
     formdata1.jump_status('accepted')
     formdata1.jump_status('finished')

tests/workflow/test_email.py

@@ -1,6 +1,7 @@
 import base64
 import json
 import os
+from unittest import mock
 
 import pytest
 from django.utils.encoding import force_bytes, force_str
@@ -715,3 +716,33 @@ def test_email_computed_recipients(pub, emails):
     assert emails.count() == 1
     assert set(formdata.evolution[-1].parts[-1].addresses) == {'user1@example.com'}
     formdata.evolution[-1].parts = []
+
+
+@pytest.mark.parametrize('req', [True, False])
+def test_email_invalid_recipients(pub, req):
+    if req is False:
+        pub._set_request(None)
+
+    FormDef.wipe()
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.store()
+    formdef.data_class().wipe()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    item = SendmailWorkflowStatusItem()
+    item.varname = 'test'
+    item.to = ['invalid,']
+    item.subject = 'xxx'
+    item.body = 'XXX'
+
+    with mock.patch('wcs.qommon.emails.EmailToSend.__call__') as send_email_job:
+        item.perform(formdata)
+        if req:
+            get_response().process_after_jobs()
+        assert send_email_job.call_count == 0

tests/workflow/test_jump.py

@@ -1,5 +1,4 @@
 import datetime
-import time
 from unittest import mock
 
 import pytest
@@ -40,11 +39,8 @@ def pub(request):
 
 def rewind(formdata, seconds):
     # utility function to move formdata back in time
-    def rewind_time(timetuple):
-        return time.localtime(datetime.datetime.fromtimestamp(time.mktime(timetuple) - seconds).timestamp())
-
-    formdata.receipt_time = rewind_time(formdata.receipt_time)
-    formdata.evolution[-1].time = rewind_time(formdata.evolution[-1].time)
+    formdata.receipt_time = formdata.receipt_time - datetime.timedelta(seconds=seconds)
+    formdata.evolution[-1].time = formdata.evolution[-1].time - datetime.timedelta(seconds=seconds)
 
 
 def test_jump_nothing(pub):
@@ -582,3 +578,30 @@ def test_timeout_tracing(pub, admin_user):
         'Status2',
         'History Message',
     ]
+
+
+def test_jump_self_timeout(pub):
+    FormDef.wipe()
+    Workflow.wipe()
+
+    workflow = Workflow(name='timeout')
+    st1 = workflow.add_status('Status1', 'st1')
+
+    jump = st1.add_action('jump')
+    jump.timeout = 30 * 60  # 30 minutes
+    jump.status = 'st1'
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    rewind(formdata, seconds=40 * 60)
+    formdata.store()
+    formdata.record_workflow_event('backoffice-created')
+    _apply_timeouts(pub)

tests/workflow/test_status_loop.py

@@ -405,3 +405,36 @@ def test_status_loop_on_items(pub):
         '<div>foo 1 plop bar</div>',
         '<div>foo 2 plop3 bar</div>',
     ]
+
+
+def test_status_loop_unknown_status_with_global_action(pub):
+    workflow = Workflow(name='foo')
+    st1 = workflow.add_status(name='baz')
+    st2 = workflow.add_status(name='bar')
+    st1.loop_items_template = '{{ "abc"|make_list }}'
+    st1.after_loop_status = str(st2.id)
+    ac1 = workflow.add_global_action('Action', 'ac1')
+    ac1.backoffice_info_text = '<p>Foo</p>'
+    add_to_journal = ac1.add_action('register-comment', id='_add_to_journal')
+    add_to_journal.comment = 'HELLO WORLD'
+    trigger = ac1.triggers[0]
+    assert trigger.key == 'manual'
+    trigger.roles = ['_submitter']
+    trigger.statuses = ['unknown']
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'bar'
+    formdef.fields = []
+    formdef.workflow = workflow
+    formdef.store()
+
+    user = pub.user_class(name='admin')
+    user.email = 'admin@localhost'
+    user.store()
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.status = 'unknown'
+    formdata.user_id = str(user.id)
+    formdata.store()
+    formdata.perform_global_action(ac1.id, user)  # no error

tests/workflow/test_wscall.py

@@ -922,7 +922,7 @@ def test_webservice_target_status(pub):
     assert targets.count(status2) == 2
 
 
-def test_webservice_with_complex_data(http_requests, pub):
+def test_webservice_with_complex_data_in_payload(http_requests, pub):
     pub.substitutions.feed(MockSubstitutionVariables())
 
     wf = Workflow(name='wf1')
@@ -1044,3 +1044,97 @@ def test_webservice_with_complex_data(http_requests, pub):
     assert http_requests.get_last('method') == 'POST'
     payload = json.loads(http_requests.get_last('body'))
     assert payload['bool'] is False
+
+
+def test_webservice_with_complex_data_in_query_string(http_requests, pub):
+    pub.substitutions.feed(MockSubstitutionVariables())
+
+    wf = Workflow(name='wf1')
+    wf.add_status('Status1', 'st1')
+    wf.add_status('StatusErr', 'sterr')
+    wf.store()
+
+    datasource = {
+        'type': 'jsonvalue',
+        'value': json.dumps(
+            [
+                {'id': 'a', 'text': 'aa', 'more': 'aaa'},
+                {'id': 'b', 'text': 'bb', 'more': 'bbb'},
+                {'id': 'c', 'text': 'cc', 'more': 'ccc'},
+            ]
+        ),
+    }
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = [
+        ItemField(id='1', label='1st field', varname='item', data_source=datasource),
+        ItemsField(id='2', label='2nd field', varname='items', data_source=datasource),
+        StringField(id='3', label='3rd field', varname='str'),
+        StringField(id='4', label='4th field', varname='empty_str'),
+        StringField(id='5', label='5th field', varname='none'),
+        BoolField(id='6', label='6th field', varname='bool'),
+    ]
+    formdef.workflow_id = wf.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {}
+    formdata.data['1'] = 'a'
+    formdata.data['1_display'] = 'aa'
+    formdata.data['1_structured'] = formdef.fields[0].store_structured_value(formdata.data, '1')
+    formdata.data['2'] = ['a', 'b']
+    formdata.data['2_display'] = 'aa, bb'
+    formdata.data['2_structured'] = formdef.fields[1].store_structured_value(formdata.data, '2')
+    formdata.data['3'] = 'tutuche'
+    formdata.data['4'] = 'empty_str'
+    formdata.data['5'] = None
+    formdata.data['6'] = False
+    formdata.just_created()
+    formdata.store()
+
+    item = WebserviceCallStatusItem()
+    item.method = 'POST'
+    item.url = 'http://remote.example.net'
+    item.qs_data = {
+        'item': '{{ form_var_item }}',
+        'items': '{{ form_var_items }}',
+        'item_raw': '{{ form_var_item_raw }}',
+        'items_raw': '{{ form_var_items_raw }}',
+        'with_items_raw': '{% with x=form_var_items_raw %}{{ x }}{% endwith %}',
+        'with_items_upper': '{% with x=form_var_items_raw %}{{ x.1|upper }}{% endwith %}',
+        'joined_items_raw': '{{ form_var_items_raw|join:"|" }}',
+        'forloop_items_raw': '{% for item in form_var_items_raw %}{{item}}|{% endfor %}',
+        'str': '{{ form_var_str }}',
+        'str_mod': '{{ form_var_str }}--plop',
+        'int': '{{ 1000 }}',
+        'decimal': '{{ "1000"|decimal }}',
+        'decimal2': '{{ "1000.1"|decimal }}',
+        'empty_string': '{{ form_var_empty }}',
+        'none': '{{ form_var_none }}',
+        'bool': '{{ form_var_bool_raw }}',
+        'time': '{{ "13:12"|time }}',
+    }
+    pub.substitutions.feed(formdata)
+    with get_publisher().complex_data():
+        item.perform(formdata)
+    assert sorted(urllib.parse.parse_qsl(urllib.parse.urlparse(http_requests.get_last('url')).query)) == [
+        ('bool', 'False'),
+        ('decimal', '1E+3'),
+        ('decimal2', '1000.1'),
+        ('forloop_items_raw', 'a|b|'),
+        ('int', '1000'),
+        ('item', 'aa'),
+        ('item_raw', 'a'),
+        ('items', 'aa, bb'),
+        ('items_raw', 'a'),
+        ('items_raw', 'b'),
+        ('joined_items_raw', 'a|b'),
+        ('str', 'tutuche'),
+        ('str_mod', 'tutuche--plop'),
+        ('time', '13:12:00'),
+        ('with_items_raw', 'a'),
+        ('with_items_raw', 'b'),
+        ('with_items_upper', 'B'),
+    ]

wcs/admin/categories.py

@@ -416,7 +416,7 @@ class CategoriesDirectory(Directory):
         new_order = [o for o in new_order if o in categories_by_id]
         for i, o in enumerate(new_order):
             categories_by_id[o].position = i + 1
-            categories_by_id[o].store()
+            categories_by_id[o].store(store_snapshot=False)
         return 'ok'
 
     def new(self):

wcs/admin/forms.py

@@ -1310,12 +1310,21 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
         return self.duplicate_submit(form)
 
     def duplicate_submit(self, form):
+        from wcs.testdef import TestDef
+
+        testdefs = TestDef.select_for_objectdef(self.formdefui.formdef)
+
         self.formdefui.formdef.name = form.get_widget('name').parse()
         self.formdefui.formdef.id = None
         self.formdefui.formdef.url_name = None
         self.formdefui.formdef.table_name = None
         self.formdefui.formdef.disabled = True
         self.formdefui.formdef.store()
+
+        for testdef in testdefs:
+            testdef = TestDef.import_from_xml_tree(testdef.export_to_xml(), self.formdefui.formdef)
+            testdef.store()
+
         return redirect('../%s/' % self.formdefui.formdef.id)
 
     def get_check_deletion_message(self):
@@ -1610,6 +1619,7 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
         return r.getvalue()
 
     def export(self):
+        self.formdef._export_tests = True
         return misc.xml_response(
             self.formdef,
             filename='%s-%s.wcs' % (self.formdef_export_prefix, self.formdef.url_name),
@@ -1764,11 +1774,11 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
 
         temp_drafts = defaultdict(int)
         total_drafts = 0
+        drafts = {}
         for formdata in self.formdef.data_class().select(clause=[Equal('status', 'draft')]):
             page_id = formdata.page_id if formdata.page_id is not None else '_unkown'
             temp_drafts[page_id] += 1
             total_drafts += 1
-        drafts = {}
         if total_drafts:
             for key in ('_unkown', '_confirmation_page', '_first_page'):
                 try:
@@ -1787,8 +1797,8 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
                 draft_percent = 100 * draft_data['total'] / total_drafts
                 draft_data['percent'] = draft_percent
                 draft_data['percent_rounded'] = '%d' % draft_percent
-            context['drafts'] = sorted(drafts.items(), reverse=True, key=lambda x: x[1]['total'])
-            context['drafts_total'] = total_drafts
+        context['drafts'] = sorted(drafts.items(), reverse=True, key=lambda x: x[1]['total'])
+        context['drafts_total'] = total_drafts
 
         return template.QommonTemplateResponse(
             templates=[self.inspect_template_name],
@@ -2052,6 +2062,7 @@ class FormsDirectory(AccessControlled, Directory):
         self.imported_formdef = formdef
         formdef.disabled = True
         formdef.store()
+        formdef.finish_tests_xml_import()
         return redirect('%s/' % formdef.id)
 
 

wcs/admin/tests.py

@@ -14,6 +14,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+import collections
+import copy
 import json
 
 from django.template.loader import render_to_string
@@ -22,15 +24,25 @@ from quixote import get_publisher, get_request, get_response, get_session, redir
 from quixote.directory import Directory
 from quixote.html import TemplateIO, htmltext
 
+from wcs.admin.workflow_tests import WorkflowTestsDirectory
 from wcs.backoffice.management import FormBackofficeEditPage, FormBackOfficeStatusPage
 from wcs.backoffice.pagination import pagination_links
 from wcs.forms.common import FormStatusPage
 from wcs.qommon import _, misc, template
 from wcs.qommon.afterjobs import AfterJob
 from wcs.qommon.errors import TraversalError
-from wcs.qommon.form import FileWidget, Form, RadiobuttonsWidget, SingleSelectWidget, StringWidget
+from wcs.qommon.form import (
+    FileWidget,
+    Form,
+    RadiobuttonsWidget,
+    SingleSelectWidget,
+    StringWidget,
+    TextWidget,
+    WidgetDict,
+)
 from wcs.sql_criterias import Equal, Null, StrictNotEqual
-from wcs.testdef import TestDef, TestError, TestResult
+from wcs.testdef import TestDef, TestError, TestResult, WebserviceResponse
+from wcs.workflow_tests import WorkflowTestError
 
 
 class TestEditPage(FormBackofficeEditPage):
@@ -125,7 +137,15 @@ class TestEditPage(FormBackofficeEditPage):
 
 
 class TestPage(FormBackOfficeStatusPage):
-    _q_extra_exports = ['delete', 'export', 'edit', ('edit-data', 'edit_data'), 'duplicate']
+    _q_extra_exports = [
+        'delete',
+        'export',
+        'edit',
+        ('edit-data', 'edit_data'),
+        'duplicate',
+        ('workflow', 'workflow_tests'),
+        ('webservice-responses', 'webservice_responses'),
+    ]
 
     def __init__(self, component, objectdef):
         try:
@@ -136,6 +156,9 @@ class TestPage(FormBackOfficeStatusPage):
         filled = self.testdef.build_formdata(objectdef, include_fields=True)
         super().__init__(objectdef, filled)
 
+        self.workflow_tests = WorkflowTestsDirectory(self.testdef, self.formdef)
+        self.webservice_responses = WebserviceResponseDirectory(self.testdef)
+
     @property
     def edit_data(self):
         return TestEditPage(self.formdef, update_breadcrumbs=False, testdef=self.testdef, filled=self.filled)
@@ -160,6 +183,8 @@ class TestPage(FormBackOfficeStatusPage):
         r += htmltext('<h2>%s</h2>') % self.testdef
         r += htmltext('<span class="actions">')
         r += htmltext('<a href="edit-data/">%s</a>') % _('Edit data')
+        if get_publisher().has_site_option('enable-workflow-tests'):
+            r += htmltext('<a href="workflow/">%s</a>') % _('Workflow tests')
         r += htmltext('</span>')
         r += htmltext('</div>')
         if self.testdef.expected_error:
@@ -190,11 +215,9 @@ class TestPage(FormBackOfficeStatusPage):
             return redirect('..')
 
     def export(self):
-        get_response().set_content_type('application/json')
-        get_response().set_header(
-            'content-disposition', 'attachment; filename=wcs_test_%s.json' % self.testdef.name
+        return misc.xml_response(
+            self.testdef, filename='test-%s.wcs' % misc.simplify(self.testdef.name), include_id=False
         )
-        return json.dumps(self.testdef.export_to_json())
 
     def edit(self):
         form = Form(enctype='multipart/form-data')
@@ -262,10 +285,10 @@ class TestPage(FormBackOfficeStatusPage):
             r += form.render()
             return r.getvalue()
 
-        self.testdef.id = None
-        self.testdef.slug = None
         self.testdef.name = form.get_widget('name').parse()
+        self.testdef = TestDef.import_from_xml_tree(self.testdef.export_to_xml(), self.formdef)
         self.testdef.store()
+
         return redirect(self.testdef.get_admin_url())
 
 
@@ -355,6 +378,7 @@ class TestsDirectory(Directory):
         if not creation_mode_widget or creation_mode_widget.parse() == 'empty':
             testdef = TestDef.create_from_formdata(self.objectdef, self.objectdef.data_class()())
             testdef.name = form.get_widget('name').parse()
+            testdef.agent_id = get_session().user
             testdef.store()
             return redirect(testdef.get_admin_url() + 'edit-data/')
         else:
@@ -363,6 +387,7 @@ class TestsDirectory(Directory):
 
             testdef = TestDef.create_from_formdata(self.objectdef, formdata)
             testdef.name = form.get_widget('name').parse()
+            testdef.agent_id = get_session().user
             testdef.store()
             return redirect(testdef.get_admin_url())
 
@@ -396,10 +421,10 @@ class TestsDirectory(Directory):
         fp = form.get_widget('file').parse().fp
 
         try:
-            testdef = TestDef.import_from_json(json.loads(fp.read()))
-        except Exception as e:
-            form.set_error('file', str(e))
-            raise ValueError()
+            testdef = TestDef.import_from_xml(fp, self.objectdef)
+        except ValueError as e:
+            form.set_error('file', _('Invalid File'))
+            raise e
 
         get_session().message = ('info', _('Test "%s" has been successfully imported.') % testdef.name)
         return redirect('.')
@@ -416,6 +441,11 @@ class TestResultDetailPage(Directory):
         except (KeyError, ValueError):
             raise TraversalError()
 
+        try:
+            self.testdef = TestDef.get(self.result['id'])
+        except KeyError:
+            self.testdef = None
+
     def _q_traverse(self, path):
         get_response().breadcrumb.append(
             (str(self.result_index) + '/', _('Details of %(test_name)s') % {'test_name': self.result['name']})
@@ -423,7 +453,39 @@ class TestResultDetailPage(Directory):
         return super()._q_traverse(path)
 
     def _q_index(self):
-        return render_to_string('wcs/backoffice/test-result-detail.html', context={'result': self.result})
+        context = {
+            'result': self.result['details'],
+            'test_name': self.result['name'],
+            'testdef': self.testdef,
+            'workflow_test_action': self.get_workflow_test_action(
+                self.result['details']['workflow_test_action_uuid']
+            ),
+        }
+
+        for request in self.result['details'].get('sent_requests', []):
+            if request['webservice_response_id']:
+                try:
+                    request['webservice_response'] = [
+                        x
+                        for x in self.testdef.get_webservice_responses()
+                        if x.id == request['webservice_response_id']
+                    ][0]
+                except IndexError:
+                    pass
+
+        return render_to_string('wcs/backoffice/test-result-detail.html', context=context)
+
+    def get_workflow_test_action(self, action_uuid):
+        if not action_uuid or not self.testdef:
+            return
+
+        try:
+            action = [x for x in self.testdef.workflow_tests.actions if x.uuid == action_uuid][0]
+        except IndexError:
+            return
+
+        action.url = self.testdef.get_admin_url() + 'workflow/#%s' % action.id
+        return action
 
 
 class TestResultPage(Directory):
@@ -452,6 +514,8 @@ class TestResultPage(Directory):
         testdefs = TestDef.select_for_objectdef(self.objectdef)
         testdefs_by_id = {x.id: x for x in testdefs}
         for test in self.test_result.results:
+            test['has_details'] = any(x for x in test['details'].values())
+
             if test['id'] in testdefs_by_id:
                 test['url'] = testdefs_by_id[test['id']].get_admin_url()
 
@@ -534,8 +598,12 @@ class TestsAfterJob(AfterJob):
         for test in testdefs:
             try:
                 test.run(objectdef)
+            except WorkflowTestError as e:
+                test.error = _('Workflow error: %s') % e
+                test.exception = e
             except TestError as e:
                 test.error = str(e)
+                test.exception = e
 
         test_result = TestResult()
         test_result.object_type = objectdef.get_table_name()
@@ -548,8 +616,13 @@ class TestsAfterJob(AfterJob):
                 'id': test.id,
                 'name': str(test),
                 'error': getattr(test, 'error', None),
-                'recorded_errors': test.recorded_errors,
-                'missing_required_fields': test.missing_required_fields,
+                'details': {
+                    'recorded_errors': test.recorded_errors,
+                    'missing_required_fields': test.missing_required_fields,
+                    'sent_requests': test.sent_requests,
+                    'workflow_test_action_uuid': test.exception.action_uuid if test.exception else None,
+                    'error_details': test.exception.details if test.exception else None,
+                },
             }
             for test in testdefs
         ]
@@ -557,3 +630,187 @@ class TestsAfterJob(AfterJob):
         test_result.store()
 
         return test_result
+
+
+class WebserviceResponsePage(Directory):
+    _q_exports = ['', 'delete', 'duplicate']
+
+    def __init__(self, component, testdef):
+        self.testdef = testdef
+        try:
+            self.webservice_response = [x for x in testdef.get_webservice_responses() if x.id == component][0]
+        except IndexError:
+            raise TraversalError()
+
+    def _q_index(self):
+        form = Form(enctype='multipart/form-data')
+        form.add(StringWidget, 'name', size=50, title=_('Name'), value=self.webservice_response.name)
+        form.add(
+            StringWidget,
+            'url',
+            title=_('URL'),
+            value=self.webservice_response.url,
+            size=80,
+        )
+        form.add(
+            WidgetDict,
+            'qs_data',
+            title=_('Restrict to query string data'),
+            value=self.webservice_response.qs_data or {},
+            element_value_type=StringWidget,
+            allow_empty_values=True,
+            value_for_empty_value='',
+        )
+        methods = collections.OrderedDict(
+            [
+                ('', _('Any')),
+                ('GET', _('GET')),
+                ('POST', _('POST (JSON)')),
+                ('PUT', _('PUT (JSON)')),
+                ('PATCH', _('PATCH (JSON)')),
+                ('DELETE', _('DELETE (JSON)')),
+            ]
+        )
+        form.add(
+            RadiobuttonsWidget,
+            'method',
+            title=_('Restrict to method'),
+            options=list(methods.items()),
+            value=self.webservice_response.method,
+            attrs={'data-dynamic-display-parent': 'true'},
+            extra_css_class='widget-inline-radio',
+        )
+        form.add(
+            WidgetDict,
+            'post_data',
+            title=_('Restrict to POST data'),
+            value=self.webservice_response.post_data or {},
+            element_value_type=StringWidget,
+            allow_empty_values=True,
+            value_for_empty_value='',
+            attrs={
+                'data-dynamic-display-child-of': 'method',
+                'data-dynamic-display-value-in': '|'.join(
+                    [
+                        str(_(methods['POST'])),
+                        str(_(methods['PUT'])),
+                        str(_(methods['PATCH'])),
+                        str(_(methods['DELETE'])),
+                    ]
+                ),
+            },
+        )
+
+        def validate_json(value):
+            try:
+                json.loads(value)
+            except ValueError as e:
+                raise ValueError(_('Invalid JSON: %s') % e)
+
+        form.add(
+            TextWidget,
+            'payload',
+            title=_('Response payload (JSON)'),
+            value=self.webservice_response.payload,
+            validation_function=validate_json,
+        )
+
+        form.add_submit('submit', _('Submit'))
+        form.add_submit('cancel', _('Cancel'))
+        form.add_media()
+
+        if form.get_widget('cancel').parse():
+            return redirect('.')
+
+        if form.get_submit() != 'submit' or form.has_errors():
+            get_response().breadcrumb.append(('edit', _('Edit webservice response')))
+            r = TemplateIO(html=True)
+            r += htmltext('<h2>%s</h2>') % (_('Edit webservice response'))
+            r += form.render()
+            return r.getvalue()
+
+        self.webservice_response.name = form.get_widget('name').parse()
+        self.webservice_response.payload = form.get_widget('payload').parse()
+        self.webservice_response.url = form.get_widget('url').parse()
+        self.webservice_response.qs_data = form.get_widget('qs_data').parse()
+        self.webservice_response.method = form.get_widget('method').parse()
+        self.webservice_response.post_data = form.get_widget('post_data').parse()
+        self.webservice_response.store()
+
+        return redirect('..')
+
+    def delete(self):
+        form = Form(enctype='multipart/form-data')
+        form.add_submit('delete', _('Delete'))
+        form.add_submit('cancel', _('Cancel'))
+
+        if form.get_widget('cancel').parse():
+            return redirect('.')
+        if not form.is_submitted() or form.has_errors():
+            get_response().breadcrumb.append(('delete', _('Delete')))
+            r = TemplateIO(html=True)
+            r += htmltext('<h2>%s %s</h2>') % (_('Deleting:'), self.webservice_response)
+            r += form.render()
+            return r.getvalue()
+
+        self.webservice_response.remove_self()
+        return redirect('..')
+
+    def duplicate(self):
+        new_webservice_response = copy.deepcopy(self.webservice_response)
+        new_webservice_response.id = None
+        new_webservice_response.name = '%s %s' % (new_webservice_response.name, _('(copy)'))
+        new_webservice_response.store()
+        return redirect('..')
+
+
+class WebserviceResponseDirectory(Directory):
+    _q_exports = ['', 'new']
+
+    def __init__(self, testdef):
+        self.testdef = testdef
+
+    def _q_traverse(self, path):
+        get_response().breadcrumb.append(('webservice-responses/', _('Webservice responses')))
+        return super()._q_traverse(path)
+
+    def _q_lookup(self, component):
+        return WebserviceResponsePage(component, self.testdef)
+
+    def _q_index(self):
+        context = {
+            'webservice_responses': self.testdef.get_webservice_responses(),
+            'has_sidebar': True,
+        }
+        get_response().add_javascript(['popup.js'])
+        get_response().set_title(_('Webservice responses'))
+        return template.QommonTemplateResponse(
+            templates=['wcs/backoffice/test-webservice-responses.html'],
+            context=context,
+            is_django_native=True,
+        )
+
+    def new(self):
+        form = Form(enctype='multipart/form-data')
+        form.add(StringWidget, 'name', title=_('Name'), required=True, size=50)
+
+        form.add_submit('submit', _('Submit'))
+        form.add_submit('cancel', _('Cancel'))
+
+        if form.get_widget('cancel').parse():
+            return redirect('.')
+
+        if not form.is_submitted() or form.has_errors():
+            get_response().breadcrumb.append(('new', _('New')))
+            get_response().set_title(_('New webservice response'))
+            r = TemplateIO(html=True)
+            r += htmltext('<h2>%s</h2>') % _('New webservice response')
+            r += form.render()
+            return r.getvalue()
+
+        webservice_response = WebserviceResponse()
+        webservice_response.testdef_id = self.testdef.id
+        webservice_response.name = form.get_widget('name').parse()
+        webservice_response.store()
+
+        return redirect(self.testdef.get_admin_url() + 'webservice-responses/%s/' % webservice_response.id)

wcs/admin/workflow_tests.py

@@ -0,0 +1,222 @@
+# w.c.s. - web application for online forms
+# Copyright (C) 2005-2023  Entr'ouvert
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+import copy
+import json
+
+from quixote import get_publisher, get_request, get_response, get_session, redirect
+from quixote.directory import Directory
+from quixote.html import TemplateIO, htmltext
+
+from wcs.qommon import _, template
+from wcs.qommon.errors import TraversalError
+from wcs.qommon.form import Form, SingleSelectWidget
+from wcs.workflow_tests import get_test_action_class_by_type, get_test_action_options
+
+
+class WorkflowTestActionPage(Directory):
+    _q_exports = ['', 'delete', 'duplicate']
+
+    def __init__(self, testdef, formdef, component):
+        self.testdef = testdef
+        self.formdef = formdef
+        try:
+            self.action = [x for x in testdef.workflow_tests.actions if x.id == component][0]
+        except IndexError:
+            raise TraversalError()
+
+    def _q_index(self):
+        form = Form(enctype='multipart/form-data')
+
+        self.action.fill_admin_form(form, self.formdef)
+
+        form.add_submit('submit', _('Submit'))
+        form.add_submit('cancel', _('Cancel'))
+
+        if form.get_widget('cancel').parse():
+            return redirect('.')
+
+        if not form.is_submitted() or form.has_errors():
+            get_response().set_title(_('Edit action'))
+            get_response().breadcrumb.append(('edit', _('Edit action')))
+            r = TemplateIO(html=True)
+            r += htmltext('<h2>%s</h2>') % (_('Edit action'))
+            r += form.render()
+            return r.getvalue()
+
+        for widget in form.widgets:
+            if hasattr(self.action, '%s_parse' % widget.name):
+                value = getattr(self.action, '%s_parse' % widget.name)(widget.value)
+            else:
+                value = widget.parse()
+
+            setattr(self.action, widget.name, value)
+
+        self.testdef.store()
+        return redirect('..')
+
+    def delete(self):
+        form = Form(enctype='multipart/form-data')
+        form.add_submit('delete', _('Delete'))
+        form.add_submit('cancel', _('Cancel'))
+
+        if form.get_widget('cancel').parse():
+            return redirect('.')
+        if not form.is_submitted() or form.has_errors():
+            get_response().set_title(_('Delete'))
+            get_response().breadcrumb.append(('delete', _('Delete')))
+            r = TemplateIO(html=True)
+            r += htmltext('<h2>%s %s</h2>') % (_('Deleting action:'), self.action)
+            r += form.render()
+            return r.getvalue()
+
+        self.testdef.workflow_tests.actions = [
+            x for x in self.testdef.workflow_tests.actions if x.id != self.action.id
+        ]
+        self.testdef.store()
+        return redirect('..')
+
+    def duplicate(self):
+        new_action = copy.deepcopy(self.action)
+        new_action.id = self.testdef.workflow_tests.get_new_action_id()
+        self.testdef.workflow_tests.actions.append(new_action)
+        self.testdef.store()
+        return redirect('..')
+
+
+class WorkflowTestsDirectory(Directory):
+    _q_exports = ['', 'options', 'update_order', 'new']
+
+    def __init__(self, testdef, formdef):
+        self.testdef = testdef
+        self.formdef = formdef
+
+    def _q_traverse(self, path):
+        get_response().set_title(_('Workflow tests'))
+        get_response().breadcrumb.append(('workflow/', _('Workflow tests')))
+        return Directory._q_traverse(self, path)
+
+    def _q_lookup(self, component):
+        return WorkflowTestActionPage(self.testdef, self.formdef, component)
+
+    def _q_index(self):
+        context = {
+            'testdef': self.testdef,
+            'has_sidebar': True,
+            'sidebar_form': self.get_sidebar_form(),
+        }
+
+        get_response().add_javascript(
+            ['popup.js', 'jquery.js', 'jquery-ui.js', 'biglist.js', 'select2.js', 'widget_list.js']
+        )
+        get_response().set_title(self.testdef.name)
+
+        return template.QommonTemplateResponse(
+            templates=['wcs/backoffice/workflow-tests.html'], context=context, is_django_native=True
+        )
+
+    def get_sidebar_form(self):
+        form = Form(enctype='multipart/form-data', action='new')
+        form.add(
+            SingleSelectWidget,
+            'type',
+            title=_('Type'),
+            required=True,
+            options=get_test_action_options(),
+        )
+        form.add_submit('submit', _('Add'))
+        return form
+
+    def options(self):
+        form = Form(enctype='multipart/form-data')
+
+        user_options = [('', '---', '')] + [
+            (str(x.id), str(x), str(x.id)) for x in get_publisher().user_class.select(order_by='name')
+        ]
+        form.add(
+            SingleSelectWidget,
+            'agent',
+            title=_('Backoffice user'),
+            value=self.testdef.agent_id,
+            options=user_options,
+            **{'data-autocomplete': 'true'},
+        )
+
+        form.add_submit('submit', _('Submit'))
+        form.add_submit('cancel', _('Cancel'))
+
+        if form.get_widget('cancel').parse():
+            return redirect('.')
+
+        if not form.is_submitted() or form.has_errors():
+            get_response().set_title(_('Options'))
+            get_response().breadcrumb.append(('options', _('Options')))
+            r = TemplateIO(html=True)
+            r += htmltext('<h2>%s</h2>') % (_('Options'))
+            r += form.render()
+            return r.getvalue()
+
+        self.testdef.agent_id = form.get_widget('agent').parse()
+        self.testdef.store()
+        return redirect('.')
+
+    def new(self):
+        form = Form(enctype='multipart/form-data')
+        form.add_hidden('type')
+
+        if not form.is_submitted() or form.has_errors():
+            get_session().message = ('error', _('Submitted form was not filled properly.'))
+            return redirect('.')
+
+        action_type = form.get_widget('type').parse()
+        action_class = get_test_action_class_by_type(action_type)
+        self.testdef.workflow_tests.add_action(action_class)
+        self.testdef.store()
+
+        return redirect('.')
+
+    def update_order(self):
+        get_response().set_content_type('application/json')
+        request = get_request()
+
+        if 'element' not in request.form:
+            return json.dumps({'success': 'ko'})
+        if 'order' not in request.form:
+            return json.dumps({'success': 'ko'})
+
+        new_order = request.form['order'].strip(';').split(';')
+        new_actions = []
+
+        # build new ordered actions list
+        for y in new_order:
+            for i, x in enumerate(self.testdef.workflow_tests.actions):
+                if x.id != y:
+                    continue
+                new_actions.append(x)
+                break
+
+        # check new actions list composition
+        if set(self.testdef.workflow_tests.actions) != set(new_actions):
+            return json.dumps({'success': 'ko'})
+
+        self.testdef.workflow_tests.actions = new_actions
+        self.testdef.store()
+
+        return json.dumps(
+            {
+                'success': 'ok',
+            }
+        )

wcs/admin/workflows.py

@@ -18,11 +18,11 @@ import io
 import itertools
 import json
 import textwrap
-import time
 import xml.etree.ElementTree as ET
 from subprocess import PIPE, Popen
 
 from django.utils.encoding import force_bytes
+from django.utils.timezone import localtime
 from quixote import get_publisher, get_request, get_response, get_session, redirect
 from quixote.directory import Directory
 from quixote.html import TemplateIO, htmltext
@@ -2295,7 +2295,7 @@ class StatusChangeJob(AfterJob):
                 else:
                     item.status = new_status
                     evo = Evolution(formdata=item)
-                    evo.time = time.localtime()
+                    evo.time = localtime()
                     evo.status = new_status
                     evo.comment = str(_('Administrator reassigned status'))
                     if not item.evolution:

wcs/api.py

@@ -18,12 +18,11 @@ import base64
 import copy
 import datetime
 import json
-import re
-import time
 import urllib.parse
 
 from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse
 from django.utils.encoding import force_bytes
+from django.utils.timezone import localtime, make_naive
 from quixote import get_publisher, get_request, get_response, get_session, redirect
 from quixote.directory import Directory
 from quixote.errors import MethodNotAllowedError, RequestError
@@ -64,7 +63,6 @@ from .backoffice.management import ManagementDirectory
 from .backoffice.submission import SubmissionDirectory
 from .qommon import _, misc
 from .qommon.errors import AccessForbiddenError, TraversalError, UnknownNameIdAccessForbiddenError
-from .qommon.form import ComputedExpressionWidget
 from .qommon.template import Template, TemplateError
 
 
@@ -155,6 +153,11 @@ def get_formdata_dict(formdata, user, consider_status_visibility=True):
     d.update(formdata.get_static_substitution_variables(minimal=True))
     if get_request().form.get('full') == 'on':
         d.update(formdata.get_json_export_dict(include_files=False, user=user))
+    if d.get('form_receipt_datetime'):
+        d['form_receipt_datetime'] = make_naive(d['form_receipt_datetime'].replace(microsecond=0))
+    if d.get('form_last_update_datetime'):
+        d['form_last_update_datetime'] = make_naive(d['form_last_update_datetime'].replace(microsecond=0))
+
     return d
 
 
@@ -705,7 +708,7 @@ class ApiFormdefDirectory(Directory):
             code.formdata = formdata  # this will .store() the code
         if meta.get('draft'):
             formdata.status = 'draft'
-            formdata.receipt_time = time.localtime()
+            formdata.receipt_time = localtime()
             formdata.store()
         else:
             formdata.just_created()
@@ -1464,30 +1467,14 @@ def geocoding(request, *args, **kwargs):
     return HttpResponse(misc.urlopen(url).read(), content_type='application/json')
 
 
-def validate_expression(request, *args, **kwargs):
-    expression = request.GET.get('expression')
-    hint = {'klass': None, 'msg': ''}
-    try:
-        ComputedExpressionWidget.validate(expression)
-    except ValidationError as e:
-        hint['klass'] = 'error'
-        hint['msg'] = str(e)
-    else:
-        if expression and re.match(r'^=.*\[[a-zA-Z_]\w*\]', expression):
-            hint['klass'] = 'warning'
-            hint['msg'] = _('Make sure you want a Python expression, not a simple template string.')
-    return JsonResponse(hint)
-
-
 def validate_condition(request, *args, **kwargs):
     condition = {}
     condition['type'] = request.GET.get('type') or ''
     condition['value'] = request.GET.get('value_' + condition['type']) or ''
-    hint = {'klass': None, 'msg': ''}
+    hint = {'msg': ''}
     try:
         Condition(condition).validate()
     except ValidationError as e:
-        hint['klass'] = 'error'
         hint['msg'] = str(e)
     return JsonResponse(hint)
 

wcs/api_export_import.py

@@ -70,6 +70,16 @@ klasses = {
 
 klass_to_slug = {y: x for x, y in klasses.items()}
 
+category_classes = [
+    Category,
+    CardDefCategory,
+    BlockCategory,
+    WorkflowCategory,
+    MailTemplateCategory,
+    CommentTemplateCategory,
+    DataSourceCategory,
+]
+
 
 def signature_required(func):
     def f(*args, **kwargs):
@@ -403,16 +413,16 @@ class BundleImportJob(AfterJob):
 
             # first pass on formdef/carddef/blockdef/workflows to create them empty
             # (name and slug); so they can be found for sure in import pass
-            for type in ('forms', 'cards', 'blocks', 'workflows'):
-                self.pre_install([x for x in manifest.get('elements') if x.get('type') == type])
+            for _type in ('forms', 'cards', 'blocks', 'workflows'):
+                self.pre_install([x for x in manifest.get('elements') if x.get('type') == _type])
 
             # real installation pass
-            for type in object_types:
-                self.install([x for x in manifest.get('elements') if x.get('type') == type])
+            for _type in object_types:
+                self.install([x for x in manifest.get('elements') if x.get('type') == _type])
 
             # again, to remove [pre-install] in dependencies labels
-            for type in object_types:
-                self.install([x for x in manifest.get('elements') if x.get('type') == type], finalize=True)
+            for _type in object_types:
+                self.install([x for x in manifest.get('elements') if x.get('type') == _type], finalize=True)
 
             # remove obsolete application elements
             self.unlink_obsolete_objects()
@@ -447,12 +457,30 @@ class BundleImportJob(AfterJob):
             get_response().process_after_jobs()
 
     def install(self, elements, finalize=False):
+        if not elements:
+            return
+
+        element_klass = klasses[elements[0]['type']]
+
+        if not finalize and element_klass in category_classes:
+            # for categories, keep positions before install
+            objects_by_slug = {i.slug: i for i in element_klass.select()}
+            initial_positions = {
+                i.slug: i.position if i.position is not None else 10000 for i in objects_by_slug.values()
+            }
+
+        imported_positions = {}
+
         for element in elements:
-            element_klass = klasses[element['type']]
             element_content = self.tar.extractfile('%s/%s' % (element['type'], element['slug'])).read()
             new_object = element_klass.import_from_xml_tree(
                 ET.fromstring(element_content), include_id=False, check_datasources=False
             )
+            if not finalize and element_klass in category_classes:
+                # for categories, keep positions of imported objects
+                imported_positions[new_object.slug] = (
+                    new_object.position if new_object.position is not None else 10000
+                )
             try:
                 existing_object = element_klass.get_by_slug(new_object.slug)
                 if existing_object is None:
@@ -500,6 +528,41 @@ class BundleImportJob(AfterJob):
             self.link_object(new_object)
             self.increment_count()
 
+        # for categories, rebuild positions
+        if not finalize and element_klass in category_classes:
+            objects_by_slug = {i.slug: i for i in element_klass.select()}
+            # find imported objects from initials
+            existing_positions = {k: v for k, v in initial_positions.items() if k in imported_positions}
+            # find not imported objects from initials
+            not_imported_positions = {
+                k: v for k, v in initial_positions.items() if k not in imported_positions
+            }
+            # determine position of application objects
+            application_position = None
+            if existing_positions:
+                application_position = min(existing_positions.values())
+            # all objects placed before application objects
+            before_positions = {
+                k: v
+                for k, v in not_imported_positions.items()
+                if application_position is None or v < application_position
+            }
+            # all objects placed after application objects
+            after_positions = {
+                k: v
+                for k, v in not_imported_positions.items()
+                if application_position is not None and v >= application_position
+            }
+            # rebuild positions
+            position = 1
+            slugs = sorted(before_positions.keys(), key=lambda a: before_positions[a])
+            slugs += sorted(imported_positions.keys(), key=lambda a: imported_positions[a])
+            slugs += sorted(after_positions.keys(), key=lambda a: after_positions[a])
+            for slug in slugs:
+                objects_by_slug[slug].position = position
+                objects_by_slug[slug].store(store_snapshot=False)
+                position += 1
+
     def link_object(self, obj):
         element = ApplicationElement.update_or_create_for_object(self.application, obj)
         self.application_elements.add((element.object_type, element.object_id))

wcs/backoffice/management.py

@@ -19,14 +19,13 @@ import datetime
 import io
 import json
 import re
-import time
 import types
 import urllib.parse
 import zipfile
 
 import vobject
 from django.utils.encoding import force_str
-from django.utils.timezone import is_naive, make_aware
+from django.utils.timezone import is_naive, make_aware, make_naive, now
 from quixote import get_publisher, get_request, get_response, get_session, redirect
 from quixote.directory import Directory
 from quixote.errors import RequestError
@@ -2784,8 +2783,12 @@ class FormPage(Directory, TempfileDirectoryMixin):
                     'digest': (filled.digests or {}).get(digest_key),
                     'text': filled.get_display_label(digest_key=digest_key),
                     'url': filled.get_url(),
-                    'receipt_time': datetime.datetime(*filled.receipt_time[:6]),
-                    'last_update_time': datetime.datetime(*filled.last_update_time[:6]),
+                    'receipt_time': make_naive(filled.receipt_time.replace(microsecond=0))
+                    if filled.receipt_time
+                    else None,
+                    'last_update_time': make_naive(filled.last_update_time.replace(microsecond=0))
+                    if filled.last_update_time
+                    else None,
                 }
                 for filled in items
             ]
@@ -3423,7 +3426,7 @@ class FormBackOfficeStatusPage(FormStatusPage):
                 formdata.backoffice_submission
                 and formdata.submission_agent_id == str(get_request().user.id)
                 and formdata.tracking_code
-                and time.time() - time.mktime(formdata.receipt_time) < 30 * 60
+                and (now() - formdata.receipt_time) < datetime.timedelta(minutes=30)
             ):
                 # keep displaying tracking code to submission agent for 30
                 # minutes after submission
@@ -3998,7 +4001,7 @@ class FormBackOfficeStatusPage(FormStatusPage):
                     '<div class="value"><a class="inspect-expand-variable" href="?expand=%s">(%s)</a>'
                 ) % (expand_value, _('expand this variable'))
             elif isinstance(v, LazyFieldVar):
-                r += htmltext('<li><code class="varname" title="%s">%s') % (k, breaking_k)
+                r += htmltext('<li><code title="%s"><span class="varname">%s</span>') % (k, breaking_k)
                 if v._formdata == self.filled:
                     field_url = None
                     if v._field.id.startswith('bo'):

wcs/backoffice/submission.py

@@ -14,10 +14,11 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
-import time
+import datetime
 import urllib.parse
 
 from django.utils.safestring import mark_safe
+from django.utils.timezone import localtime, make_aware
 from quixote import get_publisher, get_request, get_response, get_session, redirect
 from quixote.directory import Directory
 from quixote.html import TemplateIO, htmltext
@@ -124,7 +125,7 @@ class FormFillPage(PublicFormFillPage):
         formdata.submission_agent_id = str(get_request().user.id)
         formdata.submission_context = {}
         formdata.status = 'draft'
-        formdata.receipt_time = time.localtime()
+        formdata.receipt_time = localtime()
         return formdata
 
     def _q_index(self, *args, **kwargs):
@@ -506,7 +507,9 @@ class SubmissionDirectory(Directory):
                     formdef._formdatas = [
                         x for x in data_class.get_ids(formdata_ids) if x.backoffice_submission is True
                     ]
-                    formdef._formdatas.sort(key=lambda x: x.receipt_time or time.gmtime(0))
+                    formdef._formdatas.sort(
+                        key=lambda x: x.receipt_time or make_aware(datetime.datetime(1900, 1, 1))
+                    )
                 skip &= not (bool(formdef._formdatas))
             if skip:
                 return

wcs/categories.py

@@ -90,7 +90,9 @@ class Category(XmlStorableObject):
     def get_admin_url(self):
         return '%s/%s%s/' % (get_publisher().get_backoffice_url(), self.backoffice_base_url, self.id)
 
-    def store(self, *args, comment=None, snapshot_store_user=True, application=None, **kwargs):
+    def store(
+        self, *args, comment=None, snapshot_store_user=True, application=None, store_snapshot=True, **kwargs
+    ):
         if not self.url_name:
             existing_slugs = {
                 x.url_name: True for x in self.select(ignore_migration=True, ignore_errors=True)
@@ -104,7 +106,7 @@ class Category(XmlStorableObject):
                 self.url_name = '%s-%s' % (base_slug, i)
                 i += 1
         super().store(*args, **kwargs)
-        if get_publisher().snapshot_class:
+        if get_publisher().snapshot_class and store_snapshot:
             get_publisher().snapshot_class.snap(
                 instance=self, comment=comment, store_user=snapshot_store_user, application=application
             )

wcs/ctl/management/commands/delete_tenant.py

@@ -1,5 +1,5 @@
-# w.c.s. -  web application for online forms
-# Copyright (C) 2005-2014  Entr'ouvert
+# w.c.s. - web application for online forms
+# Copyright (C) 2005-2024  Entr'ouvert
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -22,30 +22,24 @@ from shutil import rmtree
 import psycopg2
 import psycopg2.errorcodes
 
-from ..qommon.ctl import Command, make_option
+from . import TenantCommand
 
 
-class CmdDeleteTenant(Command):
-    name = 'delete_tenant'
+class Command(TenantCommand):
+    support_all_tenants = False
 
-    def __init__(self):
-        Command.__init__(
-            self,
-            [
-                make_option('--force-drop', action='store_true', default=False, dest='force_drop'),
-            ],
-        )
+    def add_arguments(self, parser):
+        super().add_arguments(parser)
+        parser.add_argument('--force-drop', action='store_true')
 
-    def execute(self, base_options, sub_options, args):
-        from .. import publisher
+    def handle(self, *args, **options):
+        for domain in self.get_domains(**options):
+            publisher = self.init_tenant_publisher(domain, register_tld_names=False)
+            publisher.cleanup()
+            self.delete_tenant(publisher, **options)
 
-        publisher.WcsPublisher.configure(self.config)
-        pub = publisher.WcsPublisher.create_publisher(register_tld_names=False)
-        pub.set_tenant_by_hostname(args[0])
-        self.delete_tenant(pub, sub_options, args)
-
-    def delete_tenant(self, pub, options, args):
-        if options.force_drop:
+    def delete_tenant(self, pub, **options):
+        if options.get('force_drop'):
             rmtree(pub.app_dir)
         else:
             deletion_date = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
@@ -79,7 +73,7 @@ class CmdDeleteTenant(Command):
         dbname = postgresql_cfg.get('dbname') or postgresql_cfg.get('database')
         try:
             if createdb:
-                if options.force_drop:
+                if options.get('force_drop'):
                     cur.execute('DROP DATABASE %s' % dbname)
                 else:
                     cur.execute('ALTER DATABASE %s RENAME TO removed_%s_%s' % (dbname, deletion_date, dbname))
@@ -93,7 +87,7 @@ class CmdDeleteTenant(Command):
 
                 tables_names = [x[0] for x in cur.fetchall()]
 
-                if options.force_drop:
+                if options.get('force_drop'):
                     for table_name in tables_names:
                         cur.execute('DROP TABLE %s CASCADE' % table_name)
 
@@ -108,9 +102,8 @@ class CmdDeleteTenant(Command):
                 'failed to alter database %s: (%s)' % (dbname, psycopg2.errorcodes.lookup(e.pgcode)),
                 file=sys.stderr,
             )
+            pgconn.close()
             return
 
         cur.close()
-
-
-CmdDeleteTenant.register()
+        pgconn.close()

wcs/ctl/management/commands/hobo_deploy.py

@@ -33,6 +33,7 @@ from wcs.fields import DateField, EmailField, StringField
 from wcs.qommon import force_str, misc
 from wcs.qommon.publisher import UnknownTenantError, get_publisher_class
 from wcs.qommon.storage import atomic_write
+from wcs.sql import cleanup_connection
 
 from . import TenantCommand
 
@@ -76,6 +77,7 @@ class Command(TenantCommand):
                     options['json_filename'] = hobo_json_path
                     self.init_tenant_publisher(tenant.hostname, register_tld_names=False)
                     self.deploy(**options)
+                    cleanup_connection()
         else:
             self.deploy(**options)
 

wcs/ctl/management/commands/rebuild_indexes.py

@@ -0,0 +1,43 @@
+# w.c.s. - web application for online forms
+# Copyright (C) 2005-2024  Entr'ouvert
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+from wcs.blocks import BlockDef
+from wcs.carddef import CardDef
+from wcs.data_sources import NamedDataSource
+from wcs.formdef import FormDef
+
+from . import TenantCommand
+
+
+class Command(TenantCommand):
+    support_all_tenants = True
+
+    def add_arguments(self, parser):
+        super().add_arguments(parser)
+        parser.add_argument(
+            '--destroy',
+            action='store_true',
+            help='destroy indexes before recreating them',
+        )
+
+    def handle(self, *args, **options):
+        klasses = [BlockDef, CardDef, FormDef, NamedDataSource]
+        for domain in self.get_domains(**options):
+            self.init_tenant_publisher(domain, register_tld_names=False)
+            for klass in klasses:
+                if options.get('destroy'):
+                    klass.destroy_indexes()
+                klass.rebuild_indexes()

wcs/ctl/management/commands/replace_python.py

@@ -19,6 +19,7 @@ import json
 import requests
 
 from wcs.formdef import get_formdefs_of_all_kinds
+from wcs.mail_templates import MailTemplate
 from wcs.qommon import _
 from wcs.workflows import Workflow
 
@@ -60,6 +61,8 @@ class Command(TenantCommand):
             for field in formdef.fields or []:
                 changed |= self.replace_condition(field)
                 changed |= self.replace_prefill(field)
+                if field.key == 'page':
+                    changed |= self.replace_post_conditions(field)
             if changed:
                 formdef.store(comment=change_message)
         for workflow in Workflow.select(ignore_migration=True):
@@ -79,6 +82,17 @@ class Command(TenantCommand):
                             changed = True
             if changed:
                 workflow.store(comment=change_message)
+        for mail_template in MailTemplate.select(ignore_migration=True):
+            changed = self.replace_expression_attr(mail_template, 'subject')
+            if getattr(mail_template, 'attachments', None):
+                new_value = [
+                    self.replace_expression(x, ignore_equal_marker=True) for x in mail_template.attachments
+                ]
+                if new_value != mail_template.attachments:
+                    mail_template.attachments = new_value
+                    changed = True
+            if changed:
+                mail_template.store(comment=change_message)
 
     def replace_condition(self, obj):
         condition = getattr(obj, 'condition', None)
@@ -90,6 +104,18 @@ class Command(TenantCommand):
             return True
         return False
 
+    def replace_post_conditions(self, obj):
+        changed = False
+        for post_condition in getattr(obj, 'post_conditions', None) or []:
+            condition = post_condition.get('condition')
+            if not condition or condition.get('type') == 'django':
+                continue
+            if condition['value'] in self.replacements['conditions']:
+                condition['type'] = 'django'
+                condition['value'] = self.replacements['conditions'][condition['value']]
+                changed = True
+        return changed
+
     def replace_prefill(self, field):
         prefill = getattr(field, 'prefill', None)
         if not prefill or prefill.get('type') != 'formula' or not prefill.get('value'):
@@ -107,16 +133,21 @@ class Command(TenantCommand):
                 old_value = field.get('value')
                 field['value'] = self.replace_expression(old_value)
                 changed |= bool(field['value'] != old_value)
-        elif action.key in ('create_carddata', 'create_formdata', 'edit_carddata'):
+        if action.key in ('create_carddata', 'create_formdata', 'edit_carddata'):
             for mapping in action.mappings or []:
                 changed |= self.replace_expression_attr(mapping, 'expression')
-        elif action.key in ('sendmail', 'sendsms') and action.to:
+        if action.key in ('sendmail', 'sendsms') and action.to:
             for i, to in enumerate(action.to[:]):
                 new_value = self.replace_expression(to)
                 if new_value != to:
                     changed = True
                     action.to[i] = new_value
-        elif action.key == 'webservice_call':
+        if action.key in ('sendmail', 'register-comment') and getattr(action, 'attachments', None):
+            new_value = [self.replace_expression(x, ignore_equal_marker=True) for x in action.attachments]
+            if new_value != action.attachments:
+                action.attachments = new_value
+                changed = True
+        if action.key == 'webservice_call':
             if action.qs_data:
                 for key, value in list(action.qs_data.items()):
                     new_value = self.replace_expression(value)
@@ -170,8 +201,8 @@ class Command(TenantCommand):
             return True
         return False
 
-    def replace_expression(self, value):
-        if not isinstance(value, str) or not value.startswith('='):
+    def replace_expression(self, value, ignore_equal_marker=False):
+        if not isinstance(value, str) or (not value.startswith('=') and not ignore_equal_marker):
             return value
         python_expression = value.removeprefix('=')
         if python_expression in self.replacements['templates']:

wcs/ctl/management/commands/wipe_data.py

@@ -0,0 +1,47 @@
+# w.c.s. - web application for online forms
+# Copyright (C) 2005-2024  Entr'ouvert
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+from wcs.formdef import FormDef
+
+from . import TenantCommand
+
+
+class Command(TenantCommand):
+    def add_arguments(self, parser):
+        super().add_arguments(parser)
+        parser.add_argument(
+            '--all',
+            action='store_true',
+            help='wipe all form data',
+        )
+        parser.add_argument('--forms', metavar='FORMS', help='list of forms (slugs, separated by commas)')
+        parser.add_argument(
+            '--exclude-forms', metavar='FORMS', help='list of forms to exclude (slugs, separated by commas)'
+        )
+
+    def handle(self, *args, **options):
+        for domain in self.get_domains(**options):
+            self.init_tenant_publisher(domain, register_tld_names=False)
+            if options.get('all'):
+                formdefs = FormDef.select()
+            elif options.get('forms'):
+                formdefs = [FormDef.get_by_urlname(x) for x in options['forms'].split(',')]
+            else:
+                formdefs = []
+            if options.get('exclude_forms'):
+                formdefs = [x for x in formdefs if x.url_name not in options['exclude_forms'].split(',')]
+            for formdef in formdefs:
+                formdef.data_class().wipe()

wcs/ctl/rebuild_indexes.py

@@ -1,60 +0,0 @@
-# w.c.s. - web application for online forms
-# Copyright (C) 2005-2010  Entr'ouvert
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, see <http://www.gnu.org/licenses/>.
-
-from ..qommon.ctl import Command, make_option
-
-
-def rebuild_vhost_indexes(pub, destroy=False):
-    from wcs.formdef import FormDef
-
-    if destroy:
-        FormDef.destroy_indexes()
-    FormDef.rebuild_indexes()
-    from wcs.roles import Role
-
-    if destroy:
-        Role.destroy_indexes()
-    Role.rebuild_indexes()
-
-
-class CmdRebuildIndexes(Command):
-    name = 'rebuild_indexes'
-
-    def __init__(self):
-        Command.__init__(
-            self,
-            [
-                make_option('--all', action='store_true', dest='all', default=False),
-                make_option('--destroy', action='store_true', dest='destroy', default=False),
-            ],
-        )
-
-    def execute(self, base_options, sub_options, args):
-        from .. import publisher
-
-        publisher.WcsPublisher.configure(self.config)
-        pub = publisher.WcsPublisher.create_publisher(register_tld_names=False)
-
-        if sub_options.all:
-            hostnames = [x.domain for x in publisher.WcsPublisher.get_tenants()]
-        else:
-            hostnames = args
-        for hostname in hostnames:
-            pub.set_tenant(hostname)
-            rebuild_vhost_indexes(pub, destroy=sub_options.destroy)
-
-
-CmdRebuildIndexes.register()

wcs/ctl/wipe_data.py

@@ -1,60 +0,0 @@
-# w.c.s. - web application for online forms
-# Copyright (C) 2005-2016  Entr'ouvert
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, see <http://www.gnu.org/licenses/>.
-
-import sys
-
-from ..qommon.ctl import Command, make_option
-
-
-class CmdWipeData(Command):
-    name = 'wipe-data'
-
-    def __init__(self):
-        Command.__init__(
-            self,
-            [
-                make_option('--all', metavar='ALL', action='store_true', dest='all'),
-                make_option('--vhost', metavar='VHOST', action='store', dest='vhost'),
-            ],
-        )
-
-    def execute(self, base_options, sub_options, args):
-        if not sub_options.vhost:
-            print('you must specify --vhost', file=sys.stderr)
-            sys.exit(1)
-
-        from .. import publisher
-
-        publisher.WcsPublisher.configure(self.config)
-        pub = publisher.WcsPublisher.create_publisher(register_tld_names=False)
-        pub.set_tenant_by_hostname(sub_options.vhost)
-        self.wipe(pub, sub_options, args)
-
-    def wipe(self, pub, options, args):
-        from wcs.formdef import FormDef
-
-        if options.all:
-            formdefs = FormDef.select()
-        elif args:
-            formdefs = [FormDef.get_by_urlname(arg) for arg in args]
-        else:
-            formdefs = []
-
-        for formdef in formdefs:
-            formdef.data_class().wipe()
-
-
-CmdWipeData.register()

wcs/fields/base.py

@@ -974,7 +974,7 @@ class WidgetField(Field):
         )
 
     def check_admin_form(self, form):
-        display_locations = form.get_widget('display_locations').parse()
+        display_locations = form.get_widget('display_locations').parse() or []
         varname = form.get_widget('varname').parse()
         if 'statistics' in display_locations and not varname:
             form.set_error(

wcs/formdata.py

@@ -22,11 +22,10 @@ import html
 import itertools
 import json
 import re
-import time
 import urllib.parse
 
 from django.utils.html import strip_tags
-from django.utils.timezone import localtime
+from django.utils.timezone import localtime, make_naive
 from quixote import get_publisher, get_request, get_session
 from quixote.errors import RequestError
 from quixote.html import htmltext
@@ -213,7 +212,7 @@ class Evolution:
 
     def get_json_export_dict(self, formdata_user, anonymise=False, include_files=True, prefetched_users=None):
         data = {
-            'time': datetime.datetime(*self.time[:6]) if self.time else None,
+            'time': self.time,
             'last_jump_datetime': self.last_jump_datetime,
         }
         if self.status:
@@ -245,7 +244,7 @@ class Evolution:
 
     @property
     def datetime(self):
-        return datetime.datetime(*self.time[:6])
+        return self.time
 
     def set_user(self, formdata, user, check_submitter=True):
         if formdata.is_submitter(user) and check_submitter:
@@ -481,7 +480,7 @@ class FormData(StorableObject):
         # it should not be possible to have a formdef/carddef with a workflow without any status.
         assert self.formdef.workflow.possible_status
 
-        self.receipt_time = time.localtime()
+        self.receipt_time = localtime()
         self.status = 'wf-%s' % self.formdef.workflow.possible_status[0].id
         # we add the initial status to the history, this makes it more readable
         # afterwards (also this gets the (previous_status) code to work in all
@@ -860,11 +859,11 @@ class FormData(StorableObject):
             # just update last jump time on last evolution, do not add one
             # (ContentSnapshotPart and WorkflowTriggeredEvolutionPart are ignored
             # as they contain their own datetime attribute).
-            self.evolution[-1].last_jump_datetime = datetime.datetime.now()
+            self.evolution[-1].last_jump_datetime = localtime()
             self.store()
             return True
         evo = Evolution(self)
-        evo.time = time.localtime()
+        evo.time = localtime()
         evo.status = status
         evo.who = user_id
         self.evolution.append(evo)
@@ -1048,9 +1047,7 @@ class FormData(StorableObject):
                 }
             )
             if self.receipt_time:
-                # always get receipt time as a datetime object, this handles
-                # both normal formdata (where receipt_time is a time.struct_time)
-                # and sql.AnyFormData where it's already a datetime object.
+                # always get receipt time as a datetime object
                 d['form_receipt_datetime'] = make_datetime(self.receipt_time)
             if self.last_update_time:
                 d['form_last_update_datetime'] = make_datetime(self.last_update_time)
@@ -1355,15 +1352,14 @@ class FormData(StorableObject):
         if hasattr(self, '_last_update_time'):
             return self._last_update_time
         if self.evolution and self.evolution[-1].last_jump_datetime:
-            return self.evolution[-1].last_jump_datetime.timetuple()
+            return self.evolution[-1].last_jump_datetime
         elif self.evolution and self.evolution[-1].time:
             return self.evolution[-1].time
         else:
             return self.receipt_time
 
     def set_last_update_time(self, value):
-        if isinstance(value, datetime.datetime):
-            value = value.timetuple()
+        assert isinstance(value, (type(None), datetime.datetime))
         self._last_update_time = value
 
     last_update_time = property(get_last_update_time, set_last_update_time)
@@ -1545,8 +1541,12 @@ class FormData(StorableObject):
         data['digests'] = self.digests
         data['text'] = self.get_display_label(digest_key=digest_key)
         data['url'] = self.get_url()
-        data['receipt_time'] = datetime.datetime(*self.receipt_time[:6])
-        data['last_update_time'] = datetime.datetime(*self.last_update_time[:6])
+        data['receipt_time'] = (
+            make_naive(self.receipt_time.replace(microsecond=0)) if self.receipt_time else None
+        )
+        data['last_update_time'] = (
+            make_naive(self.last_update_time.replace(microsecond=0)) if self.last_update_time else None
+        )
 
         formdata_user = None
         if include_fields or include_workflow or include_evolution:

wcs/formdef.py

@@ -803,7 +803,9 @@ class FormDef(StorableObject):
             new_value = None
         else:
             try:
-                new_value = Template(self.submission_lateral_template, autoescape=False).render(context)
+                new_value = Template(self.submission_lateral_template, autoescape=False, raises=True).render(
+                    context
+                )
             except Exception as e:
                 get_publisher().record_error(
                     _('Could not render submission lateral template (%s)' % e),
@@ -1402,6 +1404,15 @@ class FormDef(StorableObject):
                 sub = ET.SubElement(digest_templates, 'template')
                 sub.attrib['key'] = key
                 sub.text = value
+
+        if getattr(self, '_export_tests', False):
+            from .testdef import TestDef
+
+            testdefs = TestDef.select_for_objectdef(self)
+            if testdefs:
+                elem = ET.SubElement(root, 'testdefs')
+                for testdef in testdefs:
+                    elem.append(testdef.export_to_xml())
         return root
 
     @classmethod
@@ -1601,6 +1612,8 @@ class FormDef(StorableObject):
                 value = xml_node_text(child)
                 formdef.digest_templates[key] = value
 
+        formdef.xml_testdefs = tree.find('testdefs')
+
         unknown_datasources = set()
         if check_datasources:
             # check if datasources are defined
@@ -1648,6 +1661,15 @@ class FormDef(StorableObject):
 
         return formdef
 
+    def finish_tests_xml_import(self):
+        if not self.xml_testdefs:
+            return
+
+        from .testdef import TestDef
+
+        for testdef in self.xml_testdefs:
+            TestDef.import_from_xml_tree(testdef, self)
+
     def get_detailed_email_form(self, formdata, url):
         r = ''
         if formdata.user_id and formdata.user:

wcs/forms/actions.py

@@ -259,7 +259,7 @@ class GlobalInteractiveMassActionAfterJob(AfterJob):
         form = action.get_action_form(formdata, user=user)
         get_request().form = request_form  # cancel fields overwritten by prefills
         form.method = 'get'
-        url = action.handle_form(form, formdata, user=user)
+        url = action.handle_form(form, formdata, user=user, check_replay=False)
         if afterjob:
             # reset request to avoid emails being created as afterjobs
             publisher._set_request(None)

wcs/forms/root.py

@@ -27,6 +27,7 @@ except ImportError:
 
 import ratelimit.utils
 from django.utils.http import quote
+from django.utils.timezone import localtime
 from quixote import get_publisher, get_request, get_response, get_session, get_session_manager, redirect
 from quixote.directory import AccessControlled, Directory
 from quixote.errors import MethodNotAllowedError, RequestError
@@ -1059,18 +1060,17 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
         if self._pages:
             return self._pages
         current_data = self.get_transient_formdata().data
-        pages = []
-        field_page = None
+
+        pages = [x for x in self.formdef.fields if x.key == 'page']
+        has_page_fields = bool(pages)
+
         with get_publisher().substitutions.freeze():
             # don't let evaluation of pages alter substitution variables (this
             # avoids a ConditionVars being added with current form data and
             # influencing later code evaluating field visibility based on
             # submitted data) (#27247).
-            for field in self.formdef.fields:
-                if field.key == 'page':
-                    field_page = field
-                    if field.is_visible(current_data, self.formdef):
-                        pages.append(field)
+            hidden_pages = [x for x in pages if not x.is_visible(current_data, self.formdef)]
+
         if self.edit_mode and self.edit_action and self.edit_action.operation_mode in ('single', 'partial'):
             edit_pages = []
             for page in pages:
@@ -1078,10 +1078,13 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
                     edit_pages.append(page)
                     if self.edit_action.operation_mode == 'single':
                         break
+            edit_pages = [x for x in edit_pages if x not in hidden_pages]
             if not edit_pages:
                 raise errors.TraversalError()
             pages = edit_pages
-        if not field_page:  # form without page fields
+        else:
+            pages = [x for x in pages if x not in hidden_pages]
+        if not has_page_fields:  # form without page fields
             pages = [None]
         self._pages = pages
         return pages
@@ -1632,7 +1635,7 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
         formdata.page_no = page_no
         formdata.page_id = self.get_page_id(page_no)
         formdata.data = form_data
-        formdata.receipt_time = time.localtime()
+        formdata.receipt_time = localtime()
         if not get_request().is_in_backoffice():
             formdata.user = get_request().user
         formdata.store()
@@ -1719,7 +1722,7 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
         if page_no is not None:
             filled.page_no = page_no
             filled.page_id = self.get_page_id(page_no)
-        filled.receipt_time = time.localtime()
+        filled.receipt_time = localtime()
         where = [Equal('status', 'draft')] + (where or [])
         if get_request().is_in_backoffice():
             # if submitting via backoffice store fhe formdata as is.
@@ -1954,7 +1957,7 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
                 else:
                     # add history entry
                     evo = Evolution(formdata=self.edited_data)
-                    evo.time = time.localtime()
+                    evo.time = localtime()
                     evo.who = user_id
                     self.edited_data.evolution.append(evo)
                     self.edited_data.store()

wcs/locale/fr/LC_MESSAGES/django.po

@@ -4,8 +4,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: wcs 0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-02-09 15:23+0100\n"
-"PO-Revision-Date: 2024-02-09 15:23+0100\n"
+"POT-Creation-Date: 2024-02-13 12:28+0100\n"
+"PO-Revision-Date: 2024-02-13 12:36+0100\n"
 "Last-Translator: Thomas Noël <tnoel@entrouvert.com>\n"
 "Language-Team: french\n"
 "Language: fr\n"
@@ -59,23 +59,24 @@ msgstr "Rôles donnés par cet accès"
 #: admin/api_access.py admin/blocks.py admin/categories.py
 #: admin/comment_templates.py admin/data_sources.py admin/fields.py
 #: admin/forms.py admin/logged_errors.py admin/mail_templates.py admin/roles.py
-#: admin/settings.py admin/tests.py admin/users.py admin/workflows.py
-#: admin/wscalls.py backoffice/data_management.py backoffice/i18n.py
-#: backoffice/management.py backoffice/snapshots.py forms/root.py
-#: qommon/admin/emails.py qommon/admin/texts.py qommon/ident/franceconnect.py
-#: qommon/ident/idp.py qommon/ident/password.py wf/form.py
+#: admin/settings.py admin/tests.py admin/users.py admin/workflow_tests.py
+#: admin/workflows.py admin/wscalls.py backoffice/data_management.py
+#: backoffice/i18n.py backoffice/management.py backoffice/snapshots.py
+#: forms/root.py qommon/admin/emails.py qommon/admin/texts.py
+#: qommon/ident/franceconnect.py qommon/ident/idp.py qommon/ident/password.py
+#: wf/form.py
 msgid "Submit"
 msgstr "Valider"
 
 #: admin/api_access.py admin/blocks.py admin/categories.py
 #: admin/comment_templates.py admin/data_sources.py admin/fields.py
 #: admin/forms.py admin/logged_errors.py admin/mail_templates.py admin/roles.py
-#: admin/settings.py admin/tests.py admin/users.py admin/workflows.py
-#: admin/wscalls.py backoffice/data_management.py backoffice/i18n.py
-#: backoffice/management.py backoffice/snapshots.py backoffice/submission.py
-#: forms/actions.py forms/root.py qommon/admin/emails.py qommon/admin/texts.py
-#: qommon/ident/franceconnect.py qommon/ident/idp.py qommon/ident/password.py
-#: qommon/myspace.py
+#: admin/settings.py admin/tests.py admin/users.py admin/workflow_tests.py
+#: admin/workflows.py admin/wscalls.py backoffice/data_management.py
+#: backoffice/i18n.py backoffice/management.py backoffice/snapshots.py
+#: backoffice/submission.py forms/actions.py forms/root.py
+#: qommon/admin/emails.py qommon/admin/texts.py qommon/ident/franceconnect.py
+#: qommon/ident/idp.py qommon/ident/password.py qommon/myspace.py
 msgid "Cancel"
 msgstr "Annuler"
 
@@ -99,6 +100,7 @@ msgstr "Cette valeur est déjà utilisée."
 #: templates/wcs/backoffice/mail-template.html
 #: templates/wcs/backoffice/workflow-global-action.html
 #: templates/wcs/backoffice/workflow-status.html
+#: templates/wcs/backoffice/workflow-tests.html
 #: templates/wcs/backoffice/wscall.html
 msgid "Edit"
 msgstr "Modifier"
@@ -114,13 +116,14 @@ msgstr "Vous allez définitivement supprimer cet accès aux API."
 #: admin/api_access.py admin/blocks.py admin/categories.py
 #: admin/comment_templates.py admin/data_sources.py admin/fields.py
 #: admin/forms.py admin/logged_errors.py admin/mail_templates.py admin/roles.py
-#: admin/settings.py admin/tests.py admin/users.py admin/workflows.py
-#: admin/wscalls.py backoffice/management.py
+#: admin/settings.py admin/tests.py admin/users.py admin/workflow_tests.py
+#: admin/workflows.py admin/wscalls.py backoffice/management.py
 #: templates/wcs/backoffice/api_access.html
 #: templates/wcs/backoffice/category.html templates/wcs/backoffice/formdef.html
 #: templates/wcs/backoffice/test_sidebar.html
 #: templates/wcs/backoffice/workflow-global-action.html
 #: templates/wcs/backoffice/workflow-status.html
+#: templates/wcs/backoffice/workflow-tests.html
 #: templates/wcs/backoffice/workflow.html
 msgid "Delete"
 msgstr "Supprimer"
@@ -142,6 +145,7 @@ msgstr "Accès aux API"
 #: admin/api_access.py admin/categories.py admin/data_sources.py admin/forms.py
 #: admin/roles.py admin/tests.py admin/users.py admin/workflows.py
 #: admin/wscalls.py backoffice/cards.py qommon/ident/idp.py statistics/views.py
+#: templates/wcs/backoffice/test-webservice-responses.html
 #: templates/wcs/backoffice/tests.html workflows.py
 msgid "New"
 msgstr "Nouveau"
@@ -183,7 +187,9 @@ msgstr "Utilisation"
 #: admin/fields.py admin/forms.py admin/mail_templates.py admin/tests.py
 #: admin/workflows.py qommon/admin/menu.py
 #: templates/wcs/backoffice/formdef.html
+#: templates/wcs/backoffice/test-webservice-responses.html
 #: templates/wcs/backoffice/test_sidebar.html
+#: templates/wcs/backoffice/workflow-tests.html
 #: templates/wcs/backoffice/workflow.html
 msgid "Duplicate"
 msgstr "Dupliquer"
@@ -295,7 +301,8 @@ msgid "Misc"
 msgstr "Divers"
 
 #: admin/blocks.py admin/comment_templates.py admin/fields.py
-#: admin/mail_templates.py admin/workflows.py backoffice/data_management.py
+#: admin/mail_templates.py admin/workflow_tests.py admin/workflows.py
+#: backoffice/data_management.py
 msgid "Add"
 msgstr "Ajouter"
 
@@ -339,7 +346,7 @@ msgid "Invalid File (%s)"
 msgstr "Fichier invalide (%s)"
 
 #: admin/blocks.py admin/categories.py admin/comment_templates.py
-#: admin/data_sources.py admin/forms.py admin/mail_templates.py
+#: admin/data_sources.py admin/forms.py admin/mail_templates.py admin/tests.py
 #: admin/workflows.py admin/wscalls.py
 msgid "Invalid File"
 msgstr "Fichier invalide"
@@ -973,7 +980,7 @@ msgstr "Toutes les pages"
 msgid "Next page"
 msgstr "Page suivante"
 
-#: admin/fields.py
+#: admin/fields.py templates/wcs/backoffice/workflow-tests.html
 msgid "Use drag and drop with the handles to reorder fields."
 msgstr "Vous pouvez utiliser les poignées ⣿ pour ordonner les champs."
 
@@ -1008,7 +1015,7 @@ msgstr "Nouveau champ"
 msgid "Label"
 msgstr "Libellé"
 
-#: admin/fields.py admin/workflows.py fields/base.py
+#: admin/fields.py admin/workflow_tests.py admin/workflows.py fields/base.py
 msgid "Type"
 msgstr "Type"
 
@@ -1042,7 +1049,7 @@ msgstr "Changement de l’ordre des champs"
 msgid "Also move the fields of the page"
 msgstr "Également déplacer les champs de la page"
 
-#: admin/fields.py admin/workflows.py
+#: admin/fields.py admin/workflow_tests.py admin/workflows.py
 msgid "Submitted form was not filled properly."
 msgstr "Le formulaire transmis n’a pas été correctement rempli."
 
@@ -1397,12 +1404,14 @@ msgstr "Standard"
 msgid "Open workflow page"
 msgstr "Ouvrir la page du workflow"
 
-#: admin/forms.py admin/roles.py templates/wcs/backoffice/carddef.html
+#: admin/forms.py admin/roles.py admin/workflow_tests.py
+#: templates/wcs/backoffice/carddef.html
 #: templates/wcs/backoffice/formdef-inspect.html
 #: templates/wcs/backoffice/formdef.html
 #: templates/wcs/backoffice/test_sidebar.html
 #: templates/wcs/backoffice/workflow-global-action.html
 #: templates/wcs/backoffice/workflow-status.html
+#: templates/wcs/backoffice/workflow-tests.html
 msgid "Options"
 msgstr "Options"
 
@@ -2312,8 +2321,8 @@ msgstr "Options SMS"
 msgid "Sender (number or name)"
 msgstr "Expéditeur (nom ou numéro)"
 
-#: admin/settings.py wf/notification.py wf/redirect_to_url.py wf/wscall.py
-#: wscalls.py
+#: admin/settings.py admin/tests.py wf/notification.py wf/redirect_to_url.py
+#: wf/wscall.py wscalls.py
 msgid "URL"
 msgstr "URL"
 
@@ -2488,6 +2497,11 @@ msgstr "Modifier les données"
 msgid "Mark as failing"
 msgstr "Marquer comme devant échouer"
 
+#: admin/tests.py admin/workflow_tests.py
+#: templates/wcs/backoffice/workflow-tests.html
+msgid "Workflow tests"
+msgstr "Tests de workflow"
+
 #: admin/tests.py
 #, python-format
 msgid "This test is expected to fail on error \"%s\"."
@@ -2572,6 +2586,73 @@ msgstr "Résultats des tests"
 msgid "Manual run."
 msgstr "Lancement manuel."
 
+#: admin/tests.py
+#, python-format
+msgid "Workflow error: %s"
+msgstr "Erreur du workflow : %s"
+
+#: admin/tests.py
+msgid "Restrict to query string data"
+msgstr "Limiter aux paramètres de l’URL"
+
+#: admin/tests.py wf/resubmit.py
+msgid "Any"
+msgstr "Au choix"
+
+#: admin/tests.py wf/wscall.py wscalls.py
+msgid "GET"
+msgstr "GET"
+
+#: admin/tests.py wf/wscall.py wscalls.py
+msgid "POST (JSON)"
+msgstr "POST (JSON)"
+
+#: admin/tests.py wf/wscall.py wscalls.py
+msgid "PUT (JSON)"
+msgstr "PUT (JSON)"
+
+#: admin/tests.py wf/wscall.py wscalls.py
+msgid "PATCH (JSON)"
+msgstr "PATCH (JSON)"
+
+#: admin/tests.py wf/wscall.py wscalls.py
+msgid "DELETE (JSON)"
+msgstr "DELETE (JSON)"
+
+#: admin/tests.py
+msgid "Restrict to method"
+msgstr "Limiter à la méthode"
+
+#: admin/tests.py
+msgid "Restrict to POST data"
+msgstr "Limiter au données contenues dans le corps de la requête"
+
+#: admin/tests.py
+#, python-format
+msgid "Invalid JSON: %s"
+msgstr "JSON invalide : %s"
+
+#: admin/tests.py
+msgid "Response payload (JSON)"
+msgstr "Contenu de la réponse (JSON)"
+
+#: admin/tests.py
+msgid "Edit webservice response"
+msgstr "Modifier la réponse webservice"
+
+#: admin/tests.py
+msgid "Deleting:"
+msgstr "Suppression :"
+
+#: admin/tests.py templates/wcs/backoffice/test-webservice-responses.html
+#: templates/wcs/backoffice/test_sidebar.html
+msgid "Webservice responses"
+msgstr "Réponses webservice"
+
+#: admin/tests.py
+msgid "New webservice response"
+msgstr "Nouvelle réponse webservice"
+
 #: admin/users.py fields/base.py fields/email.py formdata.py formdef.py
 #: forms/root.py qommon/admin/emails.py qommon/ident/franceconnect.py
 #: qommon/ident/idp.py qommon/ident/password.py wf/profile.py wf/sendmail.py
@@ -2699,6 +2780,18 @@ msgstr "Exporter cette version"
 msgid "Inspect version"
 msgstr "Inspecter cette version"
 
+#: admin/workflow_tests.py
+msgid "Edit action"
+msgstr "Modifier l’action"
+
+#: admin/workflow_tests.py
+msgid "Deleting action:"
+msgstr "Suppression de l’action :"
+
+#: admin/workflow_tests.py
+msgid "Backoffice user"
+msgstr "Utilisateur agent"
+
 #: admin/workflows.py
 msgid "Workflow Name"
 msgstr "Nom du workflow"
@@ -3422,12 +3515,6 @@ msgstr "%(name)s - n°%(id)s (%(status)s)"
 msgid "unknown"
 msgstr "inconnu"
 
-#: api.py
-msgid "Make sure you want a Python expression, not a simple template string."
-msgstr ""
-"Assurez-vous que vous voulez une expression Python, et non un simple gabarit "
-"de texte avec des variables de substitution (sans le signe = au début)."
-
 #: api_export_import.py backoffice/data_management.py backoffice/root.py
 #: data_sources.py templates/wcs/backoffice/data-management.html
 msgid "Cards"
@@ -4654,7 +4741,8 @@ msgstr "pas de valeur"
 msgid "variables from parent's request"
 msgstr "variables de la demande parente"
 
-#: backoffice/management.py wf/create_formdata.py workflow_traces.py
+#: backoffice/management.py templates/wcs/backoffice/test-result-detail.html
+#: wf/create_formdata.py workflow_traces.py
 msgid "deleted"
 msgstr "supprimé"
 
@@ -5234,6 +5322,7 @@ msgstr "source de données non disponible (champ : %s)"
 
 #: fields/base.py fields/computed.py qommon/ident/franceconnect.py
 #: wf/backoffice_fields.py wf/criticality.py wf/dispatch.py wf/profile.py
+#: workflow_tests.py
 msgid "Value"
 msgstr "Valeur"
 
@@ -6699,6 +6788,7 @@ msgstr ""
 "dessous :"
 
 #: qommon/admin/menu.py qommon/templates/qommon/forms/widgets/block_sub.html
+#: templates/wcs/backoffice/test-webservice-responses.html
 #: templates/wcs/backoffice/workflow-global-action.html
 msgid "Remove"
 msgstr "Supprimer"
@@ -8774,6 +8864,11 @@ msgstr "impossibilité de faire le rendu du gabarit EZT : %s"
 msgid "syntax error in Django template: %s"
 msgstr "erreur de syntaxe dans le gabarit Django : %s"
 
+#: qommon/template.py
+#, python-format
+msgid "missing variable \"%s\" in template"
+msgstr "variable « %s » absente dans le gabarit"
+
 #: qommon/template.py
 #, python-format
 msgid "failure to render Django template: %s"
@@ -8882,6 +8977,11 @@ msgstr "|objects appelé sur une source invalide (%r)"
 msgid "|objects with invalid reference (%r)"
 msgstr "|objects utilisé avec une référence invalide (%r)"
 
+#: qommon/templatetags/qommon.py
+#, python-format
+msgid "|check_no_duplicates not used on a list (%s)"
+msgstr "|check_no_duplicates pas utilisé sur une liste (%s)"
+
 #: roles.py
 msgid "Logged Users"
 msgstr "Utilisateurs identifiés"
@@ -9120,6 +9220,7 @@ msgstr "Blocs de champs"
 #: templates/wcs/backoffice/formdef.html templates/wcs/backoffice/forms.html
 #: templates/wcs/backoffice/mail-templates.html
 #: templates/wcs/backoffice/snapshots.html
+#: templates/wcs/backoffice/test-webservice-responses.html
 #: templates/wcs/backoffice/test_sidebar.html
 #: templates/wcs/backoffice/tests.html
 #: templates/wcs/backoffice/workflow-global-action.html
@@ -9829,6 +9930,10 @@ msgstr "Voir toutes les erreurs"
 msgid "No errors, congratulations!"
 msgstr "Aucune erreur, bravo !"
 
+#: templates/wcs/backoffice/test-result-detail.html
+msgid "Test action:"
+msgstr "Action de test :"
+
 #: templates/wcs/backoffice/test-result-detail.html
 msgid "Recorded errors:"
 msgstr "Erreurs enregistrées :"
@@ -9837,6 +9942,22 @@ msgstr "Erreurs enregistrées :"
 msgid "Missing required fields:"
 msgstr "Champs obligatoires manquants :"
 
+#: templates/wcs/backoffice/test-result-detail.html
+msgid "Sent requests:"
+msgstr "Requêtes envoyées :"
+
+#: templates/wcs/backoffice/test-result-detail.html
+msgid "Used webservice response:"
+msgstr "Réponse webservice utilisée :"
+
+#: templates/wcs/backoffice/test-result-detail.html
+msgid "Request was blocked since it is not a GET request."
+msgstr "La requête a été bloquée car ce n’était pas une requête GET."
+
+#: templates/wcs/backoffice/test-result-detail.html
+msgid "You can create corresponding webservice response here."
+msgstr "Vous pouvez créer la réponse webservice correspondante ici."
+
 #: templates/wcs/backoffice/test-result.html
 msgid "Result"
 msgstr "Résultat"
@@ -9874,6 +9995,16 @@ msgstr "Démarré par"
 msgid "No test results yet."
 msgstr "Pas encore de résultats des tests."
 
+#: templates/wcs/backoffice/test-webservice-responses.html
+#: wf/assign_carddata.py wf/create_formdata.py wf/redirect_to_url.py
+#: wf/roles.py
+msgid "not configured"
+msgstr "non configurée"
+
+#: templates/wcs/backoffice/test-webservice-responses.html
+msgid "There are no webservice responses yet."
+msgstr "Il n’y a pas encore de réponses webservice."
+
 #: templates/wcs/backoffice/test_edit_sidebar.html
 msgid "backoffice,frontoffice"
 msgstr "backoffice,frontoffice"
@@ -10059,6 +10190,24 @@ msgstr "Plein écran"
 msgid "Unforce Terminal Status"
 msgstr "Ne plus forcer le caractère final"
 
+#: templates/wcs/backoffice/workflow-tests.html
+msgid "Backoffice user is not defined, workflow tests will not be executed."
+msgstr ""
+"L’utilisateur agent n’est pas défini, les tests de workflow ne seront pas "
+"exécutés."
+
+#: templates/wcs/backoffice/workflow-tests.html
+msgid "Open test options"
+msgstr "Accéder aux options"
+
+#: templates/wcs/backoffice/workflow-tests.html
+msgid "There are no workflow test actions yet."
+msgstr "Il n’y a pas encore d’actions de test."
+
+#: templates/wcs/backoffice/workflow-tests.html
+msgid "New workflow test action"
+msgstr "Nouvelle action de test"
+
 #: templates/wcs/backoffice/workflow.html
 msgid "change category"
 msgstr "changer de catégorie"
@@ -10309,6 +10458,13 @@ msgstr "Valeur vide"
 msgid "%(error)s for field %(label)s: %(details)s"
 msgstr "%(error)s pour le champ %(label)s : %(details)s"
 
+#: testdef.py
+#, python-format
+msgid ""
+"Unexpected error when mocking webservice call for url %(url)s: %(error)s."
+msgstr ""
+"Erreur inattendue lors de l’appel webservice vers l’URL %(url)s : %(error)s."
+
 #: users.py
 #, python-format
 msgid "Session User Field: %s"
@@ -10569,11 +10725,6 @@ msgstr "Rattacher les fiches liées à la demande/fiche"
 msgid "Specify the list of cards which will be assigned"
 msgstr "Préciser la liste des fiches qui seront rattachées"
 
-#: wf/assign_carddata.py wf/create_formdata.py wf/redirect_to_url.py
-#: wf/roles.py
-msgid "not configured"
-msgstr "non configurée"
-
 #: wf/attachment.py wf/wscall.py
 msgid "Attachment"
 msgstr "Fichier joint"
@@ -11483,10 +11634,6 @@ msgstr "Resoumission"
 msgid "Resubmit"
 msgstr "Resoumettre"
 
-#: wf/resubmit.py
-msgid "Any"
-msgstr "Au choix"
-
 #: wf/resubmit.py
 msgid "Same as form"
 msgstr "Identique au formulaire"
@@ -11625,26 +11772,6 @@ msgstr "Raison"
 msgid "Request Signature Key"
 msgstr "Clé de signature de la requête"
 
-#: wf/wscall.py wscalls.py
-msgid "GET"
-msgstr "GET"
-
-#: wf/wscall.py wscalls.py
-msgid "POST (JSON)"
-msgstr "POST (JSON)"
-
-#: wf/wscall.py wscalls.py
-msgid "PUT (JSON)"
-msgstr "PUT (JSON)"
-
-#: wf/wscall.py wscalls.py
-msgid "PATCH (JSON)"
-msgstr "PATCH (JSON)"
-
-#: wf/wscall.py wscalls.py
-msgid "DELETE (JSON)"
-msgstr "DELETE (JSON)"
-
 #: wf/wscall.py wscalls.py
 msgid "Post formdata"
 msgstr "Envoyer les données du formulaire"
@@ -11741,6 +11868,115 @@ msgstr "Erreur lors de l’appel au webservice « %s »"
 msgid "Error calling webservice"
 msgstr "Erreur lors de l’appel au webservice"
 
+#: workflow_tests.py
+#, python-format
+msgid "Form status when error occured: %s"
+msgstr "Statut de la demande quand l’erreur s’est produite : %s"
+
+#: workflow_tests.py
+msgid "Simulate click on action button"
+msgstr "Clic sur un bouton d’action"
+
+#: workflow_tests.py
+#, python-format
+msgid "Button \"%s\" is not displayed."
+msgstr "Le bouton « %s » n’est pas affiché."
+
+#: workflow_tests.py
+msgid "not available"
+msgstr "pas disponible"
+
+#: workflow_tests.py
+msgid "Button name"
+msgstr "Texte du bouton"
+
+#: workflow_tests.py
+msgid "Assert form status"
+msgstr "Vérifier le statut de la demande"
+
+#: workflow_tests.py
+#, python-format
+msgid ""
+"Form should be in status \"%(expected_status)s\" but is in status "
+"\"%(status)s\"."
+msgstr ""
+"La demande devrait être dans le statut « %(expected_status)s » mais est dans "
+"le statut « %(status)s »."
+
+#: workflow_tests.py
+msgid "Status name"
+msgstr "Nom du statut"
+
+#: workflow_tests.py
+msgid "Assert email is sent"
+msgstr "Vérifier l’envoi d’un courriel"
+
+#: workflow_tests.py
+msgid "No email was sent."
+msgstr "Aucun courriel envoyé."
+
+#: workflow_tests.py
+#, python-format
+msgid "Email subject: %s"
+msgstr "Sujet du courriel : %s"
+
+#: workflow_tests.py
+#, python-format
+msgid "Email subject does not contain \"%s\"."
+msgstr "Le sujet du courriel ne contient pas « %s »."
+
+#: workflow_tests.py
+#, python-format
+msgid "Email body: %s"
+msgstr "Corps du courriel : %s"
+
+#: workflow_tests.py
+#, python-format
+msgid "Email body does not contain \"%s\"."
+msgstr "Le corps du courriel ne contient pas « %s »."
+
+#: workflow_tests.py
+msgid "Subject must contain"
+msgstr "Le sujet doit contenir"
+
+#: workflow_tests.py
+msgid "Add string"
+msgstr "Ajouter un texte"
+
+#: workflow_tests.py
+msgid "Body must contain"
+msgstr "Le corps doit contenir"
+
+#: workflow_tests.py
+msgid "Move forward in time"
+msgstr "Avancer dans le temps"
+
+#: workflow_tests.py
+#, python-format
+msgid "ex.: 1 day 12 hours. Usable units of time: %(variables)s."
+msgstr ""
+"exemple : « 1 jour 12 heures ». Unités de temps utilisables : %(variables)s."
+
+#: workflow_tests.py
+msgid "Assert backoffice field values"
+msgstr "Vérifier la valeur des données de traitement"
+
+#: workflow_tests.py
+#, python-format
+msgid "Field %(field_id)s not found (expected value \"%(value)s\")."
+msgstr ""
+"Le champ donnée de traitement %(field_id)s n’a pas été trouvé (valeur "
+"attendue « %(value)s »."
+
+#: workflow_tests.py
+#, python-format
+msgid ""
+"Wrong value for backoffice field \"%(field)s\" (expected "
+"\"%(expected_value)s\", got \"%(value)s\")."
+msgstr ""
+"Mauvaise valeur pour la donnée de traitement « %(field)s » (devait valoir "
+"« %(expected_value)s » mais valait « %(value)s »."
+
 #: workflow_traces.py
 msgid "Created (by API)"
 msgstr "Création (par l’API)"

wcs/qommon/emails.py

@@ -153,7 +153,14 @@ def is_sane_address(email):
     return True
 
 
-def email(
+def email(*args, **kwargs):
+    fire_and_forget = kwargs.pop('fire_and_forget', False)
+    email = get_email(*args, **kwargs)
+    if email:
+        return send_email(email, fire_and_forget)
+
+
+def get_email(
     subject,
     mail_body,
     email_rcpt,
@@ -163,7 +170,6 @@ def email(
     email_type=None,
     want_html=True,
     hide_recipients=False,
-    fire_and_forget=False,
     smtp_timeout=None,
     attachments=(),
     extra_headers=None,
@@ -171,11 +177,6 @@ def email(
 ):
     # noqa pylint: disable=too-many-arguments
 
-    if not get_request():
-        # we are not processing a request, no sense delaying the handling
-        # (for example when running a cronjob)
-        fire_and_forget = False
-
     emails_cfg = get_cfg('emails', {})
     footer = emails_cfg.get('footer') or ''
 
@@ -397,7 +398,15 @@ def email(
     if len(str(email_msg.message())) > 50_000_000:
         raise errors.TooBigEmailError()
 
-    email_to_send = EmailToSend(email_msg, smtp_timeout)
+    return EmailToSend(email_msg, smtp_timeout)
+
+
+def send_email(email_to_send, fire_and_forget=False):
+    if not get_request():
+        # we are not processing a request, no sense delaying the handling
+        # (for example when running a cronjob)
+        fire_and_forget = False
+
     if not fire_and_forget:
         email_to_send()
     else:

wcs/qommon/form.py

@@ -1871,7 +1871,7 @@ class CheckboxesWidget(Widget):
             name = option['name']
             if name in request.form and not request.form[name] in (False, '', 'False'):
                 values.append(option['value'])
-        self.value = values
+        self.value = values or None
         if self.required and not self.value:
             self.set_error(self.REQUIRED_ERROR)
         if self.value and self.min_choices and len(self.value) < self.min_choices:

wcs/qommon/misc.py

@@ -853,13 +853,22 @@ def validate_phone_fr(string_value):
     if not re.match(r'^[0\+][\d\.\s]+$', string_value):
         # leading zero or +, then digits, dots, or spaces
         return False
+
     french_country_codes = [33, 262, 508, 590, 594, 596]
-    pn = None
-    try:
-        pn = phonenumbers.parse(string_value, 'FR')
-    except phonenumbers.NumberParseException:
-        return False
-    return bool(phonenumbers.is_valid_number(pn) and pn.country_code in french_country_codes)
+    french_region_codes = [phonenumbers.region_code_for_country_code(x) for x in french_country_codes]
+
+    for region_code in french_region_codes:
+        pn = None
+        try:
+            pn = phonenumbers.parse(string_value, region_code)
+        except phonenumbers.NumberParseException:
+            continue
+        if not phonenumbers.is_valid_number(pn):
+            continue
+        if pn.country_code not in french_country_codes:
+            continue
+        return True
+    return False
 
 
 def validate_mobile_phone_local(string_value):
@@ -1197,8 +1206,8 @@ def get_document_type_value_options(current_document_type):
     return options
 
 
-def xml_response(obj, filename, content_type='text/xml'):
-    etree = obj.export_to_xml(include_id=True)
+def xml_response(obj, filename, content_type='text/xml', include_id=True):
+    etree = obj.export_to_xml(include_id=include_id)
     if hasattr(obj, 'get_admin_url'):
         etree.attrib['url'] = obj.get_admin_url()
     indent_xml(etree)
@@ -1348,7 +1357,7 @@ def parse_decimal(value, do_raise=False, keep_none=False):
         value = value.replace(',', '.')
     try:
         return decimal.Decimal(value).quantize(decimal.Decimal('1.000000')).normalize()
-    except (ArithmeticError, TypeError, decimal.InvalidOperation):
+    except (ArithmeticError, TypeError, decimal.InvalidOperation, ValueError):
         if do_raise:
             raise
         return decimal.Decimal(0)

wcs/qommon/static/css/dc2/admin.scss

@@ -3120,3 +3120,14 @@ ul.objects-list.single-links li.list-item-no-usage p {
 div.infonotice.columns-default-value-message {
 	margin-top: 0;
 }
+
+ul.biglist li.workflow-test-action:target {
+	border-left: solid;
+}
+
+ul.objects-list.single-links li a.link-action-icon.duplicate {
+	margin-right: 3em;
+	&::before {
+		content: "\f24d"; /* clone */
+	}
+}

wcs/qommon/static/js/qommon.admin.js

@@ -127,54 +127,33 @@ $(function() {
         });
     });
 
-    /* hints on the computed expression widget */
-    var validation_timeout_id = 0;
-    $('input[data-validation-url]').on('change focus input', function() {
-      var val = $(this).val();
-      var $widget = $(this).parents('.ComputedExpressionWidget');
-      var validation_url = $(this).data('validation-url');
-      clearTimeout(validation_timeout_id);
-      validation_timeout_id = setTimeout(function() {
-        $.ajax({
-          url: validation_url,
-          data: {expression: val},
-          dataType: 'json',
-          success: function(data) {
-            $widget.removeClass('hint-warning');
-            $widget.removeClass('hint-error');
-            if (data.klass) {
-              $widget.addClass('hint-' + data.klass);
-            }
-            $widget.prop('title', data.msg);
-          }
-        })}, 250);
-      return false;
-    });
-
+    // "live" update on condition widget
     $('div[data-validation-url]').each(function(idx, elem) {
       var $widget = $(this);
       var widget_name = $widget.find('input').attr('name');
       var prefix = widget_name.substr(0, widget_name.lastIndexOf('$')) + '$';
-      $(this).find('input, select').on('change focus input', function() {
-        clearTimeout($widget.validation_timeout_id);
-        $widget.validation_timeout_id = setTimeout(function() {
-          var data = Object();
-          $widget.find('select, input').each(function(idx, elem) {
-            data[$(elem).attr('name').replace(prefix, '')] = $(elem).val();
-          });
-          $.ajax({
-            url: $widget.data('validation-url'),
-            data: data,
-            dataType: 'json',
-            success: function(data) {
-              $widget.removeClass('hint-warning');
-              $widget.removeClass('hint-error');
-              if (data.klass) {
-                $widget.addClass('hint-' + data.klass);
-              }
-              $widget.prop('title', data.msg);
+      $(this).find('input, select').on('blur', function() {
+        var data = Object();
+        $widget.find('select, input').each(function(idx, elem) {
+          data[$(elem).attr('name').replace(prefix, '')] = $(elem).val();
+        });
+        $.ajax({
+          url: $widget.data('validation-url'),
+          data: data,
+          dataType: 'json',
+          success: function(data) {
+            var $error = $widget.find('.error');
+            if ($error.length == 0) {
+               $error = $('<div class="error"></div>');
+               $error.appendTo($widget);
             }
-          })}, 250);
+            if (data.msg) {
+              $error.text(data.msg);
+            } else {
+              $error.remove();
+            }
+          }
+        });
         return false;
       });
     });

wcs/qommon/static/js/qommon.fileupload.js

@@ -454,5 +454,14 @@ $.WcsFileUpload = {
         $(base_widget).find('[type=file]').hide();
         $(base_widget).find('.use-file-from-fargo').hide();
         $(base_widget).addClass('has-file').removeClass('has-no-file');
+
+        $.WcsFileUpload.image_preview(base_widget, data.token);
+    },
+
+    image_preview: function(base_widget, img_token) {
+        var file_button = base_widget.find('.file-button');
+        if(file_button.hasClass("file-image")) {
+            file_button[0].style.setProperty('--image-preview-url', `url(${window.location.href}tempfile?t=${img_token}&thumbnail=1)`);
+        }
     }
 }

wcs/qommon/static/js/qommon.forms.js

@@ -433,6 +433,9 @@ $(function() {
         $(this).parents('.widget-with-error').addClass('widget-reset-error');
       });
     });
+    $base.find('div.widget-prefilled').on('change input paste', function(ev) {
+      $(this).removeClass('widget-prefilled');
+    });
   }
 
   add_js_behaviours($('form[data-live-url], form[data-backoffice-preview]'));

wcs/qommon/template.py

@@ -301,6 +301,9 @@ class Template:
             if self.raises:
                 from . import _
 
+                if isinstance(e, DjangoVariableDoesNotExist):
+                    raise TemplateError(_('missing variable "%s" in template') % e.params[0])
+
                 raise TemplateError(_('failure to render Django template: %s'), e)
             return self.value
         except (FormDefDoesNotExist, CardDefDoesNotExist) as e:

wcs/qommon/templates/qommon/forms/widgets/file.html

@@ -3,6 +3,7 @@
 
 {% block widget-control %}
   <div class="file-button {% if widget.is_image %}file-image{% endif %}"
+       {% if widget.has_tempfile_image %}style="--image-preview-url: url({{ eservices_url }}{{ global_context.form_slug }}/tempfile?t={{ widget.tempfile.token }}&thumbnail=1)"{% endif %}
        {% if widget.automatic_image_resize %}data-image-resize="true"{% endif %}>
     {% for w in widget.get_widgets %}
       {{ w.render|safe }}

wcs/qommon/templatetags/qommon.py

@@ -1289,3 +1289,12 @@ def with_auth(value, arg):
 @register.filter
 def wbr(value):
     return mark_safe(value.replace('_', '_<wbr/>'))
+
+
+@register.filter
+def check_no_duplicates(value):
+    value = unlazy(value)
+    if not isinstance(value, (type(None), tuple, list, set)):
+        get_publisher().record_error(_('|check_no_duplicates not used on a list (%s)') % value)
+        return False
+    return bool(len(value or []) == len(set(value or [])))

wcs/snapshots.py

@@ -139,6 +139,16 @@ class Snapshot:
     _instance = None
     _user = None
 
+    _category_types = [
+        'block_category',
+        'card_category',
+        'data_source_category',
+        'category',
+        'mail_template_category',
+        'comment_template_category',
+        'workflow_category',
+    ]
+
     @classmethod
     def snap(cls, instance, comment=None, label=None, store_user=True, application=None):
         obj = cls()
@@ -147,7 +157,13 @@ class Snapshot:
         obj.timestamp = now()
         if get_session() and store_user:
             obj.user_id = get_session().user
+
         tree = instance.export_to_xml(include_id=True)
+        # remove position for categories
+        if obj.object_type in cls._category_types:
+            for position in tree.findall('position'):
+                tree.remove(position)
+
         obj.serialization = ET.tostring(tree).decode('utf-8')
         obj.comment = str(comment) if comment else None
         obj.label = label
@@ -321,15 +337,23 @@ class Snapshot:
         instance = self.instance
         if as_new:
             for attr in ('id', 'url_name', 'internal_identifier', 'slug'):
-                setattr(instance, attr, None)
+                try:
+                    setattr(instance, attr, None)
+                except AttributeError:
+                    # attribute can be a property without setter
+                    pass
+            if self.object_type in self._category_types:
+                # set position
+                instance.position = max(i.position or 0 for i in self.get_object_class().select()) + 1
             if hasattr(instance, 'disabled'):
                 instance.disabled = True
         else:
-            # keep table and max field id from current object
+            # keep table and position from current object
             current_object = self.get_object_class().get(instance.id)
-            for attr in ('table_name',):
-                if hasattr(current_object, attr):
-                    setattr(instance, attr, getattr(current_object, attr))
+            for attr in ('table_name', 'position'):
+                if attr != 'position' or self.object_type in self._category_types:
+                    if hasattr(current_object, attr):
+                        setattr(instance, attr, getattr(current_object, attr))
 
         delattr(instance, 'readonly')
         delattr(instance, 'snapshot_object')

wcs/sql.py

@@ -484,7 +484,7 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
             cur.execute(
                 '''CREATE TABLE %s (id serial PRIMARY KEY,
                                         user_id varchar,
-                                        receipt_time timestamp,
+                                        receipt_time timestamptz,
                                         anonymised timestamptz,
                                         status varchar,
                                         page_no varchar,
@@ -497,8 +497,8 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
                 '''CREATE TABLE %s_evolutions (id serial PRIMARY KEY,
                                         who varchar,
                                         status varchar,
-                                        time timestamp,
-                                        last_jump_datetime timestamp,
+                                        time timestamptz,
+                                        last_jump_datetime timestamptz,
                                         comment text,
                                         parts bytea,
                                         formdata_id integer REFERENCES %s (id) ON DELETE CASCADE)'''
@@ -509,12 +509,13 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
         cur.execute('LOCK TABLE %s;' % table_name)
 
         cur.execute(
-            '''SELECT column_name FROM information_schema.columns
+            '''SELECT column_name, data_type FROM information_schema.columns
                         WHERE table_schema = 'public'
                           AND table_name = %s''',
             (table_name,),
         )
-        existing_fields = {x[0] for x in cur.fetchall()}
+        existing_field_types = {x[0]: x[1] for x in cur.fetchall()}
+        existing_fields = set(existing_field_types.keys())
 
         needed_fields = {x[0] for x in formdef.data_class()._table_static_fields}
         needed_fields.add('fts')
@@ -536,6 +537,12 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
             if field_name not in existing_fields:
                 cur.execute('''ALTER TABLE %s ADD COLUMN %s %s''' % (table_name, field_name, field_type))
 
+        # store datetimes with timezone
+        if existing_field_types.get('receipt_time') not in (None, 'timestamp with time zone'):
+            cur.execute(f'ALTER TABLE {table_name} ALTER COLUMN receipt_time SET DATA TYPE timestamptz')
+        if existing_field_types.get('last_update_time') not in (None, 'timestamp with time zone'):
+            cur.execute(f'ALTER TABLE {table_name} ALTER COLUMN last_update_time SET DATA TYPE timestamptz')
+
         # add new fields
         for field in formdef.get_all_fields():
             assert field.id is not None
@@ -577,15 +584,24 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
 
         # migrations on _evolutions table
         cur.execute(
-            '''SELECT column_name FROM information_schema.columns
+            '''SELECT column_name, data_type FROM information_schema.columns
                         WHERE table_schema = 'public'
                           AND table_name = '%s_evolutions'
                     '''
             % table_name
         )
-        evo_existing_fields = {x[0] for x in cur.fetchall()}
+        evo_existing_fields = {x[0]: x[1] for x in cur.fetchall()}
         if 'last_jump_datetime' not in evo_existing_fields:
-            cur.execute('''ALTER TABLE %s_evolutions ADD COLUMN last_jump_datetime timestamp''' % table_name)
+            cur.execute(
+                '''ALTER TABLE %s_evolutions ADD COLUMN last_jump_datetime timestamptz''' % table_name
+            )
+
+        if evo_existing_fields.get('time') not in (None, 'timestamp with time zone'):
+            cur.execute(f'ALTER TABLE {table_name}_evolutions ALTER COLUMN time SET DATA TYPE timestamptz')
+        if evo_existing_fields.get('last_jump_datetime') not in (None, 'timestamp with time zone'):
+            cur.execute(
+                f'ALTER TABLE {table_name}_evolutions ALTER COLUMN last_jump_datetime SET DATA TYPE timestamptz'
+            )
 
         if rebuild_views or len(existing_fields - needed_fields):
             # views may have been dropped when dropping columns, so we recreate
@@ -1317,6 +1333,7 @@ def drop_views(formdef, conn, cur):
     # remove the global views
     drop_global_views(conn, cur)
 
+    view_names = []
     if formdef:
         # remove the form view itself
         view_prefix = 'wcs\\_view\\_%s\\_%%' % formdef.id
@@ -1329,14 +1346,26 @@ def drop_views(formdef, conn, cur):
             (view_prefix,),
         )
     else:
-        # if there's no formdef specified, remove all form views
+        # if there's no formdef specified, remove all form & card views
         cur.execute(
             '''SELECT table_name FROM information_schema.views
                         WHERE table_schema = 'public'
                           AND table_name LIKE %s''',
             ('wcs\\_view\\_%',),
         )
-    view_names = []
+        while True:
+            row = cur.fetchone()
+            if row is None:
+                break
+            view_names.append(row[0])
+
+        cur.execute(
+            '''SELECT table_name FROM information_schema.views
+                        WHERE table_schema = 'public'
+                          AND table_name LIKE %s''',
+            ('wcs\\_carddata\\_view\\_%',),
+        )
+
     while True:
         row = cur.fetchone()
         if row is None:
@@ -1485,13 +1514,13 @@ def do_global_views(conn, cur):
         formdef_id integer NOT NULL,
         id integer NOT NULL,
         user_id character varying,
-        receipt_time timestamp without time zone,
+        receipt_time timestamp with time zone,
         status character varying,
         id_display character varying,
         submission_agent_id character varying,
         submission_channel character varying,
         backoffice_submission boolean,
-        last_update_time timestamp without time zone,
+        last_update_time timestamp with time zone,
         digests jsonb,
         user_label character varying,
         concerned_roles_array text[],
@@ -1524,17 +1553,22 @@ def do_global_views(conn, cur):
         cur.execute('LOCK TABLE wcs_all_forms;')
 
         cur.execute(
-            '''SELECT column_name FROM information_schema.columns
+            '''SELECT column_name, data_type FROM information_schema.columns
                         WHERE table_schema = 'public'
                           AND table_name = %s''',
             ('wcs_all_forms',),
         )
-        existing_fields = {x[0] for x in cur.fetchall()}
+        existing_fields = {x[0]: x[1] for x in cur.fetchall()}
         if 'statistics_data' not in existing_fields:
             cur.execute('ALTER TABLE wcs_all_forms ADD COLUMN statistics_data jsonb')
         if 'relations_data' not in existing_fields:
             cur.execute('ALTER TABLE wcs_all_forms ADD COLUMN relations_data jsonb')
 
+        if existing_fields.get('receipt_time') not in (None, 'timestamp with time zone'):
+            cur.execute('ALTER TABLE wcs_all_forms ALTER COLUMN receipt_time SET DATA TYPE timestamptz')
+        if existing_fields.get('last_update_time') not in (None, 'timestamp with time zone'):
+            cur.execute('ALTER TABLE wcs_all_forms ALTER COLUMN last_update_time SET DATA TYPE timestamptz')
+
         clean_global_views(conn, cur)
 
         for category in wcs.categories.Category.select():
@@ -1990,6 +2024,7 @@ class SqlMixin:
             ignore_errors=ignore_errors,
             limit=limit,
             offset=offset,
+            itersize=itersize,
         )
         func_clause = parse_clause(clause)[2]
         if func_clause and (limit or offset):
@@ -2198,7 +2233,7 @@ class SqlDataMixin(SqlMixin):
     _table_static_fields = [
         ('id', 'serial'),
         ('user_id', 'varchar'),
-        ('receipt_time', 'timestamp'),
+        ('receipt_time', 'timestamptz'),
         ('status', 'varchar'),
         ('page_no', 'varchar'),
         ('page_id', 'varchar'),
@@ -2221,7 +2256,7 @@ class SqlDataMixin(SqlMixin):
         ('submission_agent_id', 'varchar'),
         ('submission_channel', 'varchar'),
         ('criticality_level', 'int'),
-        ('last_update_time', 'timestamp'),
+        ('last_update_time', 'timestamptz'),
         ('digests', 'jsonb'),
         ('user_label', 'varchar'),
         ('auto_geoloc', 'point'),
@@ -2265,8 +2300,6 @@ class SqlDataMixin(SqlMixin):
         o._sql_id, o.who, o.status, o.time, o.last_jump_datetime, o.comment = (
             str_encode(x) for x in tuple(row[:6])
         )
-        if o.time:
-            o.time = o.time.timetuple()
         if row[6]:
             o.parts = LazyEvolutionList(row[6])
         return o
@@ -2409,14 +2442,8 @@ class SqlDataMixin(SqlMixin):
             # if evolution was loaded it may have been been modified, and last update time
             # should then be refreshed.
             delattr(self, '_last_update_time')
-        if self.last_update_time:
-            sql_dict['last_update_time'] = datetime.datetime.fromtimestamp(time.mktime(self.last_update_time))
-        else:
-            sql_dict['last_update_time'] = None
-        if self.receipt_time:
-            sql_dict['receipt_time'] = datetime.datetime.fromtimestamp(time.mktime(self.receipt_time))
-        else:
-            sql_dict['receipt_time'] = None
+        sql_dict['last_update_time'] = self.last_update_time
+        sql_dict['receipt_time'] = self.receipt_time
         if self.workflow_roles:
             sql_dict['workflow_roles_array'] = []
             for x in self.workflow_roles.values():
@@ -2538,7 +2565,7 @@ class SqlDataMixin(SqlMixin):
                     {
                         'who': evo.who,
                         'status': evo.status,
-                        'time': datetime.datetime.fromtimestamp(time.mktime(evo.time)),
+                        'time': evo.time,
                         'last_jump_datetime': evo.last_jump_datetime,
                         'comment': evo.comment,
                         'formdata_id': self.id,
@@ -2619,8 +2646,6 @@ class SqlDataMixin(SqlMixin):
         o = cls()
         for static_field, value in zip(cls._table_static_fields, tuple(row[: len(cls._table_static_fields)])):
             setattr(o, static_field[0], str_encode(value))
-        if o.receipt_time:
-            o.receipt_time = o.receipt_time.timetuple()
         for attr in ('workflow_data', 'workflow_roles', 'submission_context', 'prefilling_data'):
             if getattr(o, attr):
                 setattr(o, attr, pickle_loads(getattr(o, attr)))
@@ -3945,12 +3970,12 @@ class TestDef(SqlMixin):
     _table_static_fields = [
         ('id', 'serial'),
         ('name', 'varchar'),
-        ('slug', 'varchar'),
         ('object_type', 'varchar'),
         ('object_id', 'varchar'),
         ('data', 'jsonb'),
         ('is_in_backoffice', 'boolean'),
         ('expected_error', 'varchar'),
+        ('agent_id', 'varchar'),
     ]
 
     id = None
@@ -3970,13 +3995,12 @@ class TestDef(SqlMixin):
             cur.execute(
                 '''CREATE TABLE %s (id SERIAL PRIMARY KEY,
                                             name varchar,
-                                            slug varchar NOT NULL,
                                             object_type varchar NOT NULL,
                                             object_id varchar NOT NULL,
                                             data jsonb,
                                             is_in_backoffice boolean NOT NULL DEFAULT FALSE,
                                             expected_error varchar,
-                                            UNIQUE(slug, object_type, object_id)
+                                            agent_id varchar
                                             )'''
                 % table_name
             )
@@ -3996,6 +4020,14 @@ class TestDef(SqlMixin):
         if 'expected_error' not in existing_fields:
             cur.execute('''ALTER TABLE %s ADD COLUMN expected_error varchar''' % table_name)
 
+        if 'agent_id' not in existing_fields:
+            cur.execute('''ALTER TABLE %s ADD COLUMN agent_id varchar''' % table_name)
+
+        # delete obsolete fields
+        needed_fields = {x[0] for x in TestDef._table_static_fields}
+        for field in existing_fields - needed_fields:
+            cur.execute('''ALTER TABLE %s DROP COLUMN %s''' % (table_name, field))
+
         cur.close()
 
     def store(self):
@@ -4110,7 +4142,7 @@ class TestResult(SqlMixin):
             sql_dict['id'] = self.id
             sql_statement = '''UPDATE %s SET %s WHERE id = %%(id)s RETURNING id''' % (
                 self._table_name,
-                ', '.join(['%s = %%(%s)s' % (x, x) for x in column_names]),
+                ', '.join(['%s = %s' % (x, y) for x, y in zip(column_names, column_values)]),
             )
             cur.execute(sql_statement, sql_dict)
 
@@ -4127,6 +4159,23 @@ class TestResult(SqlMixin):
     def get_data_fields(cls):
         return []
 
+    @classmethod
+    def migrate_legacy(cls):
+        for test_result in TestResult.select():
+            store = False
+            for result in test_result.results:
+                if 'details' not in result:
+                    result['details'] = {
+                        'recorded_errors': result.pop('recorded_errors', []),
+                        'missing_required_fields': result.pop('missing_required_fields', []),
+                        'workflow_test_action_uuid': None,
+                        'form_status': None,
+                    }
+                    store = True
+
+            if store:
+                test_result.store()
+
 
 class WorkflowTrace(SqlMixin):
     _table_name = 'workflow_traces'
@@ -4282,7 +4331,7 @@ class WorkflowTrace(SqlMixin):
                             elif trace.event in ('workflow-created',):
                                 trace.event_args['display_id'] = part.event_args[0]
                         trace.status_id = status_id
-                        trace.timestamp = make_aware(datetime.datetime(*evo.time[:6]), is_dst=True)
+                        trace.timestamp = evo.time
                         trace.store()
                         for action in part.actions or []:
                             trace = cls(formdata=formdata)
@@ -5046,7 +5095,7 @@ def get_period_total(
 # latest migration, number + description (description is not used
 # programmaticaly but will make sure git conflicts if two migrations are
 # separately added with the same number)
-SQL_LEVEL = (101, 'add page_id on formdata')
+SQL_LEVEL = (105, 'change test result json structure')
 
 
 def migrate_global_views(conn, cur):
@@ -5215,10 +5264,12 @@ def migrate():
         # 79: add translatable column to TranslatableMessage table
         # 100: always create translation messages table
         TranslatableMessage.do_table()
-    if sql_level < 87:
+    if sql_level < 104:
         # 72: add testdef table
         # 87: add testdef is_in_backoffice column
         # 88: add testdef expected_error column
+        # 103: drop testdef slug column
+        # 104: add testdef agent_id column
         TestDef.do_table()
     if sql_level < 95:
         # 95: add a searchable_formdefs table
@@ -5237,6 +5288,9 @@ def migrate():
         # 83: add test_result table
         # 89: rerun creation of test results table
         TestResult.do_table()
+    if sql_level < 105:
+        # 105: change test result json structure
+        set_reindex('test_result', 'needed', conn=conn, cur=cur)
     if sql_level < 84:
         # 84: add application tables
         Application.do_table()
@@ -5287,6 +5341,8 @@ def migrate():
         # 41: update full text normalization
         # 51: add index on formdata blockdef fields
         # 55: update full text normalisation (switch to unidecode)
+        # 58: add workflow_merged_roles_dict as a jsonb column with
+        #     combined formdef and formdata value.
         # 61: use setweight on formdata & user indexation
         # 62: use setweight on formdata & user indexation (reapply)
         # 96: change to fts normalization
@@ -5316,20 +5372,22 @@ def migrate():
                 continue
             for formdata in formdef.data_class().select_iterator():
                 formdata._set_auto_fields(cur)  # build digests
-    if sql_level < 91:
+    if sql_level < 102:
         # 58: add workflow_merged_roles_dict as a jsonb column with
         #     combined formdef and formdata value.
         # 69: add auto_geoloc field to form/card tables
         # 80: add jsonb column to hold statistics data
         # 91: add jsonb column to hold relations data
+        # 102: switch formdata datetime columns to timestamptz
+        drop_views(None, conn, cur)
         for formdef in FormDef.select() + CardDef.select():
             do_formdef_tables(formdef, rebuild_views=False, rebuild_global_views=False)
         migrate_views(conn, cur)
-        set_reindex('formdata', 'needed', conn=conn, cur=cur)
-    if sql_level < 99:
+    if sql_level < 102:
         # 81: add statistics data column to wcs_all_forms
         # 82: add statistics data column to wcs_all_forms, for real
         # 99: add more indexes
+        # 102: switch formdata datetime columns to timestamptz
         migrate_global_views(conn, cur)
     if sql_level < 60:
         # 59: switch wcs_all_forms to a trigger-maintained table
@@ -5398,6 +5456,7 @@ def reindex():
             klass.do_indexes(cur, concurrently=True)
         for formdef in FormDef.select() + CardDef.select():
             do_formdef_indexes(formdef, cur=cur, concurrently=True)
+        set_reindex('sqlindexes', 'done', conn=conn, cur=cur)
 
     if is_reindex_needed('user', conn=conn, cur=cur):
         for user in SqlUser.select(iterator=True):
@@ -5497,6 +5556,10 @@ def reindex():
         TestDef.migrate_legacy()
         set_reindex('testdef', 'done', conn=conn, cur=cur)
 
+    if is_reindex_needed('test_result', conn=conn, cur=cur):
+        TestResult.migrate_legacy()
+        set_reindex('test_result', 'done', conn=conn, cur=cur)
+
     cur.close()
 
 

wcs/templates/wcs/backoffice/formdef-inspect.html

@@ -92,38 +92,41 @@
       </div>
 
       <div id="inspect-drafts" role="tabpanel" tabindex="0" aria-labelledby="tab-drafts" hidden>
-        <h3>{% trans "Drafts" %}</h3>
-        <table class="stats">
-          <thead><tr><th colspan="4">{% trans "Page" %}</th></tr></thead>
-          <tbody>
-            {% for page_drafts in drafts %}
-              {% with page_id=page_drafts.0 draft_data=page_drafts.1 %}
-                {% if draft_data.total %}
-                  <tr>
-                    <td class="label">
-                      {% if page_id == "_unkown" %}
-                        {% trans "Unkown" %}
-                      {% elif page_id == "_first_page" %}
-                        {% trans "Only page" %}
-                      {% elif page_id == "_confirmation_page" %}
-                        {% trans "Confirmation page" %}
-                      {% else %}
-                        {{ draft_data.field.ellipsized_label }}
-                      {% endif %}
-                    </td>
-                    <td class="percent"> {{draft_data.percent}}&nbsp;%</td>
-                    <td class="total">({{draft_data.total}}/{{drafts_total}})</td>
-                  </tr>
-                  <tr>
-                    <td class="bar" colspan="3">
-                      <span style="width: {{draft_data.percent_rounded}}%"></span>
-                    </td>
-                  </tr>
-                {% endif %}
-              {% endwith %}
-            {% endfor %}
-          </tbody>
-        </table>
+        {% if drafts %}
+          <table class="stats">
+            <thead><tr><th colspan="4">{% trans "Page" %}</th></tr></thead>
+            <tbody>
+              {% for page_drafts in drafts %}
+                {% with page_id=page_drafts.0 draft_data=page_drafts.1 %}
+                  {% if draft_data.total %}
+                    <tr id="{{ page_id }}">
+                      <td class="label">
+                        {% if page_id == "_unkown" %}
+                          {% trans "Unkown" %}
+                        {% elif page_id == "_first_page" %}
+                          {% trans "Only page" %}
+                        {% elif page_id == "_confirmation_page" %}
+                          {% trans "Confirmation page" %}
+                        {% else %}
+                          {{ draft_data.field.ellipsized_label }}
+                        {% endif %}
+                      </td>
+                      <td class="percent"> {{draft_data.percent}}&nbsp;%</td>
+                      <td class="total">({{draft_data.total}}/{{drafts_total}})</td>
+                    </tr>
+                    <tr>
+                      <td class="bar" colspan="3">
+                        <span style="width: {{draft_data.percent_rounded}}%"></span>
+                      </td>
+                    </tr>
+                  {% endif %}
+                {% endwith %}
+              {% endfor %}
+            </tbody>
+          </table>
+        {% else %}
+          <p>{% trans "No drafts found for this form" %}</p>
+        {% endif %}
       </div>
 
       <div id="inspect-customviews" role="tabpanel" tabindex="0" aria-labelledby="tab-customviews" hidden>

wcs/templates/wcs/backoffice/test-result-detail.html

@@ -2,12 +2,25 @@
 {% load i18n %}
 
 {% block appbar-title %}
-  {% blocktrans with test_name=result.name %}Details of {{ test_name }}{% endblocktrans %}
+  {% blocktrans %}Details of {{ test_name }}{% endblocktrans %}
 {% endblock %}
 
 {% block body %}
   <div class="section">
     <ul>
+      {% if result.workflow_test_action_uuid %}
+        <li id='test-action'>
+          {% trans "Test action:" %}
+          {% if workflow_test_action %}
+            <a href="{{ workflow_test_action.url }}">{{ workflow_test_action.label }}</a>
+          {% else %}
+            {% trans "deleted" %}
+          {% endif %}
+        </li>
+      {% endif %}
+      {% for line in result.error_details %}
+        <li>{{ line }}</li>
+      {% endfor %}
       {% if result.recorded_errors %}
         <li>{% trans "Recorded errors:" %}</li>
         <ul>
@@ -21,6 +34,37 @@
           {% trans "Missing required fields:" %} {{ result.missing_required_fields|join:"," }}
         </li>
       {% endif %}
+      {% if result.sent_requests %}
+        <li>{% trans "Sent requests:" %}</li>
+        <ul>
+          {% for request in result.sent_requests %}
+            <li>
+              {{ request.method }} {{ request.url }}
+              <ul>
+                {% if request.webservice_response_id %}
+                  <li>
+                    {% trans "Used webservice response:" %}
+                    {% if request.webservice_response %}
+                      <a href="{{ testdef.get_admin_url }}webservice-responses/{{ request.webservice_response.id }}/">
+                        {{ request.webservice_response.name }}
+                      </a>
+                    {% else %}
+                      {% trans "deleted" %}
+                    {% endif %}
+                  </li>
+                {% elif request.forbidden_method %}
+                  <li>
+                    {% trans "Request was blocked since it is not a GET request." %}
+                    <a href="{{ testdef.get_admin_url }}webservice-responses/">
+                      {% trans "You can create corresponding webservice response here." %}
+                    </a>
+                  </li>
+                {% endif %}
+              </ul>
+            </li>
+          {% endfor %}
+        </ul>
+      {% endif %}
     </ul>
   </div>
 {% endblock %}

wcs/templates/wcs/backoffice/test-result.html

@@ -28,7 +28,7 @@
               <td><a {% if test.url %}href="{{ test.url }}"{% else %}disabled{% endif %}>{{ test.name }}</a></td>
               <td>{% firstof test.error _("Success!") %}</td>
               <td>
-                {% if test.missing_required_fields or test.recorded_errors %}
+                {% if test.has_details %}
                   <a rel="popup" data-selector="div.section" href="{{ forloop.counter0 }}/">{% trans "Display details" %}</a>
                 {% endif %}
               </td>

wcs/templates/wcs/backoffice/test-webservice-responses.html

@@ -0,0 +1,32 @@
+{% extends "wcs/backoffice.html" %}
+{% load i18n %}
+
+{% block appbar-title %}{% trans "Webservice responses" %}{% endblock %}
+
+{% block sidebar-content %}
+  <h3>{% trans "Actions" %}</h3>
+  <a class="button button-paragraph" href="new" rel="popup">{% trans "New" %}</a>
+{% endblock %}
+
+{% block body %}
+  <div class="section">
+    {% if webservice_responses %}
+      <ul class="objects-list single-links">
+        {% for response in webservice_responses %}
+          <li>
+            <a href="{{ response.id }}/">
+              {{ response }}
+              {% if not response.is_configured %}
+                <i>({% trans "not configured" %})</i>
+              {% endif %}
+            </a>
+            <a rel="popup" class="delete" href="{{ response.id }}/delete">{% trans "Remove" %}</a>
+            <a class="link-action-icon duplicate" href="{{ response.id }}/duplicate">{% trans "Duplicate" %}</a>
+          </li>
+        {% endfor %}
+      </ul>
+    {% else %}
+      <div><p>{% trans "There are no webservice responses yet." %}<p></div>
+    {% endif %}
+  </div>
+{% endblock %}

wcs/templates/wcs/backoffice/test_sidebar.html

@@ -10,5 +10,6 @@
 <h3>{% trans "Navigation" %}</h3>
 <ul class="sidebar--buttons">
   <li><a class="button button-paragraph" rel="popup" href="edit">{% trans "Options" %}</a></li>
+  <li><a class="button button-paragraph" href="webservice-responses/">{% trans "Webservice responses" %}</a></li>
   <li><a class="button button-paragraph" href="inspect">{% trans "Inspect" %}</a></li>
 </ul>

wcs/templates/wcs/backoffice/workflow-tests.html

@@ -0,0 +1,54 @@
+{% extends "wcs/backoffice.html" %}
+{% load i18n %}
+
+{% block appbar-title %}{% trans "Workflow tests" %}{% endblock %}
+
+{% block appbar-actions %}
+  <a href="options" rel="popup">{% trans "Options" %}</a>
+{% endblock %}
+
+{% block body %}
+  {% if not testdef.agent_id %}
+    <div class="warningnotice">
+      <p>
+        {% trans "Backoffice user is not defined, workflow tests will not be executed." %}
+        <a href="options" rel="popup">{% trans "Open test options" %}</a>
+      </p>
+    </div>
+  {% elif not testdef.workflow_tests.actions %}
+    <div class="infonotice"><p>{% trans "There are no workflow test actions yet." %}</p></div>
+  {% else %}
+    <p class="hint">{% trans "Use drag and drop with the handles to reorder fields." %}</p>
+  {% endif %}
+
+  <ul class="biglist sortable">
+    {% for action in testdef.workflow_tests.actions %}
+      <li id="{{ action.id }}" data-id="{{ action.id }}" class="biglistitem workflow-test-action">
+        <span class="biglistitem--content">
+          <strong class="label">{{ action.label }}</strong>
+          <span class="biglistitem--content-details">
+            <span class="type">{{ action.render_as_line }}</span>
+          </span>
+        </span>
+        <p class="commands">
+          <span class="edit">
+            <a href="{{ action.id }}/" rel="popup" title="{% trans "Edit" %}">{% trans "Edit" %}</a>
+          </span>
+          <span class="duplicate">
+            <a href="{{ action.id }}/duplicate" title="{% trans "Duplicate" %}">{% trans "Duplicate" %}</a>
+          </span>
+          <span class="remove">
+            <a href="{{ action.id }}/delete" rel="popup" title="{% trans "Delete" %}">{% trans "Delete" %}</a>
+          </span>
+        </p>
+      </li>
+    {% endfor %}
+  </ul>
+{% endblock %}
+
+{% block sidebar-content %}
+  <div>
+    <h3>{% trans "New workflow test action" %}</h3>
+    {{ sidebar_form.render|safe }}
+  </div>
+{% endblock %}

wcs/testdef.py

@@ -14,43 +14,107 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+import http
 import io
+import json
+import socket
+import urllib.parse
+import xml.etree.ElementTree as ET
 from contextlib import contextmanager
 
+import requests
 from django.core.handlers.wsgi import WSGIRequest
 from quixote import get_publisher, get_session_manager
+from urllib3 import HTTPResponse
 
 from wcs import sql
 from wcs.compat import CompatHTTPRequest
 from wcs.fields import Field, PageField
 from wcs.qommon.form import FileWithPreviewWidget, Form, get_selection_error_text
+from wcs.qommon.storage import Equal
 from wcs.qommon.template import TemplateError
-from wcs.sql_criterias import Equal
+from wcs.qommon.xml_storage import XmlStorableObject
 from wcs.workflows import WorkflowStatusItem
 
-from .qommon import _, misc
+from .qommon import _
 
 
 class TestError(Exception):
-    def __init__(self, msg, error=None):
+    action_uuid = None
+
+    def __init__(self, msg, error=None, details=None):
         self.msg = msg
         self.error = error or msg
+        self.details = details or []
 
     # prevent pytest from trying to collect this class (#75521)
     __test__ = False
 
 
+class TestDefXmlProxy(XmlStorableObject):
+    xml_root_node = 'testdef'
+    _names = 'testdef'
+    readonly = True
+
+    # prevent pytest from trying to collect this class
+    __test__ = False
+
+    _webservice_responses = []
+
+    @classmethod
+    @property
+    def XML_NODES(cls):
+        json_to_xml_types = {
+            'varchar': 'str',
+            'boolean': 'bool',
+            'jsonb': 'jsonb',
+        }
+        excluded_fields = ['id', 'object_type', 'object_id']
+        extra_fields = [
+            ('workflow_tests', 'workflow_tests'),
+            ('_webservice_responses', 'webservice_responses'),
+        ]
+
+        return [
+            (field, json_to_xml_types[kind])
+            for field, kind in sql.TestDef._table_static_fields
+            if field not in excluded_fields
+        ] + extra_fields
+
+    def export_jsonb_to_xml(self, element, attribute_name, **kwargs):
+        element.text = json.dumps(getattr(self, attribute_name))
+
+    def import_jsonb_from_xml(self, element, **kwargs):
+        return json.loads(element.text)
+
+    def export_workflow_tests_to_xml(self, element, attribute_name, **kwargs):
+        for subelement in self.workflow_tests.export_to_xml():
+            element.append(subelement)
+
+    def import_workflow_tests_from_xml(self, element, **kwargs):
+        from wcs.workflow_tests import WorkflowTests
+
+        return WorkflowTests.import_from_xml_tree(element)
+
+    def export_webservice_responses_to_xml(self, element, attribute_name, **kwargs):
+        for response in self._webservice_responses:
+            element.append(response.export_to_xml())
+
+    def import_webservice_responses_from_xml(self, element, **kwargs):
+        return [WebserviceResponse.import_from_xml_tree(response) for response in element]
+
+
 class TestDef(sql.TestDef):
     _names = 'testdef'
 
     name = ''
-    slug = None
     object_type = None  # (formdef, carddef, etc.)
     object_id = None
 
     data = None  # (json export of formdata, carddata, etc.)
     is_in_backoffice = False
     expected_error = None
+    agent_id = None
 
     ignored_field_types = (
         'subtitle',
@@ -66,27 +130,48 @@ class TestDef(sql.TestDef):
     def __str__(self):
         return self.name
 
+    @property
+    def workflow_tests(self):
+        from wcs.workflow_tests import WorkflowTests
+
+        if hasattr(self, '_workflow_tests'):
+            return self._workflow_tests
+
+        workflow_tests_list = WorkflowTests.select([Equal('testdef_id', self.id)])
+        self._workflow_tests = workflow_tests_list[0] if workflow_tests_list else WorkflowTests()
+        return self._workflow_tests
+
+    @workflow_tests.setter
+    def workflow_tests(self, value):
+        self._workflow_tests = value
+
+    def get_webservice_responses(self):
+        return WebserviceResponse.select([Equal('testdef_id', self.id)], order_by='name')
+
     def get_admin_url(self):
         base_url = get_publisher().get_backoffice_url()
         objects_dir = 'forms' if self.object_type == 'formdefs' else 'cards'
         return '%s/%s/%s/tests/%s/' % (base_url, objects_dir, self.object_id, self.id)
 
-    def store(self, *args, comment=None, snapshot_store_user=True, **kwargs):
-        if not self.slug:
-            existing_slugs = {
-                x.slug
-                for x in self.select(
-                    [Equal('object_type', self.object_type), Equal('object_id', self.object_id)]
-                )
-            }
-            base_slug = misc.simplify(self.name)
-            self.slug = base_slug
-            i = 2
-            while self.slug in existing_slugs:
-                self.slug = '%s-%s' % (base_slug, i)
-                i += 1
+    def store(self, *args, **kwargs):
         super().store(*args, **kwargs)
 
+        self.workflow_tests.testdef_id = self.id
+        self.workflow_tests.store()
+
+    @classmethod
+    def remove_object(cls, id):
+        super().remove_object(id)
+        from wcs.workflow_tests import WorkflowTests
+
+        workflow_tests_list = WorkflowTests.select([Equal('testdef_id', id)])
+        for workflow_tests in workflow_tests_list:
+            workflow_tests.remove_self()
+
+        responses = WebserviceResponse.select([Equal('testdef_id', id)])
+        for response in responses:
+            response.remove_self()
+
     @classmethod
     def select_for_objectdef(cls, objectdef):
         return cls.select(
@@ -126,6 +211,8 @@ class TestDef(sql.TestDef):
 
     def build_formdata(self, objectdef, include_fields=False):
         formdata = objectdef.data_class()()
+        formdata.just_created()
+
         if self.data['user']:
             formdata.set_user_from_json(self.data['user'])
 
@@ -148,6 +235,7 @@ class TestDef(sql.TestDef):
             self.recorded_errors.append(str(error_summary or exception))
 
         real_record_error = get_publisher().record_error
+        real_http_adapter = getattr(get_publisher(), '._http_adapter', None)
 
         true_request = get_publisher().get_request()
         wsgi_request = WSGIRequest({'REQUEST_METHOD': 'POST', 'wsgi.input': io.StringIO()})
@@ -157,12 +245,16 @@ class TestDef(sql.TestDef):
             get_publisher()._set_request(fake_request)
             fake_request.session = get_session_manager().new_session(None)
             get_publisher().record_error = record_error
+            get_publisher()._http_adapter = MockWebserviceResponseAdapter(self)
             yield
         finally:
             get_publisher()._set_request(true_request)
             get_publisher().record_error = real_record_error
+            get_publisher()._http_adapter = real_http_adapter
 
     def run(self, objectdef):
+        self.exception = None
+        self.sent_requests = []
         self.recorded_errors = []
         self.missing_required_fields = []
         with self.fake_request():
@@ -184,6 +276,12 @@ class TestDef(sql.TestDef):
                     )
 
     def _run(self, objectdef):
+        formdata = self.run_form_fill(objectdef)
+        if self.agent_id:
+            agent_user = get_publisher().user_class.get(self.agent_id)
+            self.workflow_tests.run(formdata, agent_user)
+
+    def run_form_fill(self, objectdef):
         formdata = self.build_formdata(objectdef)
 
         get_publisher().reset_formdata_state()
@@ -227,6 +325,8 @@ class TestDef(sql.TestDef):
         if previous_page:  # evaluate last page post conditions
             self.evaluate_page_conditions(previous_page, formdata, objectdef)
 
+        return formdata
+
     def fill_page_fields(self, fields, page, formdata, objectdef):
         self.handle_computed_fields(fields, formdata)
         for field in fields:
@@ -389,31 +489,38 @@ class TestDef(sql.TestDef):
                 if widget:
                     return widget
 
-    def export_to_json(self):
-        return {
-            'name': self.name,
-            'slug': self.slug,
-            'object_type': self.object_type,
-            'object_id': self.object_id,
-            'data': self.data,
-        }
+    def export_to_xml(self, include_id=False):
+        self._webservice_responses = self.get_webservice_responses()
+
+        testdef_xml = TestDefXmlProxy()
+        for field, dummy in TestDefXmlProxy.XML_NODES:  # pylint: disable=not-an-iterable
+            setattr(testdef_xml, field, getattr(self, field))
+
+        return testdef_xml.export_to_xml(include_id=include_id)
 
     @classmethod
-    def import_from_json(cls, data):
-        testdefs = TestDef.select(
-            [
-                Equal('object_type', data['object_type']),
-                Equal('object_id', data['object_id']),
-                Equal('slug', data['slug']),
-            ]
-        )
+    def import_from_xml(cls, fd, formdef, include_id=False):
+        try:
+            tree = ET.parse(fd)
+        except Exception:
+            raise ValueError
+        return cls.import_from_xml_tree(tree, formdef, include_id=include_id)
 
-        testdef = testdefs[0] if testdefs else TestDef()
+    @classmethod
+    def import_from_xml_tree(cls, tree, formdef, include_id=False):
+        testdef = TestDef.create_from_formdata(formdef, formdef.data_class()())
 
-        for k, v in data.items():
-            setattr(testdef, k, v)
+        testdef_xml = TestDefXmlProxy.import_from_xml_tree(tree, include_id)
+        for field, dummy in TestDefXmlProxy.XML_NODES:  # pylint: disable=not-an-iterable
+            if hasattr(testdef_xml, field):
+                setattr(testdef, field, getattr(testdef_xml, field))
 
         testdef.store()
+
+        for response in testdef._webservice_responses:
+            response.testdef_id = testdef.id
+            response.store()
+
         return testdef
 
 
@@ -431,3 +538,143 @@ class TestResult(sql.TestResult):
         base_url = get_publisher().get_backoffice_url()
         objects_dir = 'forms' if self.object_type == 'formdefs' else 'cards'
         return '%s/%s/%s/tests/results/%s/' % (base_url, objects_dir, self.object_id, self.id)
+
+
+class WebserviceResponseError(Exception):
+    pass
+
+
+class MockWebserviceResponseAdapter(requests.adapters.HTTPAdapter):
+    def __init__(self, testdef, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.testdef = testdef
+
+    def send(self, request, *args, **kwargs):
+        try:
+            return self._send(request, *args, **kwargs)
+        except WebserviceResponseError:
+            raise requests.exceptions.RequestError
+        except Exception as e:
+            # Webservice call can happen through templates which catch all exceptions.
+            # Record error to ensure we have a trace nonetheless.
+            get_publisher().record_error(
+                _('Unexpected error when mocking webservice call for url %(url)s: %(error)s.')
+                % {'url': request.url.split('?')[0], 'error': str(e)}
+            )
+            raise e
+
+    def _send(self, request, *args, **kwargs):
+        request_info = {
+            'url': request.url.split('?')[0],
+            'method': request.method,
+            'webservice_response_id': None,
+            'forbidden_method': False,
+        }
+        self.testdef.sent_requests.append(request_info)
+
+        for response in self.testdef.get_webservice_responses():
+            if response.is_configured() and response.match_request(request):
+                break
+        else:
+            if request.method != 'GET':
+                request_info['forbidden_method'] = True
+                raise WebserviceResponseError
+            return super().send(request, *args, **kwargs)
+
+        request_info['webservice_response_id'] = response.id
+
+        headers = {
+            'Content-Type': 'application/json',
+        }
+
+        raw_response = HTTPResponse(
+            status=200,
+            body=io.BytesIO(response.payload.encode()),
+            headers=headers,
+            original_response=self.make_original_response(headers),
+            preload_content=False,
+        )
+
+        return self.build_response(request, raw_response)
+
+    def make_original_response(self, headers):
+        dummy_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        original_response = http.client.HTTPResponse(sock=dummy_socket)
+
+        original_headers = http.client.HTTPMessage()
+        for k, v in headers.items():
+            original_headers.add_header(k, v)
+        original_response.msg = original_headers
+
+        return original_response
+
+
+class WebserviceResponse(XmlStorableObject):
+    _names = 'webservice-response'
+    xml_root_node = 'webservice-response'
+
+    testdef_id = None
+    name = ''
+    payload = None
+    url = None
+    qs_data = None
+    method = ''
+    post_data = None
+
+    XML_NODES = [
+        ('testdef_id', 'int'),
+        ('name', 'str'),
+        ('payload', 'str'),
+        ('url', 'str'),
+        ('qs_data', 'kv_data'),
+        ('method', 'str'),
+        ('post_data', 'kv_data'),
+    ]
+
+    def __str__(self):
+        return self.name
+
+    def is_configured(self):
+        return self.payload is not None and self.url
+
+    def match_request(self, request):
+        if request.url.split('?')[0] != self.url:
+            return False
+
+        if self.method and request.method != self.method:
+            return False
+
+        parsed_url = urllib.parse.urlparse(request.url)
+        query_string = urllib.parse.parse_qs(parsed_url.query)
+        for param, value in (self.qs_data or {}).items():
+            if value not in query_string.get(param, []):
+                return False
+
+        try:
+            request_data = json.loads(request.body)
+        except (TypeError, ValueError):
+            request_data = {}
+
+        for param, value in (self.post_data or {}).items():
+            if request_data.get(param) != value:
+                return False
+
+        return True
+
+    def export_kv_data_to_xml(self, element, attribute_name, **kwargs):
+        for key, value in getattr(self, attribute_name).items():
+            item = ET.SubElement(element, 'item')
+            ET.SubElement(item, 'name').text = key
+            ET.SubElement(item, 'value').text = value
+
+    def import_kv_data_from_xml(self, element, **kwargs):
+        if element is None:
+            return
+
+        data = {}
+        for item in element.findall('item'):
+            key = item.find('name').text
+            value = item.find('value').text or ''
+            data[key] = value
+
+        return data

wcs/urls.py

@@ -50,7 +50,6 @@ urlpatterns = [
         name='api-export-import-object-redirect',
     ),
     path('api/validate-condition', api.validate_condition, name='api-validate-condition'),
-    path('api/validate-expression', api.validate_expression, name='api-validate-expression'),
     path('api/reverse-geocoding', api.reverse_geocoding, name='api-reverse-geocoding'),
     path('api/geocoding', api.geocoding, name='api-geocoding'),
     path('api/statistics/', statistics_views.IndexView.as_view()),

wcs/wf/backoffice_fields.py

@@ -35,6 +35,9 @@ from ..qommon.form import (
 
 
 class SetBackofficeFieldRowWidget(CompositeWidget):
+    value_widget = ComputedExpressionWidget
+    value_placeholder = _('Leaving the field blank will empty the value.')
+
     def __init__(self, name, value=None, workflow=None, **kwargs):
         CompositeWidget.__init__(self, name, value, **kwargs)
         if not value:
@@ -67,11 +70,11 @@ class SetBackofficeFieldRowWidget(CompositeWidget):
             **kwargs,
         )
         self.add(
-            ComputedExpressionWidget,
+            self.value_widget,
             name='value',
             title=_('Value'),
             value=value.get('value'),
-            value_placeholder=_('Leaving the field blank will empty the value.'),
+            value_placeholder=self.value_placeholder,
         )
 
     def _parse(self, request):
@@ -83,11 +86,12 @@ class SetBackofficeFieldRowWidget(CompositeWidget):
 
 class SetBackofficeFieldsTableWidget(WidgetListAsTable):
     readonly = False
+    element_type = SetBackofficeFieldRowWidget
 
     def __init__(self, name, **kwargs):
         super().__init__(
             name,
-            element_type=SetBackofficeFieldRowWidget,
+            element_type=self.element_type,
             element_kwargs={'workflow': kwargs.pop('workflow')},
             **kwargs,
         )
@@ -243,5 +247,8 @@ class SetBackofficeFieldsWorkflowStatusItem(WorkflowStatusItem):
         if fields:
             self.fields = fields
 
+    def get_workflow_test_action(self, *args, **kwargs):
+        return self
+
 
 register_item_class(SetBackofficeFieldsWorkflowStatusItem)

wcs/wf/choice.py

@@ -216,5 +216,8 @@ class ChoiceWorkflowStatusItem(WorkflowStatusJumpItem):
         yield location, None, self.label
         yield location, None, self.confirmation_text
 
+    def get_workflow_test_action(self, *args, **kwargs):
+        return self
+
 
 register_item_class(ChoiceWorkflowStatusItem)

wcs/wf/create_formdata.py

@@ -15,10 +15,10 @@
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
 import collections
-import time
 import xml.etree.ElementTree as ET
 
 from django.utils.functional import cached_property
+from django.utils.timezone import localtime
 from quixote import get_publisher, get_request, get_session
 from quixote.html import TemplateIO, htmltext
 
@@ -707,7 +707,7 @@ class CreateFormdataWorkflowStatusItem(WorkflowStatusItem):
             return
 
         new_formdata = formdef.data_class()()
-        new_formdata.receipt_time = time.localtime()
+        new_formdata.receipt_time = localtime()
 
         self.assign_user(dest=new_formdata, src=formdata)
 
@@ -736,7 +736,7 @@ class CreateFormdataWorkflowStatusItem(WorkflowStatusItem):
 
         if self.draft:
             new_formdata.status = 'draft'
-            new_formdata.receipt_time = time.localtime()
+            new_formdata.receipt_time = localtime()
             new_formdata.store()
             if formdef.enable_tracking_codes:
                 code.formdata = new_formdata  # this will .store() the code

wcs/wf/edit_carddata.py

@@ -15,8 +15,8 @@
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
 import copy
-import time
 
+from django.utils.timezone import localtime
 from quixote import get_publisher
 
 from wcs.formdata import Evolution
@@ -59,7 +59,7 @@ class EditCarddataWorkflowStatusItem(CreateCarddataWorkflowStatusItem, ExternalW
             with get_publisher().substitutions.freeze():
                 last_evo = target_data.evolution[-1]
                 evo = Evolution(formdata=target_data)
-                evo.time = time.localtime()
+                evo.time = localtime()
                 evo.status = target_data.status
                 target_data.evolution.append(evo)
                 part = ContentSnapshotPart.take(formdata=target_data, old_data=old_data)

wcs/wf/jump.py

@@ -20,9 +20,8 @@ import itertools
 import json
 import math
 import os
-import time
 
-from django.utils.timezone import now
+from django.utils.timezone import localtime
 from quixote import get_publisher, get_request, get_response, redirect
 from quixote.directory import Directory
 from quixote.html import htmltext
@@ -57,7 +56,7 @@ class WorkflowTriggeredEvolutionPart(EvolutionPart):
         self.trigger_name = trigger_name
         self.content = content
         self.kind = kind
-        self.datetime = now()
+        self.datetime = localtime()
         self.trigger_name_key = misc.simplify(self.trigger_name, space='_')
 
 
@@ -324,7 +323,7 @@ class JumpWorkflowStatusItem(WorkflowStatusJumpItem):
                     return False
             last = formdata.last_update_time
             if last and timeout_seconds:
-                diff = time.time() - time.mktime(last)
+                diff = (localtime() - last).total_seconds()
                 if diff < timeout_seconds:
                     return False
 
@@ -334,6 +333,19 @@ class JumpWorkflowStatusItem(WorkflowStatusJumpItem):
 
         return True
 
+    def has_valid_timeout(self):
+        if not self.status or not self.timeout:
+            return False
+
+        if not self.get_target_status():
+            # this will catch status being a removed status
+            return False
+
+        return True
+
+    def get_workflow_test_action(self, *args, **kwargs):
+        return self
+
 
 register_item_class(JumpWorkflowStatusItem)
 
@@ -349,12 +361,9 @@ def workflows_with_timeout():
         for status in workflow.possible_status:
             status_str_id = 'wf-%s' % status.id
             for item in status.items:
-                if hasattr(item, 'status') and hasattr(item, 'timeout') and (item.status and item.timeout):
+                if hasattr(item, 'has_valid_timeout') and item.has_valid_timeout():
                     if workflow_id not in wfs_status:
                         wfs_status[workflow_id] = {}
-                    if not item.get_target_status():
-                        # this will catch status being a removed status
-                        continue
                     if status_str_id not in wfs_status[workflow_id]:
                         wfs_status[workflow_id][status_str_id] = []
                     wfs_status[workflow_id][status_str_id].append(item)
@@ -362,6 +371,20 @@ def workflows_with_timeout():
     return wfs_status
 
 
+def get_min_jumps_delay(jump_actions):
+    delay = math.inf
+    for jump_action in jump_actions:
+        if Template.is_template_string(jump_action.timeout):
+            delay = 0
+            break
+        delay = min(delay, int(jump_action.timeout))
+    # limit delay to minimal delay
+    if delay < JUMP_TIMEOUT_INTERVAL * 60:
+        delay = JUMP_TIMEOUT_INTERVAL * 60
+
+    return delay
+
+
 def _apply_timeouts(publisher, **kwargs):
     '''Traverse all filled form and apply expired timeout jumps if needed'''
     from ..carddef import CardDef
@@ -380,22 +403,14 @@ def _apply_timeouts(publisher, **kwargs):
             formdata_class = formdef.data_class()
             for status_id in status_ids:
                 # get minimum delay for jumps in this status
-                delay = math.inf
-                for jump_action in wfs_status[str(formdef.workflow_id)][status_id]:
-                    if Template.is_template_string(jump_action.timeout):
-                        delay = 0
-                        break
-                    delay = min(delay, int(jump_action.timeout))
-                # limit delay to minimal delay
-                if delay < JUMP_TIMEOUT_INTERVAL * 60:
-                    delay = JUMP_TIMEOUT_INTERVAL * 60
+                delay = get_min_jumps_delay(wfs_status[str(formdef.workflow_id)][status_id])
 
                 criterias = [
                     Equal('status', status_id),
                     Null('anonymised'),
                     LessOrEqual(
                         'last_update_time',
-                        (datetime.datetime.now() - datetime.timedelta(seconds=delay)).timetuple(),
+                        localtime() - datetime.timedelta(seconds=delay),
                     ),
                 ]
                 formdatas = formdata_class.select_iterator(criterias, ignore_errors=True, itersize=200)

wcs/wf/resubmit.py

@@ -14,8 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
-import time
-
+from django.utils.timezone import localtime
 from quixote import get_publisher, get_request, get_session
 
 from wcs.formdef import FormDef
@@ -82,7 +81,7 @@ class ResubmitWorkflowStatusItem(WorkflowStatusItem):
             formdef = FormDef.get_by_urlname(self.formdef_slug)
         new_formdata = formdef.data_class()()
         new_formdata.status = 'draft'
-        new_formdata.receipt_time = time.localtime()
+        new_formdata.receipt_time = localtime()
         new_formdata.user_id = formdata.user_id
         new_formdata.submission_context = (formdata.submission_context or {}).copy()
         new_formdata.submission_channel = formdata.submission_channel

wcs/wf/sendmail.py

@@ -401,33 +401,44 @@ class SendmailWorkflowStatusItem(WorkflowStatusItem):
 
         try:
             if len(addresses) > 1:
-                emails.email(
+                email = emails.get_email(
                     mail_subject,
                     mail_body,
                     email_rcpt=None,
                     bcc=addresses,
                     email_from=email_from,
                     attachments=attachments,
-                    fire_and_forget=True,
                 )
             else:
-                emails.email(
+                email = emails.get_email(
                     mail_subject,
                     mail_body,
                     email_rcpt=addresses,
                     email_from=email_from,
                     attachments=attachments,
-                    fire_and_forget=True,
                 )
+
+            if email:
+                self.send_email(email)
         except TooBigEmailError:
             get_publisher().record_error(_('Email too big to be sent'), formdata=formdata, status_item=self)
 
+    def send_email(self, email):
+        emails.send_email(email, fire_and_forget=True)
+
     def i18n_scan(self, base_location):
         location = '%sitems/%s/' % (base_location, self.id)
         if not self.mail_template:
             yield location, None, self.subject
             yield location, None, self.body
 
+    def get_workflow_test_action(self, formdata, *args, **kwargs):
+        def record_email(email):
+            formdata.sent_emails.append(email)
+
+        setattr(self, 'send_email', record_email)
+        return self
+
 
 register_item_class(SendmailWorkflowStatusItem)
 

wcs/workflow_tests.py

@@ -0,0 +1,423 @@
+# w.c.s. - web application for online forms
+# Copyright (C) 2005-2023  Entr'ouvert
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+import datetime
+import uuid
+
+from wcs import wf
+from wcs.qommon import _
+from wcs.qommon.form import SingleSelectWidget, StringWidget, WidgetList
+from wcs.qommon.humantime import humanduration2seconds, seconds2humanduration, timewords
+from wcs.qommon.xml_storage import XmlStorableObject
+from wcs.testdef import TestError
+from wcs.wf.backoffice_fields import SetBackofficeFieldRowWidget, SetBackofficeFieldsTableWidget
+from wcs.wf.profile import FieldNode
+
+
+class WorkflowTestError(TestError):
+    pass
+
+
+def get_test_action_options():
+    return [(x.key, x.label, x.key) for x in WorkflowTestAction.__subclasses__()]
+
+
+def get_test_action_class_by_type(action_type):
+    for action_class in WorkflowTestAction.__subclasses__():
+        if action_class.key == action_type:
+            return action_class
+
+    raise KeyError
+
+
+class WorkflowTests(XmlStorableObject):
+    _names = 'workflow_tests'
+    xml_root_node = 'workflow_tests'
+    testdef_id = None
+    actions = None
+
+    XML_NODES = [
+        ('testdef_id', 'int'),
+        ('actions', 'actions'),
+    ]
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.actions = []
+
+    def run(self, formdata, agent_user):
+        # mock methods so nothing is stored
+        formdata.record_workflow_event = lambda *args, **kwargs: None
+        formdata.record_workflow_action = lambda *args, **kwargs: None
+        formdata.store = lambda *args, **kwargs: None
+
+        # mark formdata as running workflow tests
+        formdata.workflow_test = True
+
+        formdata.frozen_receipt_time = formdata.receipt_time
+        formdata.sent_emails = []
+
+        formdata.perform_workflow()
+        for action in self.actions:
+            status = formdata.get_status()
+
+            try:
+                action.perform(formdata, agent_user)
+            except WorkflowTestError as e:
+                e.action_uuid = action.uuid
+                e.details.append(_('Form status when error occured: %s') % status.name)
+                raise e
+
+    def get_new_action_id(self):
+        if not self.actions:
+            return '1'
+
+        return str(int(max(x.id for x in self.actions)) + 1)
+
+    def add_action(self, action_class):
+        self.actions.append(action_class(id=self.get_new_action_id()))
+
+    def export_actions_to_xml(self, element, attribute_name, **kwargs):
+        for action in self.actions:
+            element.append(action.export_to_xml())
+
+    def import_actions_from_xml(self, element, **kwargs):
+        actions = []
+        for sub in element.findall('test-action'):
+            key = sub.findtext('key')
+
+            try:
+                klass = get_test_action_class_by_type(key)
+            except KeyError:
+                continue
+
+            actions.append(klass.import_from_xml_tree(sub))
+
+        return actions
+
+
+class WorkflowTestAction(XmlStorableObject):
+    xml_root_node = 'test-action'
+    _names = 'test-action'
+    uuid = None
+
+    optional_fields = []
+
+    XML_NODES = [
+        ('id', 'str'),
+        ('uuid', 'str'),
+        ('key', 'str'),
+    ]
+
+    def __init__(self, **kwargs):
+        self.uuid = str(uuid.uuid4())
+
+        allowed_key = {x[0] for x in self.XML_NODES}
+        for k, v in kwargs.items():
+            if k in allowed_key:
+                setattr(self, k, v)
+
+    def __str__(self):
+        return str(self.label)
+
+    def render_as_line(self):
+        for field, dummy in self.XML_NODES:
+            if field not in self.optional_fields and not getattr(self, field):
+                return 'not configured'
+
+        return self.details_label
+
+
+class ButtonClick(WorkflowTestAction):
+    label = _('Simulate click on action button')
+
+    key = 'button-click'
+    button_name = None
+
+    XML_NODES = WorkflowTestAction.XML_NODES + [
+        ('button_name', 'str'),
+    ]
+
+    @property
+    def details_label(self):
+        return 'Click on "%s"' % self.button_name
+
+    def perform(self, formdata, user):
+        status = formdata.get_status()
+        form = status.get_action_form(formdata, user)
+        if not form or not any(
+            button_widget := x for x in form.submit_widgets if x.label == self.button_name
+        ):
+            raise WorkflowTestError(_('Button "%s" is not displayed.') % self.button_name)
+
+        form.get_submit = lambda: button_widget.name
+        status.handle_form(form, formdata, user, check_replay=False)
+
+    def fill_admin_form(self, form, formdef):
+        possible_button_names = set()
+        for item in formdef.workflow.get_all_items():
+            if isinstance(item, wf.choice.ChoiceWorkflowStatusItem) and item.status:
+                possible_button_names.add(item.label)
+
+        possible_button_names = sorted(possible_button_names)
+
+        value = self.button_name
+        if value and value not in possible_button_names:
+            value = '%s (%s)' % (value, _('not available'))
+            possible_button_names.append(value)
+
+        form.add(
+            SingleSelectWidget,
+            'button_name',
+            title=_('Button name'),
+            options=possible_button_names,
+            required=True,
+            value=value,
+        )
+
+
+class AssertStatus(WorkflowTestAction):
+    label = _('Assert form status')
+
+    key = 'assert-status'
+    status_name = None
+
+    XML_NODES = WorkflowTestAction.XML_NODES + [
+        ('status_name', 'str'),
+    ]
+
+    @property
+    def details_label(self):
+        return 'Status is "%s"' % self.status_name
+
+    def perform(self, formdata, user):
+        status = formdata.get_status()
+        if status.name != self.status_name:
+            raise WorkflowTestError(
+                _('Form should be in status "%(expected_status)s" but is in status "%(status)s".')
+                % {'expected_status': self.status_name, 'status': status.name}
+            )
+
+    def fill_admin_form(self, form, formdef):
+        possible_statuses = [x.name for x in formdef.workflow.possible_status]
+
+        value = self.status_name
+        if value and value not in possible_statuses:
+            value = '%s (%s)' % (value, _('not available'))
+            possible_statuses.append(value)
+
+        form.add(
+            SingleSelectWidget,
+            'status_name',
+            title=_('Status name'),
+            options=possible_statuses,
+            required=True,
+            value=self.status_name,
+        )
+
+
+class AssertEmail(WorkflowTestAction):
+    label = _('Assert email is sent')
+
+    key = 'assert-email'
+    subject_strings = None
+    body_strings = None
+
+    optional_fields = ['subject_strings', 'body_strings']
+
+    XML_NODES = WorkflowTestAction.XML_NODES + [
+        ('subject_strings', 'str_list'),
+        ('body_strings', 'str_list'),
+    ]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.subject_strings = self.subject_strings or []
+        self.body_strings = self.body_strings or []
+
+    @property
+    def details_label(self):
+        return ''
+
+    def perform(self, formdata, user):
+        try:
+            email = formdata.sent_emails.pop(0)
+        except IndexError:
+            raise WorkflowTestError(_('No email was sent.'))
+
+        for subject in self.subject_strings:
+            details = [_('Email subject: %s') % email.email_msg.subject]
+            if subject not in email.email_msg.subject:
+                raise WorkflowTestError(_('Email subject does not contain "%s".') % subject, details=details)
+
+        for body in self.body_strings:
+            details = [_('Email body: %s') % email.email_msg.body]
+            if body not in email.email_msg.body:
+                raise WorkflowTestError(_('Email body does not contain "%s".') % body, details=details)
+
+    def fill_admin_form(self, form, formdef):
+        form.add(
+            WidgetList,
+            'subject_strings',
+            element_type=StringWidget,
+            title=_('Subject must contain'),
+            value=self.subject_strings,
+            add_element_label=_('Add string'),
+            element_kwargs={'render_br': False, 'size': 50},
+        )
+        form.add(
+            WidgetList,
+            'body_strings',
+            element_type=StringWidget,
+            title=_('Body must contain'),
+            value=self.body_strings,
+            add_element_label=_('Add string'),
+            element_kwargs={'render_br': False, 'size': 50},
+        )
+
+
+class SkipTime(WorkflowTestAction):
+    label = _('Move forward in time')
+
+    key = 'skip-time'
+    seconds = None
+
+    XML_NODES = WorkflowTestAction.XML_NODES + [
+        ('seconds', 'int'),
+    ]
+
+    @property
+    def details_label(self):
+        return seconds2humanduration(self.seconds)
+
+    def rewind(self, formdata):
+        def rewind_time(timestamp):
+            return timestamp - datetime.timedelta(seconds=self.seconds)
+
+        formdata.receipt_time = rewind_time(formdata.receipt_time)
+        formdata.evolution[-1].time = rewind_time(formdata.evolution[-1].time)
+
+    def perform(self, formdata, user):
+        self.rewind(formdata)
+
+        jump_actions = []
+        status = formdata.get_status()
+        for item in status.items:
+            if hasattr(item, 'has_valid_timeout') and item.has_valid_timeout():
+                jump_actions.append(item)
+
+        delay = wf.jump.get_min_jumps_delay(status.items)
+
+        if formdata.last_update_time > formdata.frozen_receipt_time - datetime.timedelta(seconds=delay):
+            return
+
+        for jump_action in jump_actions:
+            if jump_action.check_condition(formdata):
+                wf.jump.jump_and_perform(formdata, jump_action)
+                break
+
+    def fill_admin_form(self, form, formdef):
+        form.add(
+            StringWidget,
+            'seconds',
+            title=_('Value'),
+            value=seconds2humanduration(self.seconds),
+            hint=_('ex.: 1 day 12 hours. Usable units of time: %(variables)s.')
+            % {'variables': ','.join(timewords())},
+        )
+
+    def seconds_parse(self, value):
+        if not value:
+            return value
+        try:
+            return humanduration2seconds(value)
+        except ValueError:
+            return None
+
+
+class AssertBackofficeFieldRowWidget(SetBackofficeFieldRowWidget):
+    value_widget = StringWidget
+    value_placeholder = None
+
+
+class AssertBackofficeFieldsTableWidget(SetBackofficeFieldsTableWidget):
+    element_type = AssertBackofficeFieldRowWidget
+
+
+class AssertBackofficeFieldValues(WorkflowTestAction):
+    label = _('Assert backoffice field values')
+
+    key = 'assert-backoffice-field'
+    fields = []
+
+    XML_NODES = WorkflowTestAction.XML_NODES + [
+        ('fields', 'fields'),
+    ]
+
+    @property
+    def details_label(self):
+        return ''
+
+    def perform(self, formdata, user):
+        for field_dict in self.fields:
+            field_id = field_dict['field_id']
+            expected_value = field_dict['value']
+            formdata_value = formdata.data.get(field_id)
+
+            if formdata_value != expected_value:
+                fields = [x for x in formdata.formdef.workflow.get_backoffice_fields() if x.id == field_id]
+                if not fields:
+                    raise WorkflowTestError(
+                        _('Field %(field_id)s not found (expected value "%(value)s").')
+                        % {
+                            'field_id': field_id,
+                            'value': expected_value,
+                        }
+                    )
+
+                field = fields[0]
+                raise WorkflowTestError(
+                    _(
+                        'Wrong value for backoffice field "%(field)s" (expected "%(expected_value)s", got "%(value)s").'
+                    )
+                    % {
+                        'field': field.label,
+                        'value': formdata_value,
+                        'expected_value': expected_value,
+                    }
+                )
+
+    def fill_admin_form(self, form, formdef):
+        form.add(
+            AssertBackofficeFieldsTableWidget,
+            'fields',
+            value_widget_class=StringWidget,
+            value=self.fields,
+            workflow=formdef.workflow,
+        )
+
+    def export_fields_to_xml(self, element, attribute_name, **kwargs):
+        for field in self.fields:
+            element.append(FieldNode(field).export_to_xml(include_id=True))
+
+    def import_fields_from_xml(self, element, **kwargs):
+        fields = []
+        for field_xml_node in element.findall('field'):
+            field_node = FieldNode()
+            field_node.init_with_xml(field_xml_node, include_id=True, snapshot=None)
+            fields.append(field_node.as_dict())
+
+        return fields

wcs/workflows.py

@@ -84,7 +84,9 @@ def perform_items(items, formdata, depth=20, user=None, global_action=False):
     old_status = formdata.status
     wf_old_status = formdata.get_status()
     had_jump = False
-    loop_items = wf_old_status.get_loop_items(formdata=formdata)
+    loop_items = None
+    if wf_old_status:
+        loop_items = wf_old_status.get_loop_items(formdata=formdata)
     do_break = False
     with get_publisher().substitutions.freeze():
         if loop_items is not None:
@@ -98,6 +100,10 @@ def perform_items(items, formdata, depth=20, user=None, global_action=False):
                     wf_old_status.get_status_loop(index=i, items=loop_items, item=loop_item)
                 )
             for item in items or []:
+                if getattr(formdata, 'workflow_test', False):
+                    item = item.get_workflow_test_action(formdata)
+                    if not item:
+                        continue
                 if getattr(item.perform, 'noop', False):
                     continue
                 if not item.check_condition(formdata):
@@ -118,7 +124,9 @@ def perform_items(items, formdata, depth=20, user=None, global_action=False):
         if loop_items is not None:
             formdata.record_workflow_event('loop-end')
 
-    loop_target_status = wf_old_status.get_loop_target_status(formdata=formdata)
+    loop_target_status = None
+    if wf_old_status:
+        loop_target_status = wf_old_status.get_loop_target_status(formdata=formdata)
     if not do_break and loop_target_status:
         formdata.status = 'wf-%s' % loop_target_status.id
 
@@ -130,7 +138,7 @@ def perform_items(items, formdata, depth=20, user=None, global_action=False):
         evo = Evolution(formdata)
         if global_action:
             evo.set_user(formdata=formdata, user=user)
-        evo.time = time.localtime()
+        evo.time = localtime()
         evo.status = formdata.status
         formdata.evolution.append(evo)
         formdata.store()
@@ -2367,7 +2375,7 @@ class SerieOfActionsMixin:
     def get_action_form(self, filled, user, displayed_fields=None):
         form = Form(enctype='multipart/form-data', use_tokens=False)
         form.attrs['id'] = 'wf-actions'
-        form.add_hidden('_ts', str(time.mktime(filled.last_update_time)))
+        form.add_hidden('_ts', str(filled.last_update_time.timestamp()))
         for item in self.items:
             if not item.check_auth(filled, user):
                 continue
@@ -2412,10 +2420,10 @@ class SerieOfActionsMixin:
                 messages.append(message)
         return messages
 
-    def handle_form(self, form, filled, user, evo):
-        if form.get('_ts') != str(time.mktime(filled.last_update_time)):
+    def handle_form(self, form, filled, user, evo, check_replay=True):
+        if check_replay and form.get('_ts') != str(filled.last_update_time.timestamp()):
             raise ReplayException()
-        evo.time = time.localtime()
+        evo.time = localtime()
         evo.set_user(formdata=filled, user=user, check_submitter=get_request().is_in_frontoffice())
         if not filled.evolution:
             filled.evolution = []
@@ -2480,9 +2488,9 @@ class WorkflowGlobalAction(SerieOfActionsMixin):
         else:
             return '/actions/%s/#' % token.id
 
-    def handle_form(self, form, filled, user):
+    def handle_form(self, form, filled, user, check_replay=True):
         evo = Evolution(filled)
-        url = super().handle_form(form, filled, user, evo)
+        url = super().handle_form(form, filled, user, evo, check_replay=check_replay)
         if isinstance(url, str):
             return url
         filled.evolution.append(evo)
@@ -2683,7 +2691,7 @@ class WorkflowStatus(SerieOfActionsMixin):
         for item in self.get_active_items(form, filled, user):
             item.evaluate_live_form(form, filled, user)
 
-    def handle_form(self, form, filled, user):
+    def handle_form(self, form, filled, user, check_replay=True):
         # check for global actions
         for action in filled.formdef.workflow.get_global_actions_for_user(filled, user):
             if 'button-action-%s' % action.id in get_request().form:
@@ -2699,7 +2707,7 @@ class WorkflowStatus(SerieOfActionsMixin):
             filled.record_workflow_event('button', action_item_id=button.action_id)
 
         evo = Evolution(filled)
-        url = super().handle_form(form, filled, user, evo)
+        url = super().handle_form(form, filled, user, evo, check_replay=check_replay)
         if isinstance(url, str):
             return url
         if form.has_errors():
@@ -3543,6 +3551,10 @@ class WorkflowStatusItem(XmlSerialisable):
             markers_stack.append({'status_id': formdata.status[3:]})
             formdata.update_workflow_data({'_markers_stack': markers_stack})
 
+    def get_workflow_test_action(self, *args, **kwargs):
+        # get action to be used in workflow tests, None for skipping the action
+        return None
+
     def __repr__(self):
         parent = getattr(self, 'parent', None)  # status or global action
         parts = [self.__class__.__name__, str(self.id)]

wcs/wscalls.py

@@ -115,14 +115,17 @@ def call_webservice(
         qs = list(urllib.parse.parse_qsl(parsed.query))
         for key, value in qs_data.items():
             try:
-                value = WorkflowStatusItem.compute(value, raises=True)
-                value = str(value) if value is not None else ''
+                value = WorkflowStatusItem.compute(value, allow_complex=True, raises=True)
             except Exception as e:
                 get_publisher().record_error(exception=e, notify=True)
             else:
-                key = force_str(key)
-                value = force_str(value)
-                qs.append((key, value))
+                if value:
+                    value = get_publisher().get_cached_complex_data(value)
+                if isinstance(value, (tuple, list, set)):
+                    qs.extend((key, x) for x in value)
+                else:
+                    value = str(value) if value is not None else ''
+                    qs.append((key, value))
         qs = urllib.parse.urlencode(qs)
         url = urllib.parse.urlunparse(parsed[:4] + (qs,) + parsed[5:6])