backoffice: display drafts stats (#72542)

debian/control

@@ -43,7 +43,8 @@ Depends: graphviz,
          uwsgi-plugin-python3,
          ${misc:Depends},
          ${python3:Depends},
-Recommends: libreoffice-writer-nogui | libreoffice-writer,
+Recommends: graphicsmagick,
+            libreoffice-writer-nogui | libreoffice-writer,
             poppler-utils,
             python3-docutils,
             python3-langdetect,

debian/rules

@@ -8,8 +8,6 @@ export PYBUILD_NAME=wcs
 
 override_dh_install:
 	dh_install
-	mv $(CURDIR)/debian/wcs/usr/bin/wcsctl.py \
-		$(CURDIR)/debian/wcs/usr/bin/wcsctl
 	mv $(CURDIR)/debian/wcs/usr/bin/manage.py \
 		$(CURDIR)/debian/wcs/usr/lib/wcs/
 	install -d $(CURDIR)/debian/wcs/etc/wcs

debian/wcs.init

@@ -17,7 +17,6 @@ set -e
 PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
 DESC="Web Forms Manager"
 NAME=wcs
-WCSCTL=/usr/bin/wcsctl
 DAEMON=/usr/bin/uwsgi
 RUN_DIR=/run/$NAME
 PIDFILE=$RUN_DIR/$NAME.pid

setup.py

@@ -208,7 +208,7 @@ setup(
     package_dir={'wcs': 'wcs'},
     packages=find_packages(),
     cmdclass=cmdclass,
-    scripts=['wcsctl.py', 'manage.py'],
+    scripts=['manage.py'],
     include_package_data=True,
     data_files=data_tree('share/wcs/web/', 'data/web/')
     + data_tree('share/wcs/themes/', 'data/themes/')

tests/admin_pages/test_deprecations.py

@@ -183,6 +183,9 @@ def test_deprecations(pub):
     data_source = NamedDataSource(name='ds_jsonp')
     data_source.data_source = {'type': 'jsonp', 'value': 'xxx'}
     data_source.store()
+    data_source = NamedDataSource(name='ds_csv')
+    data_source.data_source = {'type': 'json', 'value': 'http://example.net/csvdatasource/plop/test'}
+    data_source.store()
 
     NamedWsCall.wipe()
     wscall = NamedWsCall()
@@ -190,6 +193,16 @@ def test_deprecations(pub):
     wscall.request = {'url': 'http://example.net', 'qs_data': {'a': '=1+2'}}
     wscall.store()
 
+    wscall = NamedWsCall()
+    wscall.name = 'Hello CSV'
+    wscall.request = {'url': 'http://example.net/csvdatasource/plop/test'}
+    wscall.store()
+
+    wscall = NamedWsCall()
+    wscall.name = 'Hello json data store'
+    wscall.request = {'url': 'http://example.net/jsondatastore/plop'}
+    wscall.store()
+
     MailTemplate.wipe()
     mail_template1 = MailTemplate()
     mail_template1.name = 'Hello1'
@@ -258,6 +271,13 @@ def test_deprecations(pub):
     assert [x.text for x in resp.pyquery('.section--actions li a')] == [
         'test / Daily Summary Email',
     ]
+    assert [x.text for x in resp.pyquery('.section--csv-connector li a')] == [
+        'Data source "ds_csv"',
+        'Webservice "Hello CSV"',
+    ]
+    assert [x.text for x in resp.pyquery('.section--json-data-store li a')] == [
+        'Webservice "Hello json data store"',
+    ]
     # check all links are ok
     for link in resp.pyquery('.section li a'):
         resp.click(href=link.attrib['href'], index=0)

tests/admin_pages/test_form.py

@@ -4608,6 +4608,13 @@ def test_admin_form_inspect(pub):
 
     assert '>Custom views</button>' not in resp
 
+    # check all field links
+    for href in [x.attrib['href'] for x in resp.pyquery('.inspect-field h4 a')]:
+        app.get(href)
+
+    # check field links targets per-page URL
+    assert '/pages/' in resp.pyquery('.inspect-field h4 a')[0].attrib['href']
+
     custom_view_owner = pub.custom_view_class()
     custom_view_owner.title = 'card view owner'
     custom_view_owner.formdef = formdef
@@ -4636,6 +4643,22 @@ def test_admin_form_inspect(pub):
     assert '<h4>card view any</h4>' in resp
     assert '<h4>card view datasource</h4>' in resp
 
+    # check with a form without pages
+    formdef.fields = [
+        fields.StringField(
+            id='1', label='String field', varname='var_1', condition={'type': 'django', 'value': 'true'}
+        ),
+    ]
+    formdef.store()
+    resp = app.get('/backoffice/forms/%s/inspect' % formdef.id)
+
+    # check all field links
+    for href in [x.attrib['href'] for x in resp.pyquery('.inspect-field h4 a')]:
+        app.get(href)
+
+    # check field links targets per-page URL
+    assert '/pages/' not in resp.pyquery('.inspect-field h4 a')[0].attrib['href']
+
 
 def test_admin_form_inspect_validation(pub):
     create_superuser(pub)

tests/admin_pages/test_settings.py

@@ -1084,17 +1084,31 @@ def test_i18n(pub):
 def test_submission_channels(pub):
     create_superuser(pub)
     app = login(get_app(pub))
-    resp = app.get('/backoffice/settings/submission-channels')
+    resp = app.get('/backoffice/settings/backoffice-submission')
     resp.form['include-in-global-listing'].checked = True
     resp = resp.form.submit('submit')
 
     pub.reload_cfg()
     assert pub.cfg['submission-channels']['include-in-global-listing'] is True
 
-    resp = app.get('/backoffice/settings/submission-channels')
+    resp = app.get('/backoffice/settings/backoffice-submission')
     assert resp.form['include-in-global-listing'].checked
 
 
+def test_backoffice_submission(pub):
+    create_superuser(pub)
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/settings/backoffice-submission')
+    resp.form['redirect'] = 'https://example.net'
+    resp = resp.form.submit('submit')
+
+    pub.reload_cfg()
+    assert pub.cfg['backoffice-submission']['redirect'] == 'https://example.net'
+
+    resp = app.get('/backoffice/settings/backoffice-submission')
+    assert resp.form['redirect'].value == 'https://example.net'
+
+
 def test_hobo_locked_settings(pub):
     create_superuser(pub)
     app = login(get_app(pub))

tests/admin_pages/test_tests.py

@@ -14,6 +14,7 @@ from wcs.qommon.http_request import HTTPRequest
 from wcs.qommon.upload_storage import PicklableUpload
 from wcs.testdef import TestDef, TestResult, WebserviceResponse
 from wcs.workflow_tests import WorkflowTests
+from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef
 from wcs.wscalls import NamedWsCall
 
 from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
@@ -95,6 +96,25 @@ def test_tests_page(pub):
     app.get('/backoffice/forms/1/tests/results/run').follow()
 
 
+def test_tests_page_breadcrumb(pub):
+    create_superuser(pub)
+
+    formdef = FormDef()
+    formdef.name = 'Short title'
+    formdef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get(formdef.get_admin_url() + 'tests/')
+    assert 'Short title' in resp.text
+
+    formdef.name = 'This is a long title'
+    formdef.store()
+
+    resp = app.get(formdef.get_admin_url() + 'tests/')
+    assert 'This is a long…' in resp.text
+
+
 def test_tests_page_creation_from_formdata(pub):
     user = create_superuser(pub)
 
@@ -930,12 +950,153 @@ def test_tests_result_sent_requests(pub, http_requests):
     assert 'Sent requests:' in resp.text
     assert 'POST http://remote.example.net/json' in resp.text
     assert 'Request was blocked since it is not a GET request.' in resp.text
-    assert 'Recorded errors:' not in resp.text
+    assert 'Recorded errors:' in resp.text
+    assert 'error in HTTP request to remote.example.net (method must be GET)' in resp.text
 
     resp = resp.click('You can create corresponding webservice response here.')
     assert 'Webservice responses' in resp.text
 
 
+def test_tests_result_error_field(pub):
+    create_superuser(pub)
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.fields = [
+        fields.StringField(id='0', label='Text Field', varname='text', validation={'type': 'digits'}),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.data['0'] = 'not-digits'
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.name = 'First test'
+    testdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/1/tests/results/run')
+    result_url = resp.location
+    resp = resp.follow()
+
+    assert escape('Invalid value "not-digits" for field "Text Field"') in resp.text
+
+    resp = resp.click('Display details')
+
+    assert 'Field linked to error:' in resp.text
+    assert 'deleted' not in resp.text
+
+    resp = resp.click('Text Field')
+
+    assert resp.pyquery('h2').text() == 'Text Field'
+
+    formdef.fields = []
+    formdef.store()
+
+    resp = app.get(result_url)
+    resp = resp.click('Display details')
+
+    assert 'Text Field' not in resp.text
+    assert 'deleted' in resp.text
+
+
+def test_tests_result_inspect(pub):
+    user = create_superuser(pub)
+
+    role = pub.role_class(name='test role')
+    role.store()
+    user.roles = [role.id]
+    user.store()
+
+    workflow = Workflow(name='Workflow One')
+    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
+    workflow.backoffice_fields_formdef.fields = [
+        fields.StringField(id='1', label='Text BO', varname='text_bo'),
+    ]
+
+    new_status = workflow.add_status(name='New status')
+    set_backoffice_fields = new_status.add_action('set-backoffice-fields')
+    set_backoffice_fields.fields = [{'field_id': '1', 'value': 'goodbye'}]
+
+    jump = new_status.add_action('choice')
+    jump.label = 'Loop on status'
+    jump.status = new_status.id
+    jump.by = [role.id]
+
+    wscall = new_status.add_action('webservice_call')
+    wscall.url = 'http://example.com/json'
+    wscall.varname = 'test_webservice'
+    wscall.qs_data = {'a': 'b'}
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.fields = [
+        fields.StringField(id='0', label='Text Field', varname='text'),
+    ]
+    formdef.workflow = workflow
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.data['0'] = 'hello'
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.name = 'First test'
+    testdef.agent_id = user.id
+    testdef.workflow_tests.actions = [
+        workflow_tests.ButtonClick(id='1', button_name='Loop on status'),
+    ]
+    testdef.store()
+
+    response = WebserviceResponse()
+    response.testdef_id = testdef.id
+    response.name = 'Fake response'
+    response.url = 'http://example.com/json'
+    response.payload = '{"foo": "bar"}'
+    response.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/1/tests/results/run')
+    result_url = resp.location
+    resp = resp.follow()
+    resp = resp.click('Display inspect')
+
+    assert 'form_var_text' in resp.text
+    assert 'form_var_text_bo' in resp.text
+    assert 'form_workflow_data_test_webservice_response_foo' in resp.text
+
+    assert [x.text_content() for x in resp.pyquery('div#inspect-timeline a')] == [
+        'New status',
+        'Backoffice Data',
+        'Webservice',
+        'Action button - Manual Jump Loop on status',
+        'Backoffice Data',
+        'Webservice',
+    ]
+
+    resp.form['django-condition'] = 'form_var_text == "hello"'
+    resp = resp.form.submit()
+    assert 'Condition result' in resp.text
+    assert 'result-true' in resp.text
+
+    resp.form['django-condition'] = 'form_var_text_bo == "goodbye"'
+    resp = resp.form.submit()
+    assert 'Condition result' in resp.text
+    assert 'result-true' in resp.text
+
+    # check inspect is not accessible for old results
+    light_test_result = TestResult.select()[-1]
+    test_result = TestResult.get(light_test_result.id)
+    del test_result.results[0]['formdata']
+    test_result.store()
+
+    resp = app.get(result_url)
+    assert 'Display inspect' not in resp.text
+
+
 def test_tests_run_order(pub):
     create_superuser(pub)
 
@@ -1197,18 +1358,25 @@ def test_tests_webservice_response(pub):
     resp.form['name'] = 'Test response'
     resp = resp.form.submit().follow()
 
-    resp.form['url'] = 'http://example.com/'
-    resp = resp.form.submit('submit').follow()
+    resp = resp.form.submit('submit')
+    assert resp.pyquery('.error').text() == 'required field required field'
 
+    resp = app.get('/backoffice/forms/1/tests/%s/webservice-responses/' % testdef.id)
+    assert 'Test response' in resp.text
     assert 'There are no webservice responses yet.' not in resp.text
+    assert '(not configured)' in resp.text
 
     resp = resp.click('Test response')
+    resp.form['url'] = 'http://example.com/'
     resp.form['payload'] = '{"a": "b"}'
     resp.form['qs_data$element0key'] = 'foo'
     resp.form['method'] = 'POST (JSON)'
     resp.form['post_data$element0key'] = 'bar'
     resp = resp.form.submit('submit').follow()
 
+    assert 'Test response' in resp.text
+    assert '(not configured)' not in resp.text
+
     response = testdef.get_webservice_responses()[0]
     assert response.name == 'Test response'
     assert response.url == 'http://example.com/'
@@ -1232,15 +1400,14 @@ def test_tests_webservice_response(pub):
 
     assert 'Test response (copy)' not in resp.text
 
-    resp = resp.click('Test response')
-    resp.form['payload'] = ''
-    resp = resp.form.submit('submit').follow()
-
-    assert 'Test response' in resp.text
-    assert '(not configured)' in resp.text
-
     resp = resp.click('Test response')
     resp.form['payload'] = '{"a"}'
     resp = resp.form.submit()
 
     assert "Invalid JSON: Expecting ':' delimiter: line 1 column 5 (char 4)" in resp.text
+
+    resp.form['url'] = 'xxx'
+    resp.form['payload'] = '{}'
+    resp = resp.form.submit()
+
+    assert 'must start with http://' in resp.text

tests/admin_pages/test_workflow.py

@@ -3789,8 +3789,16 @@ def test_workflows_wscall_options(pub, value):
     baz_status.add_action('webservice_call')
     workflow.store()
 
+    pub.cfg['debug'] = {}
+    pub.write_cfg()
+
     app = login(get_app(pub))
     resp = app.get('/backoffice/workflows/%s/status/%s/items/1/' % (workflow.id, baz_status.id))
+    assert 'notify_on_errors' not in resp.form.fields
+
+    pub.cfg['debug'] = {'error_email': 'test@localhost'}
+    pub.write_cfg()
+    resp = app.get('/backoffice/workflows/%s/status/%s/items/1/' % (workflow.id, baz_status.id))
     assert resp.form['notify_on_errors'].value is None
     assert resp.form['record_on_errors'].value == 'yes'
     resp.form['notify_on_errors'] = value

tests/admin_pages/test_workflow_tests.py

@@ -6,7 +6,7 @@ from django.utils.html import escape
 from wcs import workflow_tests
 from wcs.formdef import FormDef, fields
 from wcs.qommon.http_request import HTTPRequest
-from wcs.testdef import TestDef
+from wcs.testdef import TestDef, WebserviceResponse
 from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef
 
 from ..utilities import create_temporary_pub, get_app, login
@@ -144,6 +144,15 @@ def test_workflow_tests_edit_actions(pub):
     assert 'There are no workflow test actions yet.' in resp.text
     assert len(resp.pyquery('.biglist li')) == 0
 
+    option_labels = [x[2] for x in resp.form['type'].options]
+    assert (
+        option_labels.index('Assert email is sent')
+        < option_labels.index('Assert form status')
+        < option_labels.index('—')
+        < option_labels.index('Move forward in time')
+        < option_labels.index('Simulate click on action button')
+    )
+
     # add workflow test action through sidebar form
     resp.form['type'] = 'button-click'
     resp = resp.form.submit().follow()
@@ -189,18 +198,6 @@ def test_workflow_tests_action_button_click(pub):
 
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')
-
-    jump = new_status.add_action('choice')
-    jump.label = 'Button 1'
-    jump.status = new_status.id
-
-    jump = new_status.add_action('choice')
-    jump.label = 'Button 2'
-    jump.status = new_status.id
-
-    jump = new_status.add_action('choice')
-    jump.label = 'Button no target status'
-
     workflow.store()
 
     formdef = FormDef()
@@ -219,7 +216,22 @@ def test_workflow_tests_action_button_click(pub):
 
     app = login(get_app(pub))
     resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
+    assert 'Workflow has no action that displays a button.' in resp.text
 
+    jump = new_status.add_action('choice')
+    jump.label = 'Button 1'
+    jump.status = new_status.id
+
+    jump = new_status.add_action('choice')
+    jump.label = 'Button 2'
+    jump.status = new_status.id
+
+    jump = new_status.add_action('choice')
+    jump.label = 'Button no target status'
+
+    workflow.store()
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
     assert resp.form['button_name'].options == [
         ('Button 1', False, 'Button 1'),
         ('Button 2', False, 'Button 2'),
@@ -372,6 +384,70 @@ def test_workflow_tests_action_assert_backoffice_field(pub):
     assert resp.form['fields$element0$value'].value == 'xxx'
 
 
+def test_workflow_tests_action_assert_webservice_call(pub):
+    create_superuser(pub)
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertWebserviceCall(id='1'),
+    ]
+    testdef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
+    assert 'you must define corresponding webservice response' in resp.text
+
+    resp = resp.click('Add webservice response')
+    assert 'There are no webservice responses yet.' in resp.text
+
+    response = WebserviceResponse()
+    response.testdef_id = testdef.id
+    response.name = 'Fake response'
+    response.store()
+
+    response2 = WebserviceResponse()
+    response2.testdef_id = testdef.id
+    response2.name = 'Fake response 2'
+    response2.store()
+
+    response3 = WebserviceResponse()
+    response3.testdef_id = testdef.id + 1
+    response3.name = 'Other response'
+    response3.store()
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
+    assert resp.form['webservice_response_id'].options == [
+        ('1', False, 'Fake response'),
+        ('2', False, 'Fake response 2'),
+    ]
+    assert resp.form['call_count'].value == '1'
+
+    resp.form['webservice_response_id'] = 1
+    resp.form['call_count'] = 2
+    resp = resp.form.submit().follow()
+
+    assert 'Fake response' in resp.text
+    assert 'Broken' not in resp.text
+
+    assert_webservice_call = TestDef.get(testdef.id).workflow_tests.actions[0]
+    assert assert_webservice_call.webservice_response_id == '1'
+    assert assert_webservice_call.call_count == 2
+
+    response.remove_self()
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
+
+    assert 'Broken, missing webservice response' in resp.text
+    assert 'Fake response' not in resp.text
+
+
 def test_workflow_tests_actions_reorder(pub):
     create_superuser(pub)
 

tests/api/test_carddata.py

@@ -367,7 +367,7 @@ def test_api_list_formdata_phone_order_by_rank(pub):
     CardDef.wipe()
     carddef = CardDef()
     carddef.name = 'test'
-    carddef.workflow_roles = {'_receiver': role.id}
+    carddef.workflow_roles = {'_viewer': role.id}
     carddef.fields = [
         fields.StringField(id='0', label='a', display_locations=['listings']),
         fields.StringField(id='1', label='b'),
@@ -466,7 +466,7 @@ def test_api_card_list_custom_id_filter_identifier(pub):
     CardDef.wipe()
     carddef = CardDef()
     carddef.name = 'foo'
-    carddef.workflow_roles = {'_receiver': role.id}
+    carddef.workflow_roles = {'_viewer': role.id}
     carddef.fields = [
         fields.StringField(id='1', label='Test', varname='foo'),
     ]
@@ -484,9 +484,18 @@ def test_api_card_list_custom_id_filter_identifier(pub):
             '/api/cards/foo/list?filter-identifier=bar', orig=access.access_identifier, key=access.access_key
         )
     )
-    assert len(resp.json) == 1
+    assert len(resp.json['data']) == 1
+    assert resp.json['data'][0]['id'] == 'bar'
+    assert resp.json['data'][0]['internal_id'] == str(card.id)
 
     app = get_app(pub)
     app.set_authorization(('Basic', ('test', '12345')))
     resp = app.get('/api/cards/foo/list?filter-identifier=bar')
-    assert len(resp.json) == 1
+    assert len(resp.json['data']) == 1
+    assert resp.json['data'][0]['id'] == 'bar'
+    assert resp.json['data'][0]['internal_id'] == str(card.id)
+
+    resp = app.get('/api/cards/foo/list?filter-identifier=bar&full=on')
+    assert len(resp.json['data']) == 1
+    assert resp.json['data'][0]['id'] == 'bar'
+    assert resp.json['data'][0]['internal_id'] == str(card.id)

tests/api/test_custom_view.py

@@ -497,7 +497,7 @@ def test_api_list_status_filter_different(pub, local_user):
 
     resp = get_app(pub).get(sign_uri('/api/cards/test/list/custom-view/'))
     assert len(resp.json['data']) == 1
-    assert resp.json['data'][0]['id'] == carddata2.id
+    assert resp.json['data'][0]['id'] == str(carddata2.id)
 
     resp = get_app(pub).get(sign_uri('/api/cards/test/custom-view/%s/' % carddata2.id))
     assert resp.json['id'] == str(carddata2.id)

tests/api/test_formdata.py

@@ -7,6 +7,7 @@ import re
 import time
 import xml.etree.ElementTree as ET
 import zipfile
+from contextlib import contextmanager
 
 import pytest
 from django.utils.encoding import force_bytes
@@ -107,6 +108,15 @@ def ics_data(local_user):
     formdata.store()
 
 
+@contextmanager
+def low_export_limit_threshold():
+    from wcs.backoffice.management import FormPage
+
+    FormPage.WCS_SYNC_EXPORT_LIMIT = 10
+    yield
+    FormPage.WCS_SYNC_EXPORT_LIMIT = 100
+
+
 @pytest.mark.parametrize('user', ['query-email', 'api-access'])
 @pytest.mark.parametrize('auth', ['signature', 'http-basic'])
 def test_formdata(pub, local_user, user, auth):
@@ -1253,12 +1263,12 @@ def test_api_list_formdata_filter_status(pub, local_user):
     # filter on id
     resp = get_app(pub).get(sign_uri('/api/forms/foo/list?filter=new', user=local_user))
     assert len(resp.json) == 1
-    assert resp.json[0]['id'] == new.id
+    assert resp.json[0]['id'] == str(new.id)
 
     # filter on name
     resp = get_app(pub).get(sign_uri('/api/forms/foo/list?filter=Ongoing', user=local_user))
     assert len(resp.json) == 1
-    assert resp.json[0]['id'] == wip.id
+    assert resp.json[0]['id'] == str(wip.id)
 
 
 def test_api_list_formdata_unknown_filter(pub, local_user):
@@ -2964,11 +2974,15 @@ def test_api_distance_filter(pub, local_user):
     get_app(pub).get(sign_uri('/api/forms/test/list?filter-distance=150000', user=local_user), status=400)
 
 
-def test_api_ods_formdata(pub, local_user):
+@pytest.mark.parametrize('user', ['query-email', 'api-access'])
+@pytest.mark.parametrize('auth', ['signature', 'http-basic'])
+def test_api_ods_formdata(pub, local_user, user, auth):
     pub.role_class.wipe()
     role = pub.role_class(name='test')
     role.store()
 
+    app = get_app(pub)
+
     FormDef.wipe()
     formdef = FormDef()
     formdef.name = 'test'
@@ -2981,10 +2995,36 @@ def test_api_ods_formdata(pub, local_user):
     data_class = formdef.data_class()
     data_class.wipe()
 
+    if user == 'api-access':
+        ApiAccess.wipe()
+        access = ApiAccess()
+        access.name = 'test'
+        access.access_identifier = 'test'
+        access.access_key = '12345'
+        access.store()
+
+        if auth == 'http-basic':
+
+            def get_url(url, **kwargs):
+                app.set_authorization(('Basic', ('test', '12345')))
+                return app.get(url, **kwargs)
+
+        else:
+
+            def get_url(url, **kwargs):
+                return app.get(sign_uri(url, orig=access.access_identifier, key=access.access_key), **kwargs)
+
+    else:
+        if auth == 'http-basic':
+            pytest.skip('http basic authentication requires ApiAccess')
+
+        def get_url(url, **kwargs):
+            return app.get(sign_uri(url, user=local_user), **kwargs)
+
     # check access is denied if the user has not the appropriate role
-    resp = get_app(pub).get(sign_uri('/api/forms/test/ods', user=local_user), status=403)
+    resp = get_url('/api/forms/test/ods', status=403)
     # even if there's an anonymise parameter
-    resp = get_app(pub).get(sign_uri('/api/forms/test/ods?anonymise', user=local_user), status=403)
+    resp = get_url('/api/forms/test/ods?anonymise', status=403)
 
     data = {'0': 'foobar'}
     for i in range(30):
@@ -2999,28 +3039,25 @@ def test_api_ods_formdata(pub, local_user):
         formdata.store()
 
     # add proper role to user
-    local_user.roles = [role.id]
-    local_user.store()
+    if user == 'api-access':
+        access.roles = [role]
+        access.store()
+    else:
+        local_user.roles = [role.id]
+        local_user.store()
 
     # check it gets the data
-    resp = get_app(pub).get(sign_uri('/api/forms/test/ods', user=local_user))
+    resp = get_url('/api/forms/test/ods')
     assert resp.content_type == 'application/vnd.oasis.opendocument.spreadsheet'
 
-    # check it still gives a ods file when there is more data
-    for i in range(300):
-        formdata = data_class()
-        formdata.data = data
-        formdata.user_id = local_user.id
-        formdata.just_created()
-        formdata.jump_status('new')
-        formdata.store()
-
-    resp = get_app(pub).get(sign_uri('/api/forms/test/ods', user=local_user))
-    assert resp.content_type == 'application/vnd.oasis.opendocument.spreadsheet'
-    with zipfile.ZipFile(io.BytesIO(resp.body)) as zipf:
-        with zipf.open('content.xml') as fd:
-            ods_sheet = ET.parse(fd)
-    assert len(ods_sheet.findall('.//{%s}table-row' % ods.NS['table'])) == 311
+    # check it still gives a ods file when it's over the threashold for afterjobs
+    with low_export_limit_threshold():
+        resp = get_url('/api/forms/test/ods')
+        assert resp.content_type == 'application/vnd.oasis.opendocument.spreadsheet'
+        with zipfile.ZipFile(io.BytesIO(resp.body)) as zipf:
+            with zipf.open('content.xml') as fd:
+                ods_sheet = ET.parse(fd)
+        assert len(ods_sheet.findall('.//{%s}table-row' % ods.NS['table'])) == 11
 
     # check it's not subject to category permissions
     role2 = pub.role_class(name='test2')
@@ -3031,7 +3068,7 @@ def test_api_ods_formdata(pub, local_user):
     category.store()
     formdef.category = category
     formdef.store()
-    get_app(pub).get(sign_uri('/api/forms/test/ods', user=local_user), status=200)
+    get_url('/api/forms/test/ods', status=200)
 
 
 def test_api_global_geojson(pub, local_user):

tests/api/test_statistics.py

@@ -569,6 +569,7 @@ def test_statistics_forms_count_subfilters(pub, formdef):
 
     # remove fields and statuses
     workflow = Workflow(name='Empty wf')
+    workflow.add_status('New')
     workflow.store()
     formdef.workflow = workflow
     formdef.fields.clear()
@@ -578,8 +579,31 @@ def test_statistics_forms_count_subfilters(pub, formdef):
     resp = get_app(pub).get(sign_uri(url))
     assert resp.json['data'] == {
         'series': [{'data': [], 'label': 'Forms Count'}],
-        'subfilters': [],
         'x_labels': [],
+        'subfilters': [
+            {
+                'has_subfilters': True,
+                'id': 'group-by',
+                'label': 'Group by',
+                'options': [
+                    {'id': 'channel', 'label': 'Channel'},
+                    {'id': 'simple-status', 'label': 'Simplified status'},
+                    {'id': 'status', 'label': 'Status'},
+                ],
+            },
+            {
+                'default': '_all',
+                'id': 'filter-status',
+                'label': 'Status',
+                'options': [
+                    {'id': '_all', 'label': 'All'},
+                    {'id': 'pending', 'label': 'Open'},
+                    {'id': 'done', 'label': 'Done'},
+                    {'id': '1', 'label': 'New'},
+                ],
+                'required': True,
+            },
+        ],
     }
 
 
@@ -967,6 +991,10 @@ def test_statistics_forms_count_group_by(pub, formdef, anonymise):
         {'data': [6, None, None], 'label': 'Backoffice'},
     ]
 
+    # group by channel without form filter
+    new_resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?group-by=channel'))
+    assert new_resp.json['data']['series'] == resp.json['data']['series']
+
     # group by item field without time interval
     resp = get_app(pub).get(sign_uri(url + '&group-by=test-item&time_interval=none'))
     # Foo is first because it has a display value, baz is second because it has not, None is always last
@@ -1046,8 +1074,12 @@ def test_statistics_forms_count_group_by_same_varname(pub, formdef):
 
 
 def test_statistics_forms_count_group_by_form(pub):
+    category_a = Category(name='Category A')
+    category_a.store()
+
     formdef = FormDef()
     formdef.name = 'A'
+    formdef.category_id = category_a.id
     formdef.store()
 
     for i in range(10):
@@ -1058,6 +1090,7 @@ def test_statistics_forms_count_group_by_form(pub):
 
     formdef = FormDef()
     formdef.name = 'B'
+    formdef.category_id = category_a.id
     formdef.store()
 
     for i in range(5):
@@ -1071,7 +1104,11 @@ def test_statistics_forms_count_group_by_form(pub):
     assert resp.json['data']['subfilters'][1] == {
         'id': 'group-by',
         'label': 'Group by',
-        'options': [{'id': 'form', 'label': 'Form'}],
+        'options': [
+            {'id': 'channel', 'label': 'Channel'},
+            {'id': 'form', 'label': 'Form'},
+        ],
+        'has_subfilters': True,
     }
 
     resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?time_interval=year'))
@@ -1089,6 +1126,18 @@ def test_statistics_forms_count_group_by_form(pub):
     assert resp.json['data']['x_labels'] == ['A', 'B']
     assert resp.json['data']['series'] == [{'data': [10, 5], 'label': 'Forms Count'}]
 
+    resp = get_app(pub).get(
+        sign_uri('/api/statistics/forms/count/?time_interval=none&group-by=form&form=category:category-a')
+    )
+    assert resp.json['data']['x_labels'] == ['A', 'B']
+    assert resp.json['data']['series'] == [{'data': [10, 5], 'label': 'Forms Count'}]
+
+    resp = get_app(pub).get(
+        sign_uri('/api/statistics/forms/count/?time_interval=none&group-by=form&form=a&form=b')
+    )
+    assert resp.json['data']['x_labels'] == ['A', 'B']
+    assert resp.json['data']['series'] == [{'data': [10, 5], 'label': 'Forms Count'}]
+
 
 def test_statistics_forms_count_months_to_show(pub, formdef):
     for i in range(24):
@@ -1685,6 +1734,7 @@ def test_statistics_multiple_forms_count_subfilters(pub, formdef):
     group_by_filter = [x for x in resp.json['data']['subfilters'] if x['id'] == 'group-by'][0]
     assert group_by_filter['options'] == [
         {'id': 'channel', 'label': 'Channel'},
+        {'id': 'form', 'label': 'Form'},
         {'id': 'simple-status', 'label': 'Simplified status'},
         {'id': 'test-item', 'label': 'Test item'},
         {'id': 'checkbox', 'label': 'Checkbox'},
@@ -1708,3 +1758,10 @@ def test_statistics_multiple_forms_count_subfilters(pub, formdef):
 
     category_resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?form=category:category-a'))
     assert category_resp.json == resp.json
+
+    # cannot group by form if single form is selected
+    form_resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?form=test'))
+    form_group_by_filter = [x for x in form_resp.json['data']['subfilters'] if x['id'] == 'group-by'][0]
+    assert [x for x in group_by_filter['options'] if x not in form_group_by_filter['options']] == [
+        {'id': 'form', 'label': 'Form'}
+    ]

tests/backoffice_pages/test_all.py

@@ -1717,6 +1717,10 @@ def test_backoffice_parallel_handling(pub, freezer):
         resp3.pyquery('.global-errors summary + p').text()
         == 'Another action has been performed on this form in the meantime and data may have been changed.'
     )
+    # check it's possible to click it now, as it's been refreshed and the agent warned.
+    assert 'button_finish' in resp3.forms[0].fields
+    resp3 = resp3.forms[0].submit('button_finish').follow()
+    assert not resp3.pyquery('.global-errors summary')
 
 
 def test_backoffice_handling_global_action(pub):
@@ -1755,6 +1759,41 @@ def test_backoffice_handling_global_action(pub):
     assert formdef.data_class().get(formdata.id).status == 'wf-finished'
 
 
+def test_backoffice_handling_global_action_parallel_handling(pub):
+    create_user(pub)
+
+    formdef = FormDef()
+    formdef.name = 'test global action'
+    formdef.fields = []
+
+    workflow = Workflow(name='test global action')
+    workflow.add_status('st0')
+    action = workflow.add_global_action('FOOBAR')
+    register_comment = action.add_action('register-comment')
+    register_comment.comment = 'HELLO WORLD GLOBAL ACTION'
+    jump = action.add_action('jump')
+    jump.status = 'finished'
+    trigger = action.triggers[0]
+    trigger.roles = [x.id for x in pub.role_class.select() if x.name == 'foobar']
+
+    workflow.store()
+    formdef.workflow_id = workflow.id
+    formdef.workflow_roles = {'_receiver': 1}
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/%s/%s/' % (formdef.url_name, formdata.id))
+    resp.form.submit('button-action-1').follow()
+    resp3 = resp.form.submit('button-action-1')
+    assert resp3.pyquery('.global-errors summary').text() == 'Error: parallel execution.'
+    resp = app.get('/backoffice/management/%s/%s/' % (formdef.url_name, formdata.id))
+    assert resp.text.count('HELLO WORLD GLOBAL ACTION') == 1
+
+
 def test_backoffice_global_remove_action(pub):
     user = create_user(pub)
 

tests/backoffice_pages/test_carddata.py

@@ -368,6 +368,40 @@ def test_studio_card_item_link(pub):
         resp.click('card plop')
 
 
+def test_backoffice_card_item_link_id_template(pub):
+    user = create_user(pub)
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = [
+        fields.StringField(id='1', label='Test', varname='foo'),
+        fields.StringField(id='2', label='Custom id', varname='custom_id'),
+    ]
+    carddef.backoffice_submission_roles = user.roles
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.id_template = '{{form_var_custom_id}}'
+    carddef.digest_templates = {'default': 'card {{ form_var_foo }}'}
+    carddef.store()
+    carddef.data_class().wipe()
+
+    card = carddef.data_class()()
+    card.data = {'1': 'plop', '2': 'test'}
+    card.just_created()
+    card.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/data/foo/')
+    assert [x.attrib['href'] for x in resp.pyquery('table a')] == ['test/']
+    resp = resp.click('Add')
+    resp.form['f1'] = 'blah'
+    resp.form['f2'] = 'blah'
+    resp = resp.form.submit('submit')
+    assert resp.location.endswith('/backoffice/data/foo/blah/')
+    resp = resp.follow()
+    resp = app.get('/backoffice/data/foo/')
+    assert [x.attrib['href'] for x in resp.pyquery('table a')] == ['blah/', 'test/']
+
+
 def test_backoffice_cards_import_data_from_csv(pub):
     user = create_user(pub)
 
@@ -647,6 +681,57 @@ def test_backoffice_cards_import_data_csv_no_backoffice_fields(pub):
     assert carddef.data_class().count() == 2
 
 
+def test_backoffice_cards_import_data_csv_custom_id_update(pub):
+    user = create_user(pub)
+    user.name_identifiers = [str(uuid.uuid4())]
+    user.store()
+
+    Workflow.wipe()
+    workflow = Workflow(name='form-title')
+    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
+    workflow.backoffice_fields_formdef.fields = [
+        fields.StringField(id='bo0', varname='foo_bovar', label='bo variable'),
+    ]
+    workflow.add_status('st0')
+    workflow.store()
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'test'
+    carddef.fields = [
+        fields.StringField(id='1', label='String', varname='custom_id'),
+        fields.ItemField(id='2', label='List', items=['item1', 'item2']),
+    ]
+    carddef.backoffice_submission_roles = user.roles
+    carddef.id_template = '{{form_var_custom_id}}'
+    carddef.workflow = workflow
+    carddef.store()
+    carddef.data_class().wipe()
+
+    card = carddef.data_class()()
+    card.data = {'1': 'plop', '2': 'test', 'bo0': 'xxx'}
+    card.just_created()
+    card.store()
+
+    app = login(get_app(pub))
+    data = b'''\
+"String","List"
+"plop","item1"
+"test","item2"
+'''
+    resp = app.get('/backoffice/data/test/import-file')
+    resp.forms[0]['file'] = Upload('test.csv', data, 'text/csv')
+    resp = resp.forms[0].submit().follow()
+    assert carddef.data_class().count() == 2
+
+    card.refresh_from_storage()
+    assert card.data == {'1': 'plop', '2': 'item1', '2_display': 'item1', 'bo0': 'xxx'}
+
+    other_card = carddef.data_class().select(order_by='-receipt_time')[0]
+    assert other_card.data == {'1': 'test', '2': 'item2', '2_display': 'item2', 'bo0': None}
+    assert other_card.id_display == 'test'
+
+
 def test_backoffice_cards_import_data_csv_blockfield(pub):
     user = create_user(pub)
 
@@ -1070,6 +1155,10 @@ def test_backoffice_cards_update_data_from_json(pub):
     workflow = CardDef.get_default_workflow()
     workflow.id = None
     workflow.add_status('status2', 'st2')
+    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
+    workflow.backoffice_fields_formdef.fields = [
+        fields.StringField(id='bo1', label='bo field 1', varname='bo_data'),
+    ]
     workflow.store()
 
     CardDef.wipe()
@@ -1099,6 +1188,7 @@ def test_backoffice_cards_update_data_from_json(pub):
     json_export = json.loads(job.file_content)
     assert len(json_export['data']) == 1
     json_export['data'][0]['fields']['string'] = 'plop 2'
+    json_export['data'][0]['workflow']['fields']['bo_data'] = 'plop 2'
 
     # update
     resp = app.get(carddef.get_url())
@@ -1111,10 +1201,27 @@ def test_backoffice_cards_update_data_from_json(pub):
 
     assert carddef.data_class().count() == 1
     card.refresh_from_storage()
-    assert card.data == {'1': 'plop 2'}
+    assert card.data == {'1': 'plop 2', 'bo1': 'plop 2'}
     assert isinstance(card.evolution[0].parts[-1], ContentSnapshotPart)
-    assert card.evolution[0].parts[-1].old_data == {'1': 'plop'}
-    assert card.evolution[0].parts[-1].new_data == {'1': 'plop 2'}
+    assert card.evolution[0].parts[-1].old_data == {'1': 'plop', 'bo1': None}
+    assert card.evolution[0].parts[-1].new_data == {'1': 'plop 2', 'bo1': 'plop 2'}
+
+    # update and reset backoffice fields
+    json_export['data'][0]['workflow']['fields']['bo_data'] = None
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Import data from a file')
+    resp.forms[0]['file'] = Upload('test.json', json.dumps(json_export).encode(), 'application/json')
+    resp.form['update_existing_cards'].checked = True
+    resp = resp.forms[0].submit()
+    assert '/backoffice/processing?job=' in resp.location
+    resp = resp.follow()
+
+    assert carddef.data_class().count() == 1
+    card.refresh_from_storage()
+    assert card.data == {'1': 'plop 2', 'bo1': None}
+    assert isinstance(card.evolution[0].parts[-1], ContentSnapshotPart)
+    assert card.evolution[0].parts[-1].old_data == {'1': 'plop 2', 'bo1': 'plop 2'}
+    assert card.evolution[0].parts[-1].new_data == {'1': 'plop 2', 'bo1': None}
 
     # no uuid -> create
     json_export['data'][0]['uuid'] = None
@@ -1139,7 +1246,10 @@ def test_backoffice_cards_update_data_from_json(pub):
     assert '/backoffice/processing?job=' in resp.location
     resp = resp.follow()
     assert carddef.data_class().count() == 3
-    assert carddef.data_class().get_by_uuid(json_export['data'][0]['uuid']).data == {'1': 'plop 4'}
+    assert carddef.data_class().get_by_uuid(json_export['data'][0]['uuid']).data == {
+        '1': 'plop 4',
+        'bo1': None,
+    }
 
     # invalid uuid -> ignore
     json_export['data'][0]['uuid'] = 'hello world'
@@ -1165,7 +1275,7 @@ def test_backoffice_cards_update_data_from_json(pub):
     resp = resp.follow()
     assert carddef.data_class().count() == 5
     assert len({x.uuid for x in carddef.data_class().select()}) == 5  # all unique UUIDs
-    assert carddef.data_class().get_by_uuid(card.uuid).data == {'1': 'plop 2'}
+    assert carddef.data_class().get_by_uuid(card.uuid).data == {'1': 'plop 2', 'bo1': None}
 
     # update and change status
     json_export['data'][0]['uuid'] = str(card.uuid)

tests/backoffice_pages/test_submission.py

@@ -161,6 +161,52 @@ def test_backoffice_submission(pub):
     assert resp.location == 'http://www.example.org/'
 
 
+def test_backoffice_submission_menu_entry(pub):
+    user = create_user(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = []
+    formdef.workflow_roles = {'_receiver': 1}
+    formdef.backoffice_submission_roles = user.roles[:]
+    formdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/forms')
+    assert resp.pyquery('#sidepage-menu .icon-submission')
+
+    pub.cfg['backoffice-submission'] = {}
+    pub.cfg['backoffice-submission']['sidebar_menu_entry'] = 'visible'
+    pub.write_cfg()
+    resp = app.get('/backoffice/submission/', status=200)
+    assert resp.pyquery('#sidepage-menu .icon-submission')
+
+    pub.cfg['backoffice-submission']['sidebar_menu_entry'] = 'redirect'
+    pub.cfg['backoffice-submission']['redirect'] = 'https://example.net/'
+    pub.write_cfg()
+    resp = app.get('/backoffice/management/forms')
+    assert resp.pyquery('#sidepage-menu .icon-submission')
+    resp = app.get('/backoffice/submission/', status=302)
+    assert resp.location == 'https://example.net/'
+
+    pub.cfg['backoffice-submission']['sidebar_menu_entry'] = 'hidden'
+    pub.write_cfg()
+    resp = app.get('/backoffice/management/forms')
+    assert not resp.pyquery('#sidepage-menu .icon-submission')
+    resp = app.get('/backoffice/submission/', status=302)
+    assert resp.location == 'https://example.net/'
+
+    pub.cfg['backoffice-submission'][
+        'redirect'
+    ] = '{% if session_user_email == "admin@localhost" %}https://example.net/{% endif %}'
+    pub.write_cfg()
+    app.get('/backoffice/submission/', status=302)  # redirection
+    user.email = 'admin2@localhost'
+    user.store()
+    app.get('/backoffice/submission/', status=200)  # native screen
+
+
 def test_backoffice_submission_with_tracking_code(pub):
     user = create_user(pub)
 
@@ -389,8 +435,8 @@ def test_backoffice_parallel_submission(pub, autosave):
     formdata.store()
 
     app = login(get_app(pub))
-    resp = app.get('/backoffice/submission/')
-    assert 'Submission to complete' in resp.text
+    resp = app.get('/backoffice/submission/pending')
+    assert resp.pyquery('tbody tr')
     resp1 = app.get('/backoffice/submission/form-title/%s/' % formdata.id)
     resp1 = resp1.follow()
     resp2 = app.get('/backoffice/submission/form-title/%s/' % formdata.id)
@@ -600,13 +646,16 @@ def test_backoffice_submission_drafts(pub):
     tracking_code = data_class.select()[0].tracking_code
 
     # stop here, go back to index
+    pub.cfg['submission-channels'] = {'include-in-global-listing': True}
+    pub.write_cfg()
     resp = app.get('/backoffice/submission/')
-    assert '%s/%s' % (formdef.url_name, formdata_no) in resp.text
-    assert '>#%s' % formdata_no in resp.text
+    resp = resp.click('Pending submissions')
+    assert resp.pyquery('tbody tr a').text() == formdata.get_display_name()
+    assert resp.pyquery('tbody tr a')[0].attrib['href'] == f'{formdef.url_name}/{formdata_no}/'
     formdata.submission_channel = 'mail'
     formdata.store()
-    resp = app.get('/backoffice/submission/')
-    assert '>Mail #%s' % formdata_no in resp.text
+    resp = app.get('/backoffice/submission/pending')
+    assert resp.pyquery('tbody td:nth-child(1)').text() == 'Mail'
 
     # check it can also be accessed using its final URL
     resp2 = app.get('/backoffice/management/%s/%s/' % (formdef.url_name, formdata_no))
@@ -628,6 +677,33 @@ def test_backoffice_submission_drafts(pub):
     assert resp.location == 'http://example.net/backoffice/management/form-title/%s/' % formdata_no
 
 
+def test_backoffice_draft_with_digest(pub):
+    user = create_user(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [
+        fields.StringField(id='1', label='1st field', varname='foo'),
+    ]
+    formdef.backoffice_submission_roles = user.roles[:]
+    formdef.digest_templates = {'default': 'digest: {{ form_var_foo }}'}
+    formdef.workflow_roles = {'_receiver': 1}
+    formdef.store()
+
+    formdef.data_class().wipe()
+    formdata = formdef.data_class()()
+    formdata.data = {'1': 'bar'}
+    formdata.status = 'draft'
+    formdata.backoffice_submission = True
+    formdata.submission_agent_id = str(user.id)
+    formdata.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/submission/pending')
+    assert resp.pyquery('tbody td:nth-child(1)').text() == 'form title #1-1 digest: bar'
+
+
 def test_backoffice_submission_remove_drafts(pub):
     user = create_user(pub)
 
@@ -661,7 +737,7 @@ def test_backoffice_submission_remove_drafts(pub):
     formdata_no = formdata.id
 
     # stop here, go back to the index
-    resp = app.get('/backoffice/submission/')
+    resp = app.get('/backoffice/submission/pending')
     resp = resp.click('#%s' % formdata_no)
     resp = resp.follow()
 
@@ -673,7 +749,7 @@ def test_backoffice_submission_remove_drafts(pub):
     assert pub.tracking_code_class().count() == 1
 
     # and this time for real
-    resp = app.get('/backoffice/submission/')
+    resp = app.get('/backoffice/submission/pending')
     resp = resp.click('#%s' % formdata_no)
     resp = resp.follow()
     resp = resp.click('Discard this form')
@@ -951,48 +1027,6 @@ def test_backoffice_submission_conditional_jump_based_on_bo_field(pub):
     assert formdef.data_class().select()[0].status == 'wf-st1'
 
 
-def test_backoffice_submission_sections(pub):
-    user = create_user(pub)
-
-    FormDef.wipe()
-    formdef = FormDef()
-    formdef.name = 'form title'
-    formdef.fields = [
-        fields.StringField(
-            id='1', label='1st field', display_locations=['validation', 'summary', 'listings']
-        ),
-    ]
-    formdef.backoffice_submission_roles = user.roles[:]
-    formdef.store()
-
-    data_class = formdef.data_class()
-    data_class.wipe()
-
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/submission/')
-    assert 'Submission to complete' not in resp.text
-    assert 'Running submission' not in resp.text
-
-    formdata = data_class()
-    formdata.data = {}
-    formdata.status = 'draft'
-    formdata.backoffice_submission = True
-    formdata.receipt_time = make_aware(datetime.datetime(2015, 1, 1))
-    formdata.store()
-
-    resp = app.get('/backoffice/submission/')
-    assert 'Submission to complete' in resp.text
-    assert 'Running submission' not in resp.text
-    assert '>#%s' % formdata.id in resp.text
-
-    formdata.data = {'1': 'xxx'}
-    formdata.store()
-    resp = app.get('/backoffice/submission/')
-    assert 'Submission to complete' not in resp.text
-    assert 'Running submission' in resp.text
-    assert '>#%s' % formdata.id in resp.text
-
-
 def test_backoffice_submission_drafts_order(pub):
     user = create_user(pub)
 
@@ -1011,28 +1045,39 @@ def test_backoffice_submission_drafts_order(pub):
     data_class.wipe()
 
     formdata_ids = []
-    for i in range(10):
+    for i in range(25):
         formdata = data_class()
         formdata.data = {}
         formdata.status = 'draft'
         formdata.backoffice_submission = True
-        formdata.receipt_time = make_aware(datetime.datetime(2023, 11, 20 - i))
+        formdata.receipt_time = make_aware(datetime.datetime(2023, 11, 30 - i))
         formdata.store()
         formdata_ids.append(formdata.id)
 
     app = login(get_app(pub))
-    resp = app.get('/backoffice/submission/')
-    assert [x.attrib['href'] for x in resp.pyquery('.biglist.empty a:not(.fake)')] == [
-        f'form-title/{x}/' for x in reversed(formdata_ids)
+    resp = app.get('/backoffice/submission/pending')
+    assert [x.attrib['data-link'] for x in resp.pyquery('tbody tr')] == [
+        f'form-title/{x}/' for x in formdata_ids[:20]
     ]
 
     formdata.receipt_time = None  # check a missing receipt_time is ok
     formdata.store()
-    resp = app.get('/backoffice/submission/')
-    assert [x.attrib['href'] for x in resp.pyquery('.biglist.empty a:not(.fake)')] == [
-        f'form-title/{x}/' for x in reversed(formdata_ids)
+    new_order = [formdata.id] + [x for x in formdata_ids if x != formdata.id]
+    resp = app.get('/backoffice/submission/pending')
+    assert [x.attrib['data-link'] for x in resp.pyquery('tbody tr')] == [
+        f'form-title/{x}/' for x in new_order[:20]
     ]
-    assert 'unknown date' in resp.pyquery('li.smallitem:first').text()
+
+    resp = resp.click('<!--Next Page-->')
+    assert [x.attrib['data-link'] for x in resp.pyquery('tbody tr')] == [
+        f'form-title/{x}/' for x in new_order[20:]
+    ]
+
+    # check ajax call result
+    resp = app.get('/backoffice/submission/pending?ajax=true')
+    assert 'appbar' not in resp.text
+    assert '<table' in resp.text
+    assert 'page-links' in resp.text
 
 
 def test_backoffice_submission_prefill_user(pub):
@@ -1267,7 +1312,7 @@ def test_backoffice_submission_multiple_page_restore_on_validation(pub):
     assert formdef.data_class().count() == 1
     formdata = formdef.data_class().select()[0]
     # restore draft
-    resp = app.get('/backoffice/submission/')
+    resp = app.get('/backoffice/submission/pending')
     resp = resp.click(href='form-title/%s' % formdata.id)
     resp = resp.follow()
     assert 'Check values then click submit.' in resp.text

tests/backoffice_pages/test_workflow_inspect.py

@@ -118,7 +118,8 @@ def test_workflow_inspect_page(pub):
 
     resp = app.get('/backoffice/workflows/%s/inspect' % workflow.id)
     assert (
-        '<span class="parameter">Model:</span> '
+        '<span class="parameter">Model:</span> File</li>'
+        '<li class="parameter-model_file">'
         '<a href="status/st3/items/_export_to/?file=model_file">test.odt</a></li>'
     ) in resp.text
 

tests/form_pages/test_all.py

@@ -3970,6 +3970,24 @@ def test_email_actions(pub, emails):
         assert html_payload.count('/actions/') == 6
         assert html_payload.count('button link start') == 4  # 2x2 buttons + 2x1 button
 
+    # check with missing label parameter
+    pub.loggederror_class.wipe()
+    workflow.possible_status[0].items[1].body = '''{% action_button "ok" %}'''
+    workflow.store()
+    emails.empty()
+    formdef.data_class().wipe()
+    app = login(get_app(pub), username='foo', password='foo')
+    resp = app.get(formdef.get_url())
+    resp = resp.form.submit('submit')
+    resp = resp.form.submit('submit')
+    assert not emails.get('New form2 (test email action)')
+    assert pub.loggederror_class.count() == 1
+    assert pub.loggederror_class.select()[0].summary == 'Error in body template, mail could not be generated'
+    assert (
+        pub.loggederror_class.select()[0].exception_message
+        == '{% action_button %} requires a label parameter'
+    )
+
 
 def test_card_email_actions(pub, emails):
     create_user(pub)
@@ -4059,6 +4077,26 @@ def test_email_temporary_form_button(pub, emails):
     resp = app.get(form_url).follow()
     assert 'The form has been recorded' in resp.text
 
+    # check with missing label parameter
+    pub.loggederror_class.wipe()
+    workflow.possible_status[0].items[1].body = 'Hello;\n{% temporary_access_button %}\nAdiós.'
+    workflow.store()
+    formdef.refresh_from_storage()
+    emails.empty()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+    formdata.perform_workflow()
+    formdata.store()
+    assert not emails.get('New form')
+    assert pub.loggederror_class.count() == 1
+    assert pub.loggederror_class.select()[0].summary == 'Error in body template, mail could not be generated'
+    assert (
+        pub.loggederror_class.select()[0].exception_message
+        == '{% temporary_action_button %} requires a label parameter'
+    )
+
 
 def test_manager_public_access(pub):
     user, manager = create_user_and_admin(pub)
@@ -5973,3 +6011,65 @@ def test_form_errors_summary(pub):
     resp = resp.forms[0].submit('submit')
     assert 'The following field has an error: testblock' in resp.pyquery('.errornotice').text()
     assert resp.pyquery('.error').text() == 'required field required field '
+
+
+def test_form_submit_no_csrf(pub):
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [fields.StringField(id='0', label='string')]
+    formdef.confirmation = False
+    formdef.store()
+    formdef.data_class().wipe()
+
+    create_user(pub)
+    app = get_app(pub)
+    login(app, username='foo', password='foo')
+    resp = app.get(formdef.get_url())
+    resp.form['f0'] = 'hello'
+    # get expected data
+    form_data = {x: y for x, y in resp.form.submit_fields('submit')}
+    # remove token values
+    form_data['_form_id'] = 'xxx'
+    form_data['_ajax_form_token'] = 'xxx'
+    form_data['magictoken'] = 'xxx'
+    # simulate call from remote/attacker site (form token prevents this)
+    resp = app.post(formdef.get_url(), params=form_data)
+    assert 'The form you have submitted is invalid.' in resp.text
+
+    # with confirmation page
+    formdef.confirmation = True
+    formdef.store()
+    resp = app.get(formdef.get_url())
+    resp.form['f0'] = 'hello'
+    resp = resp.form.submit('submit')
+    # get expected data
+    form_data = {x: y for x, y in resp.form.submit_fields('submit')}
+    # remove token values
+    form_data['_form_id'] = 'xxx'
+    form_data['_ajax_form_token'] = 'xxx'
+    form_data['magictoken'] = 'xxx'
+    # simulate call from remote/attacker site (magictoken prevents this)
+    resp = app.post(formdef.get_url(), params=form_data, status=302)
+    assert resp.location == formdef.get_url()
+
+    # with multiple pages
+    formdef.confirmation = False
+    formdef.fields = [
+        fields.PageField(id='1', label='page1'),
+        fields.PageField(id='2', label='page2'),
+        fields.StringField(id='3', label='string'),
+    ]
+    formdef.store()
+    resp = app.get(formdef.get_url())
+    resp = resp.form.submit('submit')
+    resp.form['f3'] = 'hello'
+    # get expected data
+    form_data = {x: y for x, y in resp.form.submit_fields('submit')}
+    # remove token values
+    form_data['_form_id'] = 'xxx'
+    form_data['_ajax_form_token'] = 'xxx'
+    form_data['magictoken'] = 'xxx'
+
+    # simulate call from remote/attacker site (magictokens prevents this)
+    resp = app.post(formdef.get_url(), params=form_data, status=302)
+    assert resp.location == formdef.get_url()

tests/form_pages/test_computed_field.py

@@ -2,6 +2,7 @@ import datetime
 import decimal
 
 import pytest
+import responses
 from django.utils.timezone import make_aware
 from webtest import Upload
 
@@ -303,12 +304,14 @@ def test_computed_field_recall_draft(pub):
     assert formdata.data == {'1': 'value'}
 
 
-def test_computed_field_complex_data(pub, http_requests):
+@pytest.mark.parametrize('http_method', ['get', 'post'])
+def test_computed_field_complex_data(pub, http_method):
     FormDef.wipe()
+    NamedWsCall.wipe()
 
     wscall = NamedWsCall()
     wscall.name = 'Hello world'
-    wscall.request = {'url': 'http://remote.example.net/json'}
+    wscall.request = {'url': 'http://remote.example.net/json', 'method': http_method.upper()}
     wscall.store()
 
     formdef = FormDef()
@@ -326,14 +329,69 @@ def test_computed_field_complex_data(pub, http_requests):
     formdef.store()
     formdef.data_class().wipe()
 
-    resp = get_app(pub).get('/test/')
-    assert 'XbarY' in resp.text
-    resp = resp.forms[0].submit('submit')  # -> validation
-    resp = resp.forms[0].submit('submit').follow()  # -> submit
-    assert 'The form has been recorded' in resp.text
-    assert formdef.data_class().count() == 1
-    formdata = formdef.data_class().select()[0]
-    assert formdata.data['1'] == {'foo': 'bar'}
+    # check with a dictionary as response
+    with responses.RequestsMock() as rsps:
+        getattr(rsps, http_method)('http://remote.example.net/json', json={'foo': 'bar'})
+        resp = get_app(pub).get('/test/')
+        assert 'XbarY' in resp.text
+        resp = resp.forms[0].submit('submit')  # -> validation
+        resp = resp.forms[0].submit('submit').follow()  # -> submit
+        assert 'The form has been recorded' in resp.text
+        assert formdef.data_class().count() == 1
+        formdata = formdef.data_class().select()[0]
+        assert formdata.data['1'] == {'foo': 'bar'}
+
+    # check with a list as response
+    formdef.data_class().wipe()
+    with responses.RequestsMock() as rsps:
+        formdef.fields[1].label = 'X{{form_var_computed_1_foo}}Y'
+        formdef.store()
+        getattr(rsps, http_method)('http://remote.example.net/json', json=[{'foo': 'xxx'}, {'foo': 'bar'}])
+        resp = get_app(pub).get('/test/')
+        assert 'XbarY' in resp.text
+        resp = resp.forms[0].submit('submit')  # -> validation
+        resp = resp.forms[0].submit('submit').follow()  # -> submit
+        assert 'The form has been recorded' in resp.text
+        assert formdef.data_class().count() == 1
+        formdata = formdef.data_class().select()[0]
+        assert formdata.data['1'] == [{'foo': 'xxx'}, {'foo': 'bar'}]
+
+    # check with the computed field extracting a list from the response
+    formdef.data_class().wipe()
+    with responses.RequestsMock() as rsps:
+        formdef.fields[0].value_template = '{{ webservice.hello_world|get:"data" }}'
+        formdef.fields[1].label = 'X{{form_var_computed_1_foo}}Y'
+        formdef.store()
+        getattr(rsps, http_method)(
+            'http://remote.example.net/json', json={'data': [{'foo': 'xxx'}, {'foo': 'bar'}]}
+        )
+        resp = get_app(pub).get('/test/')
+        assert 'XbarY' in resp.text
+        resp = resp.forms[0].submit('submit')  # -> validation
+        resp = resp.forms[0].submit('submit').follow()  # -> submit
+        assert 'The form has been recorded' in resp.text
+        assert formdef.data_class().count() == 1
+        formdata = formdef.data_class().select()[0]
+        assert formdata.data['1'] == [{'foo': 'xxx'}, {'foo': 'bar'}]
+
+    if http_method == 'post':
+        # check with (complex) post data
+        formdef.data_class().wipe()
+        with responses.RequestsMock() as rsps:
+            wscall.request['post_data'] = {'test': '{{ "test"|qrcode }}'}
+            wscall.store()
+            formdef.fields[0].value_template = '{{ webservice.hello_world|get:"data" }}'
+            formdef.fields[1].label = 'X{{form_var_computed_1_foo}}Y'
+            formdef.store()
+            getattr(rsps, http_method)('http://remote.example.net/json', json={'data': {'1': {'foo': 'bar'}}})
+            resp = get_app(pub).get('/test/')
+            assert 'XbarY' in resp.text
+            resp = resp.forms[0].submit('submit')  # -> validation
+            resp = resp.forms[0].submit('submit').follow()  # -> submit
+            assert 'The form has been recorded' in resp.text
+            assert formdef.data_class().count() == 1
+            formdata = formdef.data_class().select()[0]
+            assert formdata.data['1'] == {'1': {'foo': 'bar'}}
 
 
 def test_computed_field_decimal_data(pub, http_requests):

tests/form_pages/test_file_field.py

@@ -238,6 +238,42 @@ def test_form_file_field_image_submit(pub):
     assert '<img alt="" src="tempfile?' not in resp.text
 
 
+def test_form_file_field_html_submit(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [fields.FileField(id='0', label='file')]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    html_content = b'<html><body>hello</body></html>'
+    upload = Upload('test.html', html_content, 'text/html')
+
+    app = get_app(pub)
+    resp = app.get('/test/')
+    resp.forms[0]['f0$file'] = upload
+    resp = resp.forms[0].submit('submit')
+    assert 'Check values then click submit.' in resp.text
+    tempfile_id = resp.pyquery('.fileinfo .filename a').attr.href.split('=')[1]
+
+    resp_tempfile = app.get('/test/tempfile?t=%s' % tempfile_id)
+    assert resp_tempfile.body == html_content
+
+    resp = resp.form.submit('submit').follow()
+    assert resp.click('test.html').follow().content_type == 'text/html'
+    assert resp.click('test.html').follow().body == html_content
+
+    # check it's also served raw from backoffice
+    user = create_user(pub)
+    user.is_admin = True
+    user.store()
+    app = get_app(pub)
+    login(app, username='foo', password='foo')
+    resp = app.get(formdef.data_class().select()[0].get_backoffice_url())
+    assert resp.click('test.html').follow().content_type == 'text/html'
+    assert resp.click('test.html').follow().body == html_content
+
+
 def test_form_file_field_submit_document_type(pub):
     FormDef.wipe()
     formdef = FormDef()

tests/form_pages/test_map_field.py

@@ -233,6 +233,7 @@ def test_form_map_field_default_position(pub):
     resp.form['f1'] = '169 rue du chateau, paris'
     resp = resp.form.submit('submit')
     assert resp.pyquery('.qommon-map').attr('data-def-lat') == '13'
+    assert resp.pyquery('.qommon-map').attr('data-def-template')
 
     formdef.fields[3].initial_position = 'template'
     formdef.fields[3].position_template = '{{ form_var_address }}'
@@ -244,3 +245,4 @@ def test_form_map_field_default_position(pub):
         rsps.get('https://nominatim.entrouvert.org/search', json=[{'lat': '48.8337085', 'lon': '2.3233693'}])
         resp = resp.form.submit('submit')
         assert resp.pyquery('.qommon-map').attr('data-def-lat') == '48.83370850'
+        assert resp.pyquery('.qommon-map').attr('data-def-template')

tests/test_carddef.py

@@ -1309,3 +1309,22 @@ def test_card_custom_id_draft(pub):
     card.anonymise()
     with pytest.raises(KeyError):
         carddef.data_class().get_by_id('id1')
+
+
+def test_card_custom_id_format(pub):
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.store()
+
+    data_class = carddef.data_class()
+    assert data_class.force_valid_id_characters('foobar') == 'foobar'
+    assert data_class.force_valid_id_characters('Foobar') == 'Foobar'
+    assert data_class.force_valid_id_characters(' Foobar') == 'Foobar'
+    assert data_class.force_valid_id_characters(' Foo bar') == 'Foo-bar'
+    assert data_class.force_valid_id_characters(' Fôô bar') == 'Foo-bar'
+    assert data_class.force_valid_id_characters(' Fôô bar...') == 'Foo-bar'
+    assert data_class.force_valid_id_characters('_Fôô bar-') == '_Foo-bar-'
+    assert data_class.force_valid_id_characters('_Fôô  bar-') == '_Foo-bar-'
+    assert data_class.force_valid_id_characters('_Fôô  bar☭-') == '_Foo-bar-'
+    assert data_class.force_valid_id_characters('_Fôô  bar❗') == '_Foo-bar'

tests/test_ctl.py

@@ -1,8 +1,8 @@
+import io
 import json
 import os
 import pickle
 import shutil
-import sys
 import tempfile
 import zipfile
 from unittest import mock
@@ -13,7 +13,6 @@ import pytest
 import responses
 from django.core.management import CommandError, call_command
 
-import wcs.qommon.ctl
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.ctl.management.commands.trigger_jumps import select_and_jump_formdata
@@ -53,16 +52,6 @@ def alt_tempdir():
     shutil.rmtree(alt_tempdir)
 
 
-def test_loading():
-    ctl = wcs.qommon.ctl.Ctl(cmd_prefixes=['wcs.ctl'])
-    ctl.load_all_commands(ignore_errors=False)
-    # noqa pylint: disable=consider-iterating-dictionary
-    assert 'shell' in ctl.get_commands().keys()
-    # call all __init__() methods
-    for cmd in ctl.get_commands().values():
-        cmd()
-
-
 def test_collectstatic(pub, tmp_path):
     CmdCollectStatic.collectstatic(pub)
     assert os.path.exists(os.path.join(pub.app_dir, 'collectstatic', 'css', 'required.png'))
@@ -124,25 +113,42 @@ def test_wipe_formdata(pub):
     with pytest.raises(CommandError):
         call_command('wipe_data', '--all-tenants')
 
+    # dry-run mode
+    output = io.StringIO()
+    call_command('wipe_data', '--domain=example.net', '--all', stdout=output)
+    assert form_1.data_class().count() == 1
+    assert form_2.data_class().count() == 1
+    assert (
+        output.getvalue()
+        == '''SIMULATION MODE: no actual wiping will happen.
+(use --no-simulate after checking results)
+
+example: 1
+example2: 1
+'''
+    )
+
     # test with no options
-    call_command('wipe_data', '--domain=example.net')
+    call_command('wipe_data', '--domain=example.net', '--no-simulate')
     assert form_1.data_class().count() == 1
     assert form_2.data_class().count() == 1
 
     # wipe one form formdatas
-    call_command('wipe_data', '--domain=example.net', '--forms=%s' % form_1.url_name)
+    call_command('wipe_data', '--domain=example.net', '--no-simulate', '--forms=%s' % form_1.url_name)
     assert form_1.data_class().count() == 0
     assert form_2.data_class().count() == 1
 
     # wipe all formdatas
-    call_command('wipe_data', '--domain=example.net', '--all')
+    call_command('wipe_data', '--domain=example.net', '--no-simulate', '--all')
     assert form_1.data_class().count() == 0
     assert form_2.data_class().count() == 0
 
     # exclude some forms
     formdata_1.store()
     formdata_2.store()
-    call_command('wipe_data', '--domain=example.net', '--all', '--exclude-forms=%s' % form_2.url_name)
+    call_command(
+        'wipe_data', '--domain=example.net', '--no-simulate', '--all', '--exclude-forms=%s' % form_2.url_name
+    )
     assert form_1.data_class().count() == 0
     assert form_2.data_class().count() == 1
 
@@ -499,26 +505,6 @@ def test_runjob(pub):
         call_command('runjob', '--domain=example.net', '--job-id=%s' % job.id, '--force-replay', '--raise')
 
 
-def test_ctl_print_help(capsys):
-    ctl = wcs.qommon.ctl.Ctl(cmd_prefixes=['wcs.ctl'])
-    with pytest.raises(SystemExit):
-        ctl.print_help()
-    captured = capsys.readouterr()
-    assert 'runscript' in captured.out
-
-
-def test_ctl_no_command(capsys):
-    ctl = wcs.qommon.ctl.Ctl(cmd_prefixes=['wcs.ctl'])
-    old_argv, sys.argv = sys.argv, ['wcsctl']
-    try:
-        with pytest.raises(SystemExit):
-            ctl.run(None)
-        captured = capsys.readouterr()
-        assert 'error: You must use a command' in captured.err
-    finally:
-        sys.argv = old_argv
-
-
 def test_dbshell(pub):
     with pytest.raises(CommandError):
         call_command('dbshell')  # missing tenant name

tests/test_fields.py

@@ -153,11 +153,11 @@ def test_text(pub):
 
     form = Form(use_tokens=False)
     fields.TextField(display_mode='rich').add_to_form(form)
-    assert 'data-godo-schema="full"' in str(form.render())
+    assert PyQuery(str(form.render()))('godo-editor[schema=full]')
 
     form = Form(use_tokens=False)
     fields.TextField(display_mode='basic-rich').add_to_form(form)
-    assert 'data-godo-schema="basic"' in str(form.render())
+    assert PyQuery(str(form.render()))('godo-editor[schema=basic]')
 
 
 def test_text_anonymise(pub):

tests/test_formdef.py

@@ -19,6 +19,7 @@ from wcs.fields import DateField, ItemField, StringField
 from wcs.formdef import FormDef, get_formdefs_of_all_kinds, update_storage_all_formdefs
 from wcs.qommon.http_request import HTTPRequest
 from wcs.qommon.upload_storage import PicklableUpload
+from wcs.variables import LazyFormDef
 from wcs.wf.form import FormWorkflowStatusItem, WorkflowFormEvolutionPart, WorkflowFormFieldsFormDef
 from wcs.workflows import (
     AttachmentEvolutionPart,
@@ -662,3 +663,39 @@ def test_update_storage_all_formdefs(pub):
     with mock.patch('wcs.formdef.FormDef.update_storage') as update_storage:
         update_storage_all_formdefs(pub)
         assert update_storage.call_count == 10
+
+
+def test_lazy_formdef(pub):
+    FormDef.wipe()
+
+    formdef = FormDef()
+    formdef.name = 'test form'
+
+    lazy_formdef = LazyFormDef(formdef)
+    assert lazy_formdef.publication_disabled is False
+    assert lazy_formdef.publication_datetime is None
+    assert lazy_formdef.publication_expiration_datetime is None
+
+    formdef.disabled = True
+    assert lazy_formdef.publication_disabled is True
+    assert lazy_formdef.publication_datetime is None
+    assert lazy_formdef.publication_expiration_datetime is None
+
+    formdef.disabled = False
+    formdef.publication_date = '2000-01-01'
+    assert lazy_formdef.publication_disabled is False
+    assert lazy_formdef.publication_datetime == datetime.datetime(2000, 1, 1)
+    assert lazy_formdef.publication_expiration_datetime is None
+
+    formdef.disabled = False
+    formdef.publication_date = '2200-01-01'
+    assert lazy_formdef.publication_disabled is True
+    assert lazy_formdef.publication_datetime == datetime.datetime(2200, 1, 1)
+    assert lazy_formdef.publication_expiration_datetime is None
+
+    formdef.disabled = False
+    formdef.publication_date = '2000-01-01'
+    formdef.expiration_date = '2000-01-01 10:00'
+    assert lazy_formdef.publication_disabled is True
+    assert lazy_formdef.publication_datetime == datetime.datetime(2000, 1, 1)
+    assert lazy_formdef.publication_expiration_datetime == datetime.datetime(2000, 1, 1, 10, 0)

tests/test_misc.py

@@ -1,4 +1,5 @@
 import datetime
+import decimal
 import json
 import math
 import os
@@ -27,6 +28,7 @@ from wcs.qommon.misc import (
     format_time,
     get_as_datetime,
     normalize_geolocation,
+    parse_decimal,
     parse_isotime,
     simplify,
     validate_phone_fr,
@@ -707,3 +709,49 @@ def test_validate_phone_fr(pub):
 
     assert all(validate_phone_fr(pn) for pn in valid)
     assert all(not validate_phone_fr(pn) for pn in invalid)
+
+
+@pytest.mark.parametrize(
+    'value, expected',
+    [
+        ('1.3', decimal.Decimal('1.3')),
+        ('1,5', decimal.Decimal(1.5)),
+        (True, decimal.Decimal(0)),
+        (False, decimal.Decimal(0)),
+        (None, 0),
+        ('', 0),
+    ],
+    ids=['1.3', '1,5', 'True', 'False', 'None', 'empty-string'],
+)
+def test_parse_decimal_base(value, expected):
+    assert parse_decimal(value) == expected
+
+
+@pytest.mark.parametrize(
+    'value, expected',
+    [
+        ('1.3', decimal.Decimal('1.3')),
+        ('1,5', decimal.Decimal(1.5)),
+        (True, decimal.Decimal(0)),
+        (False, decimal.Decimal(0)),
+        (None, None),
+        ('', None),
+    ],
+    ids=['1.3', '1,5', 'True', 'False', 'None', 'empty-string'],
+)
+def test_parse_decimal_keep_none(value, expected):
+    assert parse_decimal(value, keep_none=True) == expected
+
+
+@pytest.mark.parametrize(
+    'value, exception',
+    [
+        (None, TypeError),
+        ('', decimal.InvalidOperation),
+        ('xyz', decimal.InvalidOperation),
+    ],
+    ids=['None', 'empty-string', 'alpha'],
+)
+def test_parse_decimal_do_raise(value, exception):
+    with pytest.raises(exception):
+        parse_decimal(value, do_raise=True)

tests/test_templates.py

@@ -2,6 +2,8 @@ import datetime
 import html
 import os
 import string
+import subprocess
+from unittest import mock
 
 import pytest
 from django.test import override_settings
@@ -1664,3 +1666,115 @@ def test_check_no_duplicates(pub):
     assert pub.loggederror_class.count() == 1
     logged_error = pub.loggederror_class.select()[0]
     assert logged_error.summary == '|check_no_duplicates not used on a list (12)'
+
+
+def test_details_format(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'foo-details'
+    formdef.fields = [fields.StringField(id='1', label='String')]
+    formdef.store()
+    formdef.data_class().wipe()
+    formdata = formdef.data_class()()
+    formdata.data = {'1': 'foo'}
+    formdata.just_created()
+    formdata.store()
+
+    pub.substitutions.feed(formdata)
+    context = pub.substitutions.get_context_variables(mode='lazy')
+    tmpl = Template('{{ form_details|details_format }}')
+    pub.loggederror_class.wipe()
+    assert tmpl.render(context) == ''
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.summary == '|details_format called without specifying a format'
+
+    tmpl = Template('{{ form_details|details_format:"xxx" }}')
+    pub.loggederror_class.wipe()
+    assert tmpl.render(context) == ''
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.summary == '|details_format called with unknown format (xxx)'
+
+    tmpl = Template('{{ form_details|details_format:"text" }}')
+    pub.loggederror_class.wipe()
+    assert tmpl.render(context) == 'String:\n  foo'
+    assert pub.loggederror_class.count() == 0
+
+
+@pytest.mark.parametrize('image_format', ['jpeg', 'png', 'pdf'])
+def test_convert_image_format(pub, image_format):
+    with pub.complex_data():
+        img = Template('{{ url|qrcode|convert_image_format:"%s" }}' % image_format).render(
+            {'url': 'http://example.com/', 'allow_complex': True}
+        )
+        assert pub.has_cached_complex_data(img)
+        value = pub.get_cached_complex_data(img)
+        assert value.orig_filename == 'qrcode.%s' % image_format
+        assert value.content_type == {'jpeg': 'image/jpeg', 'png': 'image/png', 'pdf': 'application/pdf'}.get(
+            image_format
+        )
+        with value.get_file_pointer() as fp:
+            if image_format in ('jpeg', 'png'):
+                img = PIL.Image.open(fp)
+                assert img.format == image_format.upper()
+                assert img.size == (330, 330)
+                assert (
+                    zbar_decode_qrcode(img, symbols=[ZBarSymbol.QRCODE])[0].data.decode()
+                    == 'http://example.com/'
+                )
+            else:
+                assert b'%PDF-' in fp.read()[:200]
+
+
+def test_convert_image_format_no_name(pub):
+    with pub.complex_data():
+        img = Template('{{ url|qrcode|rename_file:""|convert_image_format:"jpeg" }}').render(
+            {'url': 'http://example.com/', 'allow_complex': True}
+        )
+        assert pub.has_cached_complex_data(img)
+        value = pub.get_cached_complex_data(img)
+        assert value.orig_filename == 'file.jpeg'
+
+
+def test_convert_image_format_errors(pub):
+    pub.loggederror_class.wipe()
+    with pub.complex_data():
+        img = Template('{{ "xxx"|convert_image_format:"gif" }}').render({'allow_complex': True})
+        assert pub.has_cached_complex_data(img)
+        assert pub.get_cached_complex_data(img) is None
+    assert pub.loggederror_class.count() == 1
+    assert (
+        pub.loggederror_class.select()[0].summary
+        == '|convert_image_format: unknown format (must be one of jpeg, pdf, png)'
+    )
+
+    pub.loggederror_class.wipe()
+    with pub.complex_data():
+        img = Template('{{ "xxx"|convert_image_format:"jpeg" }}').render({'allow_complex': True})
+        assert pub.has_cached_complex_data(img)
+        assert pub.get_cached_complex_data(img) is None
+    assert pub.loggederror_class.count() == 1
+    assert pub.loggederror_class.select()[0].summary == '|convert_image_format: missing input'
+
+    pub.loggederror_class.wipe()
+    with mock.patch('subprocess.run', side_effect=FileNotFoundError()):
+        with pub.complex_data():
+            img = Template('{{ url|qrcode|convert_image_format:"jpeg" }}').render(
+                {'url': 'http://example.com/', 'allow_complex': True}
+            )
+            assert pub.has_cached_complex_data(img)
+            assert pub.get_cached_complex_data(img) is None
+    assert pub.loggederror_class.count() == 1
+    assert pub.loggederror_class.select()[0].summary == '|convert_image_format: not supported'
+
+    pub.loggederror_class.wipe()
+    with mock.patch(
+        'subprocess.run', side_effect=subprocess.CalledProcessError(returncode=-1, cmd='xx', stderr=b'xxx')
+    ):
+        with pub.complex_data():
+            img = Template('{{ url|qrcode|convert_image_format:"jpeg" }}').render(
+                {'url': 'http://example.com/', 'allow_complex': True}
+            )
+            assert pub.has_cached_complex_data(img)
+            assert pub.get_cached_complex_data(img) is None
+    assert pub.loggederror_class.count() == 1
+    assert pub.loggederror_class.select()[0].summary == '|convert_image_format: conversion error (xxx)'

tests/test_testdef.py

@@ -8,13 +8,14 @@ from unittest import mock
 
 import pytest
 import responses
-from django.utils.timezone import make_aware
+from django.utils.timezone import make_aware, now
 
 from wcs import fields, workflow_tests
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.formdef import FormDef
 from wcs.qommon.http_request import HTTPRequest
+from wcs.qommon.storage import Equal
 from wcs.qommon.upload_storage import PicklableUpload
 from wcs.testdef import TestDef, TestDefXmlProxy, TestError, TestResult, WebserviceResponse
 from wcs.wscalls import NamedWsCall
@@ -34,6 +35,8 @@ def pub():
     BlockDef.wipe()
     WebserviceResponse.wipe()
     NamedWsCall.wipe()
+    TestResult.wipe()
+    TestDef.wipe()
     return pub
 
 
@@ -146,6 +149,66 @@ def test_testdef_result_migrate_legacy_json(pub):
     ]
 
 
+def test_testdef_result_clean(pub, freezer):
+    def make_result(formdef_id, success):
+        test_result = TestResult()
+        test_result.object_type = 'formdef'
+        test_result.object_id = formdef_id
+        test_result.timestamp = now()
+        test_result.success = success
+        test_result.reason = 'xxx'
+        test_result.store()
+
+    # FormDef 1
+    freezer.move_to('2024-01-25 12:00')
+    for i in range(20):
+        make_result(formdef_id='1', success=True)
+
+    # FormDef 2
+    freezer.move_to('2024-01-10 12:00')
+    for i in range(15):
+        make_result(formdef_id='2', success=True)
+
+    for i in range(15):
+        freezer.move_to('2024-01-15 12:%s' % i)
+        make_result(formdef_id='2', success=True)
+
+    # FormDef 3
+    freezer.move_to('2024-01-10 12:00')
+    for i in range(15):
+        make_result(formdef_id='3', success=False)
+
+    freezer.move_to('2024-01-11 12:00')
+    make_result(formdef_id='3', success=True)
+
+    freezer.move_to('2024-01-12 12:00')
+    for i in range(5):
+        make_result(formdef_id='3', success=False)
+
+    freezer.move_to('2024-01-25 12:00')
+    for i in range(10):
+        make_result(formdef_id='3', success=False)
+
+    freezer.move_to('2024-02-01 12:00')
+    TestResult.clean()
+
+    # no deletion for FormDef 1
+    results_formdef1 = TestResult.select(clause=[Equal('object_id', '1')])
+    assert len(results_formdef1) == 20
+
+    # 10 most recent results were kept for FormDef 2
+    results_formdef2 = TestResult.select(clause=[Equal('object_id', '2')])
+    assert len(results_formdef2) == 10
+    assert all(x.timestamp.day == 15 for x in results_formdef2)
+
+    # all recently failed results were kept for FormDef 3, including last success
+    results_formdef3 = TestResult.select(clause=[Equal('object_id', '3')])
+    assert len(results_formdef3) == 16
+    assert len([x for x in results_formdef3 if x.success]) == 1
+    assert len([x for x in results_formdef3 if x.timestamp.day == 12]) == 5
+    assert len([x for x in results_formdef3 if x.timestamp.day == 25]) == 10
+
+
 def test_testdef_create_from_formdata_boolean(pub):
     formdef = FormDef()
     formdef.name = 'test title'

tests/test_widgets.py

@@ -595,7 +595,7 @@ def test_wysiwygwidget_img():
 def test_mini_rich_text_widget():
     widget = MiniRichTextWidget('test')
     form = MockHtmlForm(widget)
-    assert 'data-godo-schema="basic"' in form.as_html
+    assert PyQuery(form.as_html)('godo-editor[schema=basic]')
 
 
 def test_mini_rich_text_widget_maxlength():
@@ -613,7 +613,7 @@ def test_mini_rich_text_widget_maxlength():
 def test_rich_text_widget():
     widget = RichTextWidget('test')
     form = MockHtmlForm(widget)
-    assert 'data-godo-schema="full"' in form.as_html
+    assert PyQuery(form.as_html)('godo-editor[schema=full]')
 
 
 def test_select_hint_widget():

tests/test_workflow_tests.py

@@ -5,7 +5,7 @@ import pytest
 from wcs import workflow_tests
 from wcs.formdef import FormDef, fields
 from wcs.qommon.http_request import HTTPRequest
-from wcs.testdef import TestDef
+from wcs.testdef import TestDef, WebserviceResponse
 from wcs.wf.jump import JumpWorkflowStatusItem
 from wcs.workflow_tests import WorkflowTestError
 from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef, WorkflowStatusItem
@@ -64,6 +64,32 @@ def test_workflow_tests_ignore_unsupported_items(pub, monkeypatch):
     assert str(excinfo.value) == 'Form should be in status "End status" but is in status "New status".'
 
 
+def test_workflow_tests_no_actions(pub):
+    user = pub.user_class(name='test user')
+    user.store()
+
+    workflow = Workflow(name='Workflow One')
+    workflow.add_status(name='New status')
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.agent_id = user.id
+    testdef.workflow_tests.actions = []
+
+    with mock.patch('wcs.workflow_tests.WorkflowTests.run') as mocked_run:
+        testdef.run(formdef)
+    mocked_run.assert_not_called()
+
+
 def test_workflow_tests_button_click(pub):
     role = pub.role_class(name='test role')
     role.store()
@@ -230,6 +256,11 @@ def test_workflow_tests_automatic_jump_timeout(pub):
     jump.timeout = 120 * 60  # 2 hours
     jump.condition = {'type': 'django', 'value': 'form_receipt_datetime|age_in_days >= 1'}
 
+    sendmail = new_status.add_action('sendmail')
+    sendmail.to = ['test@example.org']
+    sendmail.subject = 'In new status'
+    sendmail.body = 'xxx'
+
     workflow.store()
 
     formdef = FormDef()
@@ -311,7 +342,9 @@ def test_workflow_tests_sendmail(mocked_send_email, pub):
     testdef = TestDef.create_from_formdata(formdef, formdata)
     testdef.agent_id = user.id
     testdef.workflow_tests.actions = [
-        workflow_tests.AssertEmail(subject_strings=['In new status'], body_strings=['xxx']),
+        workflow_tests.AssertEmail(
+            addresses=['test@example.org'], subject_strings=['In new status'], body_strings=['xxx']
+        ),
         workflow_tests.ButtonClick(button_name='Go to end status'),
         workflow_tests.AssertStatus(status_name='End status'),
         workflow_tests.AssertEmail(subject_strings=['end status'], body_strings=['yyy']),
@@ -342,6 +375,25 @@ def test_workflow_tests_sendmail(mocked_send_email, pub):
         testdef.run(formdef)
     assert str(excinfo.value) == 'Email body does not contain "bli".'
 
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertEmail(addresses=['test@example.org', 'other@example.org']),
+    ]
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Email was not sent to address "other@example.org".'
+    assert 'Email addresses: test@example.org' in excinfo.value.details
+
+    testdef.workflow_tests.actions = [
+        workflow_tests.ButtonClick(button_name='Go to end status'),
+        workflow_tests.AssertEmail(subject_strings=['In new status'], body_strings=['xxx']),
+        workflow_tests.AssertEmail(subject_strings=['end status'], body_strings=['yyy']),
+    ]
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Email subject does not contain "In new status".'
+
 
 def test_workflow_tests_backoffice_fields(pub):
     user = pub.user_class(name='test user')
@@ -393,3 +445,101 @@ def test_workflow_tests_backoffice_fields(pub):
     with pytest.raises(WorkflowTestError) as excinfo:
         testdef.run(formdef)
     assert str(excinfo.value) == 'Field bo2 not found (expected value "abc").'
+
+
+def test_workflow_tests_webservice(pub):
+    user = pub.user_class(name='test user')
+    user.store()
+
+    workflow = Workflow(name='Workflow One')
+    new_status = workflow.add_status(name='New status')
+    end_status = workflow.add_status(name='End status')
+
+    wscall = new_status.add_action('webservice_call')
+    wscall.url = 'http://example.com/json'
+    wscall.varname = 'test_webservice'
+    wscall.qs_data = {'a': 'b'}
+
+    jump = new_status.add_action('jump')
+    jump.status = end_status.id
+    jump.condition = {'type': 'django', 'value': 'form_workflow_data_test_webservice_response_foo == "bar"'}
+
+    wscall = end_status.add_action('webservice_call')
+    wscall.url = 'http://example.com/json'
+    wscall.varname = 'test_webservice_2'
+    wscall.method = 'POST'
+    wscall.post_data = {'a': 'b'}
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.agent_id = user.id
+    testdef.store()
+
+    response = WebserviceResponse()
+    response.testdef_id = testdef.id
+    response.name = 'Fake response'
+    response.url = 'http://example.com/json'
+    response.payload = '{"foo": "foo"}'
+    response.store()
+
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertStatus(status_name='End status'),
+        workflow_tests.AssertWebserviceCall(webservice_response_id=response.id, call_count=1),
+    ]
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Form should be in status "End status" but is in status "New status".'
+
+    response.payload = '{"foo": "bar"}'
+    response.store()
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Webservice response Fake response was used 2 times (instead of 1).'
+
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertWebserviceCall(webservice_response_id=response.id, call_count=2),
+    ]
+
+    testdef.run(formdef)
+
+    response.qs_data = {'a': 'b'}
+    response.store()
+
+    response2 = WebserviceResponse()
+    response2.testdef_id = testdef.id
+    response2.name = 'Fake response 2'
+    response2.url = 'http://example.com/json'
+    response2.payload = '{}'
+    response2.method = 'POST'
+    response2.store()
+
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertWebserviceCall(webservice_response_id=response.id, call_count=1),
+        workflow_tests.AssertWebserviceCall(webservice_response_id=response2.id, call_count=1),
+    ]
+
+    testdef.run(formdef)
+
+    testdef.workflow_tests.actions = reversed(testdef.workflow_tests.actions)
+
+    testdef.run(formdef)
+
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertWebserviceCall(webservice_response_id=response.id, call_count=1),
+        workflow_tests.AssertWebserviceCall(webservice_response_id=response.id, call_count=1),
+    ]
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Webservice response Fake response was used 0 times (instead of 1).'

tests/workflow/test_backoffice_field.py

@@ -818,7 +818,7 @@ def test_set_backoffice_field_card_item(pub):
     assert formdata.data.get('bo1_structured') is None
     assert pub.loggederror_class.count() == 1
     logged_error = pub.loggederror_class.select()[0]
-    assert logged_error.summary.startswith('Failed to convert')
+    assert logged_error.summary == "Failed to assign field (bo1): unknown card value ('xxx')"
 
     # reset, and get empty value
     formdata.data = {}

tests/workflow/test_display_message.py

@@ -58,7 +58,7 @@ def test_display_message_rich_text(pub):
 
     app = login(get_app(pub))
     resp = app.get(display_message.get_admin_url())
-    assert resp.pyquery('textarea[data-godo-schema]')  # godo
+    assert resp.pyquery('godo-editor')  # godo
 
     display_message.message = '<table><tr><td>hello world</td></tr></table>'
     workflow.store()
@@ -76,13 +76,13 @@ def test_display_message_rich_text(pub):
     display_message.message = '<ul>{% for item in lists %}<li>{{ item }}</li>{% endfor %}</ul>'
     workflow.store()
     resp = app.get(display_message.get_admin_url())
-    assert resp.pyquery('textarea:not([data-config]):not([data-godo-schema])')  # plain textarea
+    assert resp.pyquery('textarea:not([data-config])')  # plain textarea
 
     pub.site_options.set('options', 'rich-text-wf-displaymsg', 'auto-textarea')
     with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
         pub.site_options.write(fd)
     resp = app.get(display_message.get_admin_url())
-    assert resp.pyquery('textarea:not([data-config]):not([data-godo-schema])')  # plain textarea
+    assert resp.pyquery('textarea:not([data-config])')  # plain textarea
 
     pub.site_options.set('options', 'rich-text-wf-displaymsg', 'auto-ckeditor')
     with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
@@ -102,10 +102,10 @@ def test_display_message_rich_text(pub):
     with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
         pub.site_options.write(fd)
     resp = app.get(display_message.get_admin_url())
-    assert resp.pyquery('textarea[data-godo-schema]')  # godo
+    assert resp.pyquery('godo-editor')  # godo
 
     pub.site_options.set('options', 'rich-text-wf-displaymsg', 'textarea')
     with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
         pub.site_options.write(fd)
     resp = app.get(display_message.get_admin_url())
-    assert resp.pyquery('textarea:not([data-config]):not([data-godo-schema])')  # plain textarea
+    assert resp.pyquery('textarea:not([data-config])')  # plain textarea

tests/workflow/test_editable.py

@@ -0,0 +1,81 @@
+import pytest
+from quixote import cleanup
+
+from wcs.formdef import FormDef
+from wcs.workflows import Workflow
+
+from ..utilities import clean_temporary_pub, create_temporary_pub, get_app
+
+
+def setup_module(module):
+    cleanup()
+
+
+def teardown_module(module):
+    clean_temporary_pub()
+
+
+@pytest.fixture
+def pub():
+    pub = create_temporary_pub()
+    pub.cfg['language'] = {'language': 'en'}
+    pub.write_cfg()
+    return pub
+
+
+def test_editable_line_details(pub):
+    workflow = Workflow(name='test')
+    st1 = workflow.add_status('Status1', 'st1')
+    action = st1.add_action('editable')
+
+    assert action.get_line_details() == 'not completed'
+
+    role = pub.role_class(name='foorole')
+    role.store()
+    action.by = [role.id]
+    assert action.get_line_details() == '"Edit Form", by foorole'
+
+    action.label = 'foobar'
+    assert action.get_line_details() == '"foobar", by foorole'
+
+
+def test_editable_set_marker(pub):
+    FormDef.wipe()
+    Workflow.wipe()
+
+    workflow = Workflow(name='test')
+    st1 = workflow.add_status('Status1', 'st1')
+    st2 = workflow.add_status('Status2', 'st2')
+    editable = st1.add_action('editable')
+    editable.status = st2.id
+    editable.set_marker_on_status = True
+    editable.by = ['_submitter']
+    back = st2.add_action('choice')
+    back.label = 'go back'
+    back.status = '_previous'
+    back.by = ['_submitter']
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.workflow = workflow
+    formdef.store()
+
+    resp = get_app(pub).get(formdef.get_url())
+    resp = resp.form.submit('submit')  # -> validation page
+    resp = resp.form.submit('submit').follow()  # -> done
+    formdata = formdef.data_class().select()[0]
+
+    # edit
+    resp = resp.form.submit(f'button{editable.id}').follow()
+    resp = resp.form.submit('submit').follow()  # -> done
+
+    formdata.refresh_from_storage()
+    assert formdata.get_status().id == st2.id
+    assert formdata.workflow_data.get('_markers_stack')
+
+    # back
+    resp = resp.form.submit(f'button{editable.id}').follow()
+    formdata.refresh_from_storage()
+    assert formdata.get_status().id == st1.id
+    assert not formdata.workflow_data.get('_markers_stack')

tests/workflow/test_email.py

@@ -13,6 +13,7 @@ from wcs.formdef import FormDef
 from wcs.qommon.http_request import HTTPRequest
 from wcs.qommon.substitution import CompatibilityNamesDict
 from wcs.qommon.upload_storage import PicklableUpload
+from wcs.testdef import TestDef
 from wcs.wf.backoffice_fields import SetBackofficeFieldsWorkflowStatusItem
 from wcs.wf.sendmail import SendmailWorkflowStatusItem
 from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef
@@ -39,6 +40,7 @@ def pub(request):
     pub._set_request(req)
     req.session = sessions.BasicSession(id=1)
     pub.set_config(req)
+    TestDef.wipe()
     return pub
 
 

tests/workflow/test_export_to_model.py

@@ -14,6 +14,7 @@ from webtest import Radio, Upload
 
 from wcs import sessions
 from wcs.blocks import BlockDef
+from wcs.carddef import CardDef
 from wcs.fields import (
     BlockField,
     BoolField,
@@ -512,6 +513,46 @@ def test_interactive_create_doc_and_jump_on_submit(pub):
     assert formdef.data_class().select()[0].status == f'wf-{st1.id}'  # no change
 
 
+def test_interactive_create_doc_update_ts(pub):
+    wf = Workflow(name='create doc')
+    st0 = wf.add_status('Status0')
+    export_to_model = st0.add_action('export_to_model', id='_export_to_model')
+    export_to_model.by = ['_submitter', '_receiver']
+    export_to_model.method = 'interactive'
+    export_to_model.convert_to_pdf = False
+    template_filename = os.path.join(os.path.dirname(__file__), '..', 'template.odt')
+    with open(template_filename, 'rb') as fd:
+        template = fd.read()
+    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
+    upload.fp = io.BytesIO()
+    upload.fp.write(template)
+    upload.fp.seek(0)
+    export_to_model.model_file = UploadedFile(pub.app_dir, None, upload)
+    wf.store()
+
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [StringField(id='1', label='string', varname='toto')]
+    formdef.workflow_id = wf.id
+    formdef.store()
+    formdef.data_class().wipe()
+
+    app = get_app(pub)
+    resp = app.get(formdef.get_url())
+    resp.form['f1'] = 'test'
+    resp = resp.form.submit('submit')
+    resp = resp.form.submit('submit').follow()
+
+    resp2 = resp.form.submit(f'button{export_to_model.id}').follow().follow()
+    assert resp2.body.startswith(b'PK')  # odt
+
+    # emulate js that will update workflow form ts field
+    resp_js = app.get(resp.request.path + 'tsupdate')
+    formdata = formdef.data_class().select()[0]
+    assert resp_js.json['ts'] != resp.forms['wf-actions']['_ts']
+    assert str(formdata.last_update_time.timestamp()) == resp_js.json['ts']
+
+
 def test_workflows_edit_export_to_model_action(pub):
     create_superuser(pub)
     Workflow.wipe()
@@ -674,3 +715,120 @@ def test_workflows_edit_export_to_model_action_check_template(pub):
         .text()
         .startswith('syntax error in Django template: Invalid block')
     )
+
+    # error in field declaration
+    zip_out_fp = io.BytesIO()
+    with open(os.path.join(os.path.dirname(__file__), '../template.odt'), 'rb') as fd:
+        with zipfile.ZipFile(fd, mode='r') as zip_in, zipfile.ZipFile(zip_out_fp, mode='w') as zip_out:
+            for filename in zip_in.namelist():
+                content = zip_in.read(filename)
+                if filename == 'content.xml':
+                    assert b'"[if-any form_name][form_name][end]"' in content
+                    content = content.replace(
+                        b'"[if-any form_name][form_name][end]"', b'"{% if foo %}{{ foo }}{% end %}"'
+                    )
+                zip_out.writestr(filename, content)
+    model_content = zip_out_fp.getvalue()
+    resp.form['model_file'] = Upload('test.odt', model_content)
+    resp = resp.form.submit('submit')
+    assert (
+        resp.pyquery('#form_error_model_file')
+        .text()
+        .startswith('syntax error in Django template: Invalid block')
+    )
+
+    # error in unused field declaration
+    zip_out_fp = io.BytesIO()
+    with open(os.path.join(os.path.dirname(__file__), '../template.odt'), 'rb') as fd:
+        with zipfile.ZipFile(fd, mode='r') as zip_in, zipfile.ZipFile(zip_out_fp, mode='w') as zip_out:
+            for filename in zip_in.namelist():
+                content = zip_in.read(filename)
+                if filename == 'content.xml':
+                    assert b'office:string-value="[if-any form_name][form_name][end]"' in content
+                    content = content.replace(
+                        b'"[if-any form_name][form_name][end]"', b'"{% if foo %}{{ foo }}{% end %}"'
+                    )
+                    content = content.replace(
+                        b'text:user-field-get text:name="nawak"', b'text:user-field-get text:name="other"'
+                    )
+                zip_out.writestr(filename, content)
+    model_content = zip_out_fp.getvalue()
+    resp.form['model_file'] = Upload('test.odt', model_content)
+    resp.form.submit('submit').follow()  # success
+
+
+def test_export_to_model_from_template(pub):
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'card'
+    carddef.fields = [
+        FileField(id='1', label='File', varname='file'),
+        StringField(id='2', label='String', varname='string'),
+    ]
+    carddef.store()
+
+    template_filename = os.path.join(os.path.dirname(__file__), '..', 'template.odt')
+    with open(template_filename, 'rb') as fd:
+        template = fd.read()
+    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
+    upload.fp = io.BytesIO()
+    upload.fp.write(template)
+    upload.fp.seek(0)
+
+    carddata = carddef.data_class()()
+    carddata.data = {'1': upload, '2': 'blah'}
+    carddata.just_created()
+    carddata.store()
+
+    wf = Workflow(name='test_export_to_model_from_template')
+    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
+    wf.backoffice_fields_formdef.fields = [
+        FileField(id='bo1', label='bo field 1', varname='backoffice_file1'),
+    ]
+    st1 = wf.add_status('Status1')
+    wf.store()
+
+    formdef = FormDef()
+    formdef.name = 'foo-export'
+    formdef.fields = [
+        StringField(id='1', label='String', varname='string'),
+    ]
+    formdef.workflow_id = wf.id
+    formdef.store()
+    formdata = formdef.data_class()()
+    formdata.data = {}
+    formdata.just_created()
+    formdata.store()
+    pub.substitutions.feed(formdata)
+
+    item = ExportToModel()
+    item.method = 'non-interactive'
+    item.convert_to_pdf = False
+    item.model_file_mode = 'template'
+    item.model_file_template = '{{cards|objects:"card"|first|get:"form_var_file" }}'
+    item.parent = st1
+    item.backoffice_filefield_id = 'bo1'
+    item.perform(formdata)
+
+    assert 'bo1' in formdata.data
+    fbo1 = formdata.data['bo1']
+    assert fbo1.base_filename == 'template.odt'
+    assert fbo1.content_type == 'application/octet-stream'
+    with zipfile.ZipFile(fbo1.get_file()) as zfile:
+        assert b'foo-export' in zfile.read('content.xml')
+
+    pub.loggederror_class.wipe()
+    item.model_file_template = '{{cards|objects:"card"|first|get:"form_var_string" }}'
+    formdata.data = {}
+    item.perform(formdata)
+    assert 'bo1' not in formdata.data
+    assert pub.loggederror_class.count() == 1
+    assert pub.loggederror_class.select()[0].summary == 'Invalid value obtained for model file (\'blah\')'
+
+    pub.loggederror_class.wipe()
+    item.model_file_template = '{% if foo %}{{ foo }}{% end %}'  # invalid template
+    formdata.data = {}
+    item.perform(formdata)
+    assert 'bo1' not in formdata.data
+    assert pub.loggederror_class.count() == 1
+    assert pub.loggederror_class.select()[0].summary == 'Failed to evaluate template for action'

wcs/admin/data_sources.py

@@ -388,7 +388,7 @@ class NamedDataSourcePage(Directory):
     def preview_block(self):
         get_request().disable_error_notifications = True
         get_request().ignore_session = True
-        get_response().filter = {'raw': True}
+        get_response().raw = True
         data_source = self.datasource.extended_data_source
         try:
             items = get_structured_items(data_source)

wcs/admin/fields.py

@@ -331,7 +331,12 @@ class FieldsDirectory(Directory):
 
     def _q_traverse(self, path):
         if self.page_id:
-            get_response().breadcrumb.append(('pages/%s/' % self.page_id, _('Page')))
+            try:
+                page_field = [x for x in self.objectdef.fields if x.id == self.page_id][0]
+            except IndexError:
+                raise errors.TraversalError()
+            label = misc.ellipsize(page_field.unhtmled_label, 40)
+            get_response().breadcrumb.append(('pages/%s/' % self.page_id, _('Page "%s"') % label))
         else:
             get_response().breadcrumb.append(('fields/', _('Fields')))
         return Directory._q_traverse(self, path)

wcs/admin/forms.py

@@ -1731,6 +1731,11 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
                 context['workflow_options'][field.label] = htmltext('%s') % field.get_view_value(
                     variables_form_data.get(field.id)
                 )
+        page = None
+        for field in self.formdef.fields:
+            if field.key == 'page':
+                page = field
+            field.on_page = page
         context['workflow_roles'] = list(self.get_workflow_roles_elements())
         context['backoffice_submission_roles'] = self._get_roles_label('backoffice_submission_roles')
         if self.formdef.tracking_code_verify_fields:

wcs/admin/settings.py

@@ -39,7 +39,7 @@ from wcs.carddef import CardDef
 from wcs.data_sources import NamedDataSource
 from wcs.fields.map import MapOptionsMixin
 from wcs.formdef import FormDef, FormdefImportError, get_formdefs_of_all_kinds
-from wcs.qommon import _, audit, errors, get_cfg, ident, misc, template
+from wcs.qommon import _, audit, errors, get_cfg, ident, misc, pgettext_lazy, template
 from wcs.qommon.admin.cfg import cfg_submit, hobo_kwargs
 from wcs.qommon.admin.emails import EmailsDirectory
 from wcs.qommon.admin.texts import TextsDirectory
@@ -487,7 +487,7 @@ class SettingsDirectory(AccessControlled, Directory):
         ('data-sources', 'data_sources'),
         'wscalls',
         ('api-access', 'api_access'),
-        ('submission-channels', 'submission_channels'),
+        ('backoffice-submission', 'backoffice_submission'),
     ]
 
     emails = EmailsDirectory()
@@ -636,10 +636,10 @@ class SettingsDirectory(AccessControlled, Directory):
                 _('Geolocation'),
                 _('Configure geolocation and geocoding'),
             )
-        if enabled('submission-channels'):
-            r += htmltext('<dt><a href="submission-channels">%s</a></dt> <dd>%s</dd>') % (
-                _('Submission channels'),
-                _('Configure submission channels related options'),
+        if enabled('backoffice-submission'):
+            r += htmltext('<dt><a href="backoffice-submission">%s</a></dt> <dd>%s</dd>') % (
+                _('Backoffice Submission'),
+                _('Configure backoffice submission related options'),
             )
         if enabled('users'):
             r += htmltext('<dt><a href="users/">%s</a></dt> <dd>%s</dd>') % (_('Users'), _('Configure users'))
@@ -1272,9 +1272,29 @@ $('#form_default-zoom-level').on('change', function() {
             )
             return redirect('.')
 
-    def submission_channels(self):
+    def backoffice_submission(self):
         form = Form(enctype='multipart/form-data')
         submission_channels_cfg = get_cfg('submission-channels', {})
+        backoffice_submission_cfg = get_cfg('backoffice-submission', {})
+        form.add(
+            RadiobuttonsWidget,
+            'sidebar_menu_entry',
+            title=_('Sidebar menu entry'),
+            value=backoffice_submission_cfg.get('sidebar_menu_entry', 'visible'),
+            options=[
+                ('visible', pgettext_lazy('sidebar_menu_entry', 'Visible'), 'visible'),
+                ('hidden', pgettext_lazy('sidebar_menu_entry', 'Hidden'), 'hidden'),
+            ],
+            extra_css_class='widget-inline-radio',
+        )
+        form.add(
+            StringWidget,
+            'redirect',
+            title=_('URL for backoffice submission'),
+            hint=_('Leave empty to use native screen.'),
+            value=backoffice_submission_cfg.get('redirect', ''),
+            size=80,
+        )
         form.add(
             CheckboxWidget,
             'include-in-global-listing',
@@ -1288,18 +1308,15 @@ $('#form_default-zoom-level').on('change', function() {
             return redirect('.')
 
         if not form.is_submitted() or form.has_errors():
-            get_response().breadcrumb.append(('submission-channels', _('Submission channels')))
-            get_response().set_title(_('Submission channels'))
+            get_response().breadcrumb.append(('backoffice-submission', _('Backoffice Submission')))
+            get_response().set_title(_('Backoffice submission settings'))
             r = TemplateIO(html=True)
-            r += htmltext('<h2>%s</h2>') % _('Submission channels')
+            r += htmltext('<h2>%s</h2>') % _('Backoffice submission settings')
             r += form.render()
             return r.getvalue()
         else:
-            cfg_submit(
-                form,
-                'submission-channels',
-                ('include-in-global-listing',),
-            )
+            cfg_submit(form, 'submission-channels', ('include-in-global-listing',))
+            cfg_submit(form, 'backoffice-submission', ('sidebar_menu_entry', 'redirect'))
             return redirect('.')
 
 

wcs/admin/tests.py

@@ -25,6 +25,7 @@ from quixote.directory import Directory
 from quixote.html import TemplateIO, htmltext
 
 from wcs.admin.workflow_tests import WorkflowTestsDirectory
+from wcs.api import posted_json_data_to_formdata_data
 from wcs.backoffice.management import FormBackofficeEditPage, FormBackOfficeStatusPage
 from wcs.backoffice.pagination import pagination_links
 from wcs.forms.common import FormStatusPage
@@ -38,11 +39,13 @@ from wcs.qommon.form import (
     SingleSelectWidget,
     StringWidget,
     TextWidget,
+    UrlWidget,
     WidgetDict,
 )
 from wcs.sql_criterias import Equal, Null, StrictNotEqual
 from wcs.testdef import TestDef, TestError, TestResult, WebserviceResponse
 from wcs.workflow_tests import WorkflowTestError
+from wcs.workflow_traces import WorkflowTrace
 
 
 class TestEditPage(FormBackofficeEditPage):
@@ -301,6 +304,10 @@ class TestsDirectory(Directory):
         self.results = TestResultsDirectory(objectdef)
 
     def _q_traverse(self, path):
+        last_page_path, last_page_label = get_response().breadcrumb.pop()
+        last_page_label = misc.ellipsize(last_page_label, 15, '…')
+        get_response().breadcrumb.append((last_page_path, last_page_label))
+
         get_response().breadcrumb.append(('tests/', _('Tests')))
         return super()._q_traverse(path)
 
@@ -431,10 +438,11 @@ class TestsDirectory(Directory):
 
 
 class TestResultDetailPage(Directory):
-    _q_exports = ['']
+    _q_exports = ['', 'inspect', ('inspect-tool', 'inspect_tool')]
 
-    def __init__(self, component, test_result):
+    def __init__(self, component, test_result, formdef):
         self.result_index = component
+        self.formdef = formdef
 
         try:
             self.result = test_result.results[int(component)]
@@ -460,6 +468,7 @@ class TestResultDetailPage(Directory):
             'workflow_test_action': self.get_workflow_test_action(
                 self.result['details']['workflow_test_action_uuid']
             ),
+            'error_field': self.get_error_field(self.result['details'].get('error_field_id')),
         }
 
         for request in self.result['details'].get('sent_requests', []):
@@ -473,7 +482,11 @@ class TestResultDetailPage(Directory):
                 except IndexError:
                     pass
 
-        return render_to_string('wcs/backoffice/test-result-detail.html', context=context)
+        return template.QommonTemplateResponse(
+            templates=['wcs/backoffice/test-result-detail.html'],
+            context=context,
+            is_django_native=True,
+        )
 
     def get_workflow_test_action(self, action_uuid):
         if not action_uuid or not self.testdef:
@@ -487,6 +500,68 @@ class TestResultDetailPage(Directory):
         action.url = self.testdef.get_admin_url() + 'workflow/#%s' % action.id
         return action
 
+    def get_error_field(self, field_id):
+        if not field_id:
+            return
+
+        try:
+            field = [x for x in self.formdef.fields if x.id == field_id][0]
+        except IndexError:
+            return
+
+        field.url = self.formdef.get_field_admin_url(field)
+        return field
+
+    def inspect(self):
+        formdata_json = json.loads(self.result['formdata'])
+        formdata = self.import_formdata_from_json(formdata_json)
+
+        return FormBackOfficeStatusPage(self.formdef, formdata).inspect()
+
+    def inspect_tool(self):
+        formdata_json = json.loads(self.result['formdata'])
+        formdata = self.import_formdata_from_json(formdata_json)
+
+        return FormBackOfficeStatusPage(self.formdef, formdata).inspect_tool()
+
+    def import_formdata_from_json(self, formdata_json):
+        formdata = self.formdef.data_class()()
+        formdata.receipt_time = formdata_json['receipt_time']
+        formdata.user_id = formdata_json.get('user', {}).get('id')
+        formdata.digests = formdata_json['digests']
+        formdata.backoffice_submission = formdata_json['submission']['backoffice']
+        formdata.submission_channel = formdata_json['submission']['channel']
+        formdata.submission_agent_id = formdata_json['submission'].get('agent', {}).get('id')
+        formdata.geolocations = formdata_json.get('geolocations')
+        formdata.criticality_level = formdata_json['criticality_level']
+        formdata.anonymised = formdata_json['anonymised']
+        formdata.workflow_data = formdata_json.get('workflow', {}).get('data', {})
+        formdata.set_auto_fields()
+
+        # load fields
+        formdata.data = posted_json_data_to_formdata_data(self.formdef, formdata_json['fields'])
+
+        # load backoffice fields if any
+        if 'fields' in (formdata_json.get('workflow') or {}):
+            backoffice_data_dict = posted_json_data_to_formdata_data(
+                self.formdef, formdata_json['workflow']['fields']
+            )
+            formdata.data.update(backoffice_data_dict)
+
+        # set status
+        formdata.status = formdata_json['workflow']['real_status']['id']
+
+        formdata.workflow_traces = [
+            WorkflowTrace.import_from_json_dict(x) for x in formdata_json['workflow_traces']
+        ]
+
+        def get_workflow_traces():
+            return formdata.workflow_traces
+
+        formdata.get_workflow_traces = get_workflow_traces
+
+        return formdata
+
 
 class TestResultPage(Directory):
     _q_exports = ['']
@@ -506,7 +581,7 @@ class TestResultPage(Directory):
         return super()._q_traverse(path)
 
     def _q_lookup(self, component):
-        return TestResultDetailPage(component, self.test_result)
+        return TestResultDetailPage(component, self.test_result, self.objectdef)
 
     def _q_index(self):
         get_response().add_javascript(['popup.js'])
@@ -605,6 +680,11 @@ class TestsAfterJob(AfterJob):
                 test.error = str(e)
                 test.exception = e
 
+        formdata_json = test.formdata.get_json_export_dict()
+        formdata_json['criticality_level'] = test.formdata.criticality_level
+        formdata_json['anonymised'] = test.formdata.anonymised
+        formdata_json['workflow_traces'] = [x.get_json_export_dict() for x in test.formdata.workflow_traces]
+
         test_result = TestResult()
         test_result.object_type = objectdef.get_table_name()
         test_result.object_id = objectdef.id
@@ -616,12 +696,14 @@ class TestsAfterJob(AfterJob):
                 'id': test.id,
                 'name': str(test),
                 'error': getattr(test, 'error', None),
+                'formdata': json.dumps(formdata_json, cls=misc.JSONEncoder),
                 'details': {
                     'recorded_errors': test.recorded_errors,
                     'missing_required_fields': test.missing_required_fields,
                     'sent_requests': test.sent_requests,
                     'workflow_test_action_uuid': test.exception.action_uuid if test.exception else None,
                     'error_details': test.exception.details if test.exception else None,
+                    'error_field_id': test.exception.field_id if test.exception else None,
                 },
             }
             for test in testdefs
@@ -644,14 +726,34 @@ class WebserviceResponsePage(Directory):
 
     def _q_index(self):
         form = Form(enctype='multipart/form-data')
-        form.add(StringWidget, 'name', size=50, title=_('Name'), value=self.webservice_response.name)
         form.add(
-            StringWidget,
+            StringWidget, 'name', size=50, title=_('Name'), required=True, value=self.webservice_response.name
+        )
+
+        form.add(
+            UrlWidget,
             'url',
             title=_('URL'),
+            required=True,
             value=self.webservice_response.url,
             size=80,
         )
+
+        def validate_json(value):
+            try:
+                json.loads(value)
+            except ValueError as e:
+                raise ValueError(_('Invalid JSON: %s') % e)
+
+        form.add(
+            TextWidget,
+            'payload',
+            title=_('Response payload (JSON)'),
+            required=True,
+            value=self.webservice_response.payload,
+            validation_function=validate_json,
+        )
+
         form.add(
             WidgetDict,
             'qs_data',
@@ -701,20 +803,6 @@ class WebserviceResponsePage(Directory):
             },
         )
 
-        def validate_json(value):
-            try:
-                json.loads(value)
-            except ValueError as e:
-                raise ValueError(_('Invalid JSON: %s') % e)
-
-        form.add(
-            TextWidget,
-            'payload',
-            title=_('Response payload (JSON)'),
-            value=self.webservice_response.payload,
-            validation_function=validate_json,
-        )
-
         form.add_submit('submit', _('Submit'))
         form.add_submit('cancel', _('Cancel'))
         form.add_media()

wcs/admin/users.py

@@ -401,7 +401,7 @@ class UsersDirectory(Directory):
         r += htmltext('</div>')
 
         if get_request().form.get('ajax') == 'true':
-            get_response().filter = {'raw': True}
+            get_response().raw = True
             return r.getvalue()
 
         ident_methods = get_cfg('identification', {}).get('methods', [])

wcs/admin/workflow_tests.py

@@ -43,7 +43,10 @@ class WorkflowTestActionPage(Directory):
 
         self.action.fill_admin_form(form, self.formdef)
 
-        form.add_submit('submit', _('Submit'))
+        if not form.widgets:
+            form.add_global_errors([htmltext(self.action.empty_form_error)])
+        else:
+            form.add_submit('submit', _('Submit'))
         form.add_submit('cancel', _('Cancel'))
 
         if form.get_widget('cancel').parse():

wcs/admin/wscalls.py

@@ -161,7 +161,7 @@ class NamedWsCallPage(Directory):
     def usage(self):
         get_request().disable_error_notifications = True
         get_request().ignore_session = True
-        get_response().filter = {'raw': True}
+        get_response().raw = True
 
         usage = {}
 

wcs/api.py

@@ -255,7 +255,7 @@ class ApiFormPageMixin:
         if not self.is_webhook:
             self.check_access()
         try:
-            formdata = self.formdef.data_class().get(component)
+            formdata = self.formdef.data_class().get_by_id(component)
         except KeyError:
             raise TraversalError()
         return ApiFormdataPage(self.formdef, formdata, custom_view=self._view)

wcs/backoffice/cards.py

@@ -52,7 +52,11 @@ class CardDefOptionsDirectory(OptionsDirectory):
         criterias = [
             StrictNotEqual('status', 'draft'),
         ]
-        kwargs = {}
+        kwargs = {
+            'hint': 'The template should produce a unique identifier with only letters, '
+            'lowercase or uppercase (without accents), digits, dashes and underscores. '
+            'Other characters will be automatically replaced or removed.'
+        }
         if self.formdef.data_class().exists(criterias):
             kwargs['readonly'] = True
             kwargs['hint'] = _('Identifier cannot be modified if there are existing cards.')

wcs/backoffice/data_management.py

@@ -365,7 +365,7 @@ class CardPage(FormPage):
             return view_lookup_response
 
         try:
-            filled = self.formdef.data_class().get(component)
+            filled = self.formdef.data_class().get_by_id(component)
         except KeyError:
             raise errors.TraversalError()
         return CardBackOfficeStatusPage(self.formdef, filled, parent_view=self)
@@ -460,17 +460,22 @@ class ImportFromCsvAfterJob(AfterJob):
             data_instance.data = {}
             block_data = {}
 
+            data_field_ids = set()
+
             for i, field in enumerate(carddef_fields):
+                block_field = getattr(field, 'block_field', None)
                 value = csv_line[i].strip()
                 # skip empty values
                 if not value:
+                    if not block_field:
+                        data_field_ids.add(field.id)
                     continue
                 # skip unsupported field types
                 if field.convert_value_from_str is None:
                     continue
-                block_field = getattr(field, 'block_field', None)
                 if not block_field:
                     field.set_value(data_instance.data, field.convert_value_from_str(value))
+                    data_field_ids.add(field.id)
                     continue
 
                 # field in a BlockField
@@ -492,15 +497,43 @@ class ImportFromCsvAfterJob(AfterJob):
             }
             data_instance.submission_agent_id = self.submission_agent_id
             data_instance.submission_channel = 'file-import'
-            data_instance.just_created()
-            data_instance.store()
 
-            get_publisher().reset_formdata_state()
-            get_publisher().substitutions.feed(data_instance)
+            new_card = True
+            if self.carddef.id_template:
+                # check id is unique
+                data_instance.set_auto_fields()
+                try:
+                    carddata_with_same_id = self.carddef.data_class().get_by_id(data_instance.id_display)
+                except KeyError:
+                    pass  # unique id, fine
+                else:
+                    # overwrite (only fields from CSV columns, not unsupported or backoffice fields)
+                    new_card = False
+                    orig_data = copy.copy(carddata_with_same_id.data)
+                    for data_field_id in data_field_ids:
+                        for key in (
+                            str(data_field_id),
+                            f'{data_field_id}_display',
+                            f'{data_field_id}_structured',
+                        ):
+                            carddata_with_same_id.data[key] = data_instance.data.get(key)
+                    ContentSnapshotPart.take(
+                        formdata=carddata_with_same_id, old_data=orig_data, user=self.submission_agent_id
+                    )
+                    carddata_with_same_id.record_workflow_event('csv-import-updated')
+                    carddata_with_same_id.store()
+
+            if new_card:
+                data_instance.just_created()
+                data_instance.store()
+
+                get_publisher().reset_formdata_state()
+                get_publisher().substitutions.feed(data_instance)
+
+                data_instance.refresh_from_storage()
+                data_instance.record_workflow_event('csv-import-created')
+                data_instance.perform_workflow()
 
-            data_instance.refresh_from_storage()
-            data_instance.record_workflow_event('csv-import-created')
-            data_instance.perform_workflow()
             self.increment_count()
 
     def done_action_url(self):

wcs/backoffice/deprecations.py

@@ -88,6 +88,8 @@ class DeprecationsDirectory(Directory):
             'script': _('Filesystem Script'),
             'fields': _('Obsolete field types'),
             'actions': _('Obsolete action types'),
+            'csv-connector': _('CSV connector'),
+            'json-data-store': _('JSON Data Store connector'),
         }
 
     @property
@@ -99,6 +101,8 @@ class DeprecationsDirectory(Directory):
             'python-expression': _('Use Django templates.'),
             'python-prefill': _('Use Django templates.'),
             'python-data-source': _('Use cards.'),
+            'csv-connector': _('Use cards.'),
+            'json-data-store': _('Use cards.'),
             'rtf': _('Use OpenDocument format.'),
             'script': _('Use a dedicated template tags application.'),
             'fields': _('Use block fields to replace tables and ranked order fields.'),
@@ -118,6 +122,8 @@ class DeprecationsDirectory(Directory):
             'script': 'https://doc-publik.entrouvert.com/admin-fonctionnel/elements-deprecies/',
             'fields': 'https://doc-publik.entrouvert.com/admin-fonctionnel/elements-deprecies/',
             'actions': 'https://doc-publik.entrouvert.com/admin-fonctionnel/elements-deprecies/',
+            'csv-connector': 'https://doc-publik.entrouvert.com/admin-fonctionnel/elements-deprecies/',
+            'json-data-store': 'https://doc-publik.entrouvert.com/admin-fonctionnel/elements-deprecies/',
         }
 
 
@@ -253,6 +259,10 @@ class DeprecationsScanAfterJob(AfterJob):
                                 source=source,
                             )
                             break
+                if action.key == 'webservice_call':
+                    self.check_remote_call_url(
+                        action.url, location_label=location_label, url=url, source=source
+                    )
 
             for global_action in workflow.global_actions or []:
                 location_label = '%s / %s' % (workflow.name, _('trigger in %s') % global_action.name)
@@ -294,6 +304,10 @@ class DeprecationsScanAfterJob(AfterJob):
             url = named_ws_call.get_admin_url()
             for string in named_ws_call.get_computed_strings():
                 self.check_string(string, location_label=location_label, url=url, source=source)
+            if named_ws_call.request and named_ws_call.request.get('url'):
+                self.check_remote_call_url(
+                    named_ws_call.request['url'], location_label=location_label, url=url, source=source
+                )
             self.increment_count()
 
         for mail_template in mail_templates:
@@ -337,6 +351,7 @@ class DeprecationsScanAfterJob(AfterJob):
             self.check_string(
                 data_source.get('value'), location_label, url, python_check=False, source=source
             )
+            self.check_remote_call_url(data_source.get('value'), location_label, url, source=source)
 
     def check_string(self, string, location_label, url, source, python_check=True):
         if not isinstance(string, str):
@@ -361,6 +376,16 @@ class DeprecationsScanAfterJob(AfterJob):
             if re.findall(r'\Wscript\.\w', string):
                 self.add_report_line(location_label=location_label, url=url, category='script', source=source)
 
+    def check_remote_call_url(self, wscall_url, location_label, url, source):
+        if 'csvdatasource/' in (wscall_url or ''):
+            self.add_report_line(
+                location_label=location_label, url=url, category='csv-connector', source=source
+            )
+        if 'jsondatastore/' in (wscall_url or ''):
+            self.add_report_line(
+                location_label=location_label, url=url, category='json-data-store', source=source
+            )
+
     def add_report_line(self, **kwargs):
         if kwargs not in self.report_lines:
             self.report_lines.append(kwargs)

wcs/backoffice/management.py

@@ -717,7 +717,7 @@ class ManagementDirectory(Directory):
 
         if get_request().form.get('ajax') == 'true':
             get_request().ignore_session = True
-            get_response().filter = {'raw': True}
+            get_response().raw = True
             return r.getvalue()
 
         get_response().filter['sidebar'] = self.get_global_listing_sidebar(
@@ -1240,15 +1240,16 @@ class FormPage(Directory, TempfileDirectoryMixin):
             result = htmltext('<div class="widget operator-and-value-widget">')
             result += htmltext('<div class="title-and-operator">')
             result += filter_widget.render_title(filter_widget.get_title())
-            result += htmltext('<div class="operator">')
-            operator_widget = SingleSelectWidget(
-                filter_field_operator_key,
-                options=[(o[0], o[1], o[0]) for o in operators],
-                value=filter_field_operator,
-                render_br=False,
-            )
-            result += operator_widget.render_content()
-            result += htmltext('</div>')
+            if operators:
+                result += htmltext('<div class="operator">')
+                operator_widget = SingleSelectWidget(
+                    filter_field_operator_key,
+                    options=[(o[0], o[1], o[0]) for o in operators],
+                    value=filter_field_operator,
+                    render_br=False,
+                )
+                result += operator_widget.render_content()
+                result += htmltext('</div>')
             result += htmltext('</div>')
             result += htmltext('<div class="value">')
             result += filter_widget.render_content()
@@ -1330,13 +1331,14 @@ class FormPage(Directory, TempfileDirectoryMixin):
                         filtered_user = None
                     filtered_user_value = filtered_user.display_name if filtered_user else _('Unknown')
                     options += [(filter_field_value, filtered_user_value, filter_field_value)]
-                r += SingleSelectWidget(
+                widget = SingleSelectWidget(
                     filter_field_key,
                     title=filter_field.label,
                     options=options,
                     value=filter_field_value,
                     render_br=False,
-                ).render()
+                )
+                r += render_widget(widget, operators=[])
 
             elif filter_field.key == 'submission-agent-id':
                 r += HiddenWidget(filter_field_key, value=filter_field_value).render()
@@ -1356,13 +1358,14 @@ class FormPage(Directory, TempfileDirectoryMixin):
                 options = [('', '', '')] + [
                     (x[0], x[1], x[0]) for x in self.formdef.workflow.get_sorted_functions()
                 ]
-                r += SingleSelectWidget(
+                widget = SingleSelectWidget(
                     filter_field_key,
                     title=filter_field.label,
                     options=options,
                     value=filter_field_value,
                     render_br=False,
-                ).render()
+                )
+                r += render_widget(widget, operators=[])
 
             elif filter_field.key == 'internal-id':
                 widget = StringWidget(
@@ -2442,7 +2445,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
 
         if get_request().form.get('ajax') == 'true':
             get_request().ignore_session = True
-            get_response().filter = {'raw': True}
+            get_response().raw = True
             r = TemplateIO(html=True)
             r += multi_form.render()
             r += get_session().display_message()
@@ -2588,7 +2591,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
             fields=fields,
             selected_filter=selected_filter,
             selected_filter_operator=selected_filter_operator,
-            user_id=user.id,
+            user=user,
             query=query,
             criterias=criterias,
             order_by=order_by,
@@ -2646,7 +2649,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
             fields=fields,
             selected_filter=selected_filter,
             selected_filter_operator=selected_filter_operator,
-            user_id=user.id,
+            user=user,
             query=query,
             criterias=criterias,
             order_by=order_by,
@@ -2678,7 +2681,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
             fields=fields,
             selected_filter=selected_filter,
             selected_filter_operator=selected_filter_operator,
-            user_id=user.id,
+            user=user,
             query=query,
             criterias=criterias,
             order_by=order_by,
@@ -2777,7 +2780,8 @@ class FormPage(Directory, TempfileDirectoryMixin):
         else:
             output = [
                 {
-                    'id': filled.id,
+                    'id': filled.identifier,
+                    'internal_id': str(filled.id),
                     'display_id': filled.get_display_id(),
                     'display_name': filled.get_display_name(),
                     'digest': (filled.digests or {}).get(digest_key),
@@ -3128,7 +3132,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
 
         if get_request().form.get('ajax') == 'true':
             get_request().ignore_session = True
-            get_response().filter = {'raw': True}
+            get_response().raw = True
             return r.getvalue()
 
         page = TemplateIO(html=True)
@@ -3286,7 +3290,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
             return view_lookup_response
 
         try:
-            filled = self.formdef.data_class().get(component)
+            filled = self.formdef.data_class().get_by_id(component)
         except KeyError:
             raise errors.TraversalError()
 
@@ -3316,6 +3320,7 @@ class FormBackOfficeStatusPage(FormStatusPage):
         'live',
         'inspect',
         'tempfile',
+        'tsupdate',
         ('inspect-tool', 'inspect_tool'),
         ('download-as-zip', 'download_as_zip'),
         ('lateral-block', 'lateral_block'),
@@ -3366,13 +3371,13 @@ class FormBackOfficeStatusPage(FormStatusPage):
 
     def lateral_block(self):
         self.check_receiver()
-        get_response().filter = {'raw': True}
+        get_response().raw = True
         response = self.get_lateral_block()
         return response
 
     def user_pending_forms(self):
         self.check_receiver()
-        get_response().filter = {'raw': True}
+        get_response().raw = True
         response = self.get_user_pending_forms()
 
         # preemptive locking of forms
@@ -4157,7 +4162,7 @@ class FormBackOfficeStatusPage(FormStatusPage):
             or get_publisher().get_backoffice_root().is_accessible('workflows')
         ):
             raise errors.AccessForbiddenError()
-        get_response().filter = {'raw': True}
+        get_response().raw = True
         return self.test_tool_result()
 
 
@@ -4525,6 +4530,13 @@ class CsvExportAfterJob(AfterJob):
     label = _('Exporting to CSV file')
 
     def __init__(self, formdef, **kwargs):
+        user = kwargs.pop('user', None)
+        if user and user.is_api_user:
+            kwargs['user_is_api_user'] = True
+            kwargs['user_id'] = user.api_access.id
+        else:
+            kwargs['user_is_api_user'] = False
+            kwargs['user_id'] = user.id if user else None
         super().__init__(formdef_class=formdef.__class__, formdef_id=formdef.id, **kwargs)
         self.file_name = '%s.csv' % formdef.url_name
 
@@ -4595,7 +4607,10 @@ class CsvExportAfterJob(AfterJob):
         query = self.kwargs['query']
         criterias = self.kwargs['criterias']
         order_by = self.kwargs['order_by']
-        user = get_publisher().user_class.get(self.kwargs['user_id'])
+        if self.kwargs['user_is_api_user']:
+            user = ApiAccess.get(self.kwargs['user_id']).get_as_api_user()
+        else:
+            user = get_publisher().user_class.get(self.kwargs['user_id'])
 
         items, total_count = FormDefUI(formdef).get_listing_items(
             fields,

wcs/backoffice/submission.py

@@ -23,14 +23,15 @@ from quixote import get_publisher, get_request, get_response, get_session, redir
 from quixote.directory import Directory
 from quixote.html import TemplateIO, htmltext
 
+from wcs.backoffice.pagination import pagination_links
 from wcs.categories import Category
 from wcs.formdata import FormData
 from wcs.formdef import FormDef
 from wcs.forms.common import FormStatusPage
 from wcs.forms.root import FormPage as PublicFormFillPage
-from wcs.sql_criterias import Equal, StrictNotEqual
+from wcs.sql_criterias import Contains, Equal, StrictNotEqual
 
-from ..qommon import _, errors, misc
+from ..qommon import _, errors, get_cfg, misc, template
 from ..qommon.form import Form, HtmlWidget
 
 
@@ -174,7 +175,7 @@ class FormFillPage(PublicFormFillPage):
         return super()._q_index(*args, **kwargs)
 
     def lateral_block(self):
-        get_response().filter = {'raw': True}
+        get_response().raw = True
         response = self.get_lateral_block()
         return response
 
@@ -427,15 +428,18 @@ class FormFillPage(PublicFormFillPage):
 
 
 class SubmissionDirectory(Directory):
-    _q_exports = ['', 'count']
+    _q_exports = ['', 'pending', 'count']
 
     def _q_traverse(self, path):
         get_response().set_backoffice_section('submission')
+        get_response().breadcrumb.append(('submission/', _('Submission')))
         return super()._q_traverse(path)
 
     def is_accessible(self, user, traversal=False):
         if not user.can_go_in_backoffice():
             return False
+        if traversal is False and get_cfg('backoffice-submission', {}).get('sidebar_menu_entry') == 'hidden':
+            return False
         # check user has at least one role set for backoffice submission
         for role_id in user.roles or []:
             ids = FormDef.get_ids_with_indexed_value('backoffice_submission_roles', role_id)
@@ -443,9 +447,10 @@ class SubmissionDirectory(Directory):
                 return True
         return False
 
-    def get_submittable_formdefs(self):
+    def get_submittable_formdefs(self, prefetch=True):
         user = get_request().user
 
+        agent_ids = set()
         list_forms = []
         for formdef in FormDef.select(order_by='name', ignore_errors=True):
             if formdef.is_disabled():
@@ -459,126 +464,157 @@ class SubmissionDirectory(Directory):
                 continue
             list_forms.append(formdef)
 
+            if prefetch:
+                # prefetch formdatas
+                data_class = formdef.data_class()
+                formdef._formdatas = data_class.select(
+                    [Equal('status', 'draft'), Equal('backoffice_submission', True)]
+                )
+                formdef._formdatas.sort(
+                    key=lambda x: x.receipt_time or make_aware(datetime.datetime(1900, 1, 1))
+                )
+                agent_ids.update([x.submission_agent_id for x in formdef._formdatas if x.submission_agent_id])
+
+        if prefetch:
+            # prefetch agents
+            self.prefetched_agents = {
+                str(x.id): x
+                for x in get_publisher().user_class.get_ids(list(agent_ids), ignore_errors=True)
+                if x is not None
+            }
+
         return list_forms
 
-    def _q_index(self):
-        get_response().breadcrumb.append(('submission/', _('Submission')))
-        get_response().set_title(_('Submission'))
-
-        list_forms = self.get_submittable_formdefs()
+    def get_categories(self, list_formdefs):
         cats = Category.select()
         Category.sort_by_position(cats)
         for cat in cats:
-            cat.formdefs = [x for x in list_forms if str(x.category_id) == str(cat.id)]
+            cat.formdefs = [x for x in list_formdefs if str(x.category_id) == str(cat.id)]
         misc_cat = Category(name=_('Misc'))
-        misc_cat.formdefs = [x for x in list_forms if not x.category]
+        misc_cat.formdefs = [x for x in list_formdefs if not x.category]
         cats.append(misc_cat)
+        return cats
+
+    def _q_index(self):
+        redirect_url = get_cfg('backoffice-submission', {}).get('redirect')
+        if redirect_url:
+            redirect_url = misc.get_variadic_url(
+                redirect_url, get_publisher().substitutions.get_context_variables(mode='lazy')
+            )
+            if redirect_url:
+                return redirect(redirect_url)
+
+        get_response().breadcrumb.append(('submission/', _('Submission')))
+        get_response().set_title(_('Submission'))
+
+        list_forms = self.get_submittable_formdefs(prefetch=False)
+
+        context = {'categories': self.get_categories(list_forms)}
+        return template.QommonTemplateResponse(
+            templates=['wcs/backoffice/submission.html'], context=context, is_django_native=True
+        )
+
+    def pending(self):
+        get_response().breadcrumb.append(('pending', _('Pending submissions')))
+        get_response().set_title(_('Pending submissions'))
+        get_response().add_javascript(['wcs.listing.js'])
+
+        limit = misc.get_int_or_400(
+            get_request().form.get('limit', get_publisher().get_site_option('default-page-size') or 20)
+        )
+        offset = misc.get_int_or_400(get_request().form.get('offset', 0))
+        order_by = misc.get_order_by_or_400(
+            get_request().form.get(
+                'order_by', get_publisher().get_site_option('default-sort-order') or '-receipt_time'
+            )
+        )
+        include_submission_channel = misc.get_cfg('submission-channels', {}).get('include-in-global-listing')
+
+        list_formdefs = self.get_submittable_formdefs(prefetch=False)
+        criterias = [
+            Equal('status', 'draft'),
+            Equal('backoffice_submission', True),
+            Contains('formdef_id', [x.id for x in list_formdefs]),
+        ]
 
         r = TemplateIO(html=True)
-        r += get_session().display_message()
-        modes = ['empty', 'create', 'existing']
-        for mode in modes:
-            list_content = TemplateIO()
-            for cat in cats:
-                if not cat.formdefs:
-                    continue
-                list_content += self.form_list(cat.formdefs, title=cat.name, mode=mode)
-            if not list_content.getvalue().strip():
-                continue
-            r += htmltext('<h2>%s</h2>') % {
-                'create': _('New submission'),
-                'existing': _('Running submission'),
-                'empty': _('Submission to complete'),
-            }.get(mode)
-            r += htmltext(f'<ul class="biglist {mode}">')
-            r += htmltext(list_content.getvalue())
-            r += htmltext('</ul>')
 
-        return r.getvalue()
+        r += htmltext('<table id="listing" class="main">')
+        r += htmltext('<thead><tr>')
+        if include_submission_channel:
+            r += htmltext('<th data-field-sort-key="submission_channel"><span>%s</span></th>') % _('Channel')
+        r += htmltext('<th data-field-sort-key="formdef_name"><span>%s</span></th>') % _('Form')
+        r += htmltext('<th data-field-sort-key="receipt_time"><span>%s</span></th>') % _('Created')
+        r += htmltext('<th><span>%s</span></th>') % _('Submission Agent')
+        r += htmltext('</tr></thead>')
+        r += htmltext('<tbody>\n')
 
-    def form_list(self, formdefs, title=None, mode='create'):
-        r = TemplateIO(html=True)
-        if mode != 'create':
-            skip = True
-            for formdef in formdefs:
-                if not hasattr(formdef, '_formdatas'):
-                    data_class = formdef.data_class()
-                    formdata_ids = data_class.get_ids_with_indexed_value('status', 'draft')
-                    formdef._formdatas = [
-                        x for x in data_class.get_ids(formdata_ids) if x.backoffice_submission is True
-                    ]
-                    formdef._formdatas.sort(
-                        key=lambda x: x.receipt_time or make_aware(datetime.datetime(1900, 1, 1))
-                    )
-                skip &= not (bool(formdef._formdatas))
-            if skip:
-                return
+        from wcs.sql import AnyFormData
 
-        first = True
+        total_count = AnyFormData.count(criterias)
+        formdatas = AnyFormData.select(criterias, order_by=order_by, limit=limit, offset=offset)
 
-        for formdef in formdefs:
-            if mode != 'create':
-                formdatas = formdef._formdatas[:]
-                if mode == 'empty':
-                    formdatas = [x for x in formdatas if x.has_empty_data()]
-                elif mode == 'existing':
-                    formdatas = [x for x in formdatas if not x.has_empty_data()]
-                if not formdatas:
-                    continue
+        # prefetch agents
+        agent_ids = set()
+        agent_ids.update([x.submission_agent_id for x in formdatas if x.submission_agent_id])
+        self.prefetched_agents = {
+            str(x.id): x
+            for x in get_publisher().user_class.get_ids(list(agent_ids), ignore_errors=True)
+            if x is not None
+        }
 
-            if first and title:
-                r += htmltext('<li><h3>%s</h3></li>') % title
-                first = False
-
-            r += htmltext('<li>')
-            if mode == 'create':
-                r += htmltext('<strong class="label"><a href="%s/">%s</a></strong>') % (
-                    formdef.url_name,
-                    formdef.name,
-                )
+        for formdata in formdatas:
+            url = f'{formdata.formdef.url_name}/{formdata.id}/'
+            r += htmltext(f'<tr data-link="{url}">')
+            if include_submission_channel:
+                r += htmltext('<td>%s</td>') % formdata.get_submission_channel_label()
+            r += htmltext(f'<td><a href="{url}">{formdata.get_display_name()}')
+            if formdata.default_digest:
+                r += htmltext(' <small>%s</small>') % formdata.default_digest
+            r += htmltext('</a></td>')
+            r += htmltext('<td class="cell-time">%s</td>') % misc.localstrftime(formdata.receipt_time)
+            agent_user = self.prefetched_agents.get(formdata.submission_agent_id)
+            if agent_user:
+                r += htmltext('<td class="cell-user">%s</td>') % agent_user.get_display_name()
             else:
-                r += htmltext('<strong class="label"><a class="fake">%s</a></strong>') % formdef.name
-            r += htmltext('</li>')
-            if mode == 'create':
-                continue
-            for formdata in formdatas:
-                r += htmltext('<li class="smallitem">')
-                label = ''
-                if formdata.submission_channel:
-                    label = '%s ' % formdata.get_submission_channel_label()
-                label += _('#%(id)s, %(time)s') % {
-                    'id': formdata.id,
-                    'time': misc.localstrftime(formdata.receipt_time)
-                    if formdata.receipt_time
-                    else _('unknown date'),
-                }
-                if formdata.submission_agent_id:
-                    agent_user = get_publisher().user_class.get(
-                        formdata.submission_agent_id, ignore_errors=True
-                    )
-                    if agent_user:
-                        label += ' (%s)' % agent_user.display_name
-                r += htmltext('<a href="%s/%s/">%s</a>') % (formdef.url_name, formdata.id, label)
-                r += htmltext('</li>')
+                r += htmltext('<td class="cell-user cell-no-user">-</td>')
+            r += htmltext('</tr>\n')
 
-        return r.getvalue()
+        r += htmltext('</tbody></table>')
+
+        if (offset > 0) or (total_count > limit > 0):
+            r += pagination_links(offset, limit, total_count)
+
+        if get_request().form.get('ajax') == 'true':
+            get_request().ignore_session = True
+            get_response().filter = {'raw': True}
+            return r.getvalue()
+
+        rt = TemplateIO(html=True)
+        rt += htmltext('<div id="appbar">')
+        rt += htmltext('<h2>%s</h2>') % _('Pending submissions')
+        rt += htmltext('</div>')
+        rt += r.getvalue()
+        form = Form(use_tokens=False, id='listing-settings', method='get', action='pending')
+        form.add_hidden('offset', offset)
+        form.add_hidden('limit', limit)
+        form.add_hidden('order_by', order_by)
+        rt += form.render()
+
+        return rt.getvalue()
 
     def count(self):
         formdefs = self.get_submittable_formdefs()
         count = 0
         mode = get_request().form.get('mode')
         for formdef in formdefs:
-            if not hasattr(formdef, '_formdatas'):
-                data_class = formdef.data_class()
-                formdata_ids = data_class.get_ids_with_indexed_value('status', 'draft')
-                formdatas = [x for x in data_class.get_ids(formdata_ids) if x.backoffice_submission is True]
-                if mode == 'empty':
-                    formdatas = [x for x in formdatas if x.has_empty_data()]
-                elif mode == 'existing':
-                    formdatas = [x for x in formdatas if not x.has_empty_data()]
-                count += len(formdatas)
+            formdatas = formdef._formdatas
+            if mode == 'empty':
+                formdatas = [x for x in formdatas if x.has_empty_data()]
+            elif mode == 'existing':
+                formdatas = [x for x in formdatas if not x.has_empty_data()]
+            count += len(formdatas)
         return misc.json_response({'count': count})
 
     def _q_lookup(self, component):
-        get_response().breadcrumb.append(('submission/', _('Submission')))
         return FormFillPage(component)

wcs/compat.py

@@ -92,7 +92,7 @@ class TemplateWithFallbackView(TemplateView):
             response.reason_phrase = self.quixote_response.reason_phrase
         elif request.headers.get('X-Popup') == 'true':
             response = HttpResponse('<div><div class="popup-content">%s</div></div>' % context['body'])
-        elif 'raw' in (getattr(self.quixote_response, 'filter') or {}):
+        elif self.quixote_response.raw:
             # used for raw HTML snippets (for example in the test tool
             # results in inspect page).
             response = HttpResponse(context['body'])
@@ -161,7 +161,7 @@ class CompatWcsPublisher(WcsPublisher):
         if response.status_code == 304:
             # clients don't like to receive content with a 304
             return ''
-        if response.content_type != 'text/html':
+        if response.content_type != 'text/html' or response.raw:
             return output
         if not hasattr(response, 'filter') or not response.filter:
             return output

wcs/ctl/management/commands/wipe_data.py

@@ -31,12 +31,17 @@ class Command(TenantCommand):
         parser.add_argument(
             '--exclude-forms', metavar='FORMS', help='list of forms to exclude (slugs, separated by commas)'
         )
+        parser.add_argument('--no-simulate', action='store_true', help='perform the wipe for real')
 
     def handle(self, *args, **options):
+        if not options.get('no_simulate'):
+            self.stdout.write('SIMULATION MODE: no actual wiping will happen.\n')
+            self.stdout.write('(use --no-simulate after checking results)\n\n')
+
         for domain in self.get_domains(**options):
             self.init_tenant_publisher(domain, register_tld_names=False)
             if options.get('all'):
-                formdefs = FormDef.select()
+                formdefs = FormDef.select(order_by='url_name')
             elif options.get('forms'):
                 formdefs = [FormDef.get_by_urlname(x) for x in options['forms'].split(',')]
             else:
@@ -44,4 +49,9 @@ class Command(TenantCommand):
             if options.get('exclude_forms'):
                 formdefs = [x for x in formdefs if x.url_name not in options['exclude_forms'].split(',')]
             for formdef in formdefs:
-                formdef.data_class().wipe()
+                if options.get('no_simulate'):
+                    formdef.data_class().wipe()
+                else:
+                    count = formdef.data_class().count()
+                    if count:
+                        self.stdout.write(f'{formdef.url_name}: {count}\n')

wcs/ctl/runscript.py

@@ -1,53 +0,0 @@
-# w.c.s. - web application for online forms
-# Copyright (C) 2005-2013  Entr'ouvert
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, see <http://www.gnu.org/licenses/>.
-
-import os.path
-import runpy
-import sys
-import warnings
-
-from ..qommon.ctl import Command, make_option
-
-
-class CmdRunScript(Command):
-    '''Run a script within a given host publisher context'''
-
-    name = 'runscript'
-
-    def __init__(self):
-        Command.__init__(
-            self,
-            [
-                make_option('--vhost', metavar='VHOST', action='store', dest='vhost'),
-            ],
-        )
-
-    def execute(self, base_options, sub_options, args):
-        warnings.warn('Deprecated command, use management command', DeprecationWarning)
-        from .. import publisher
-
-        self.config.remove_option('main', 'error_log')
-        publisher.WcsPublisher.configure(self.config)
-        publisher = publisher.WcsPublisher.create_publisher(register_tld_names=False)
-        publisher.set_tenant_by_hostname(sub_options.vhost)
-        fullpath = os.path.dirname(os.path.abspath(args[0]))
-        sys.path.insert(0, fullpath)
-        module_name = os.path.splitext(os.path.basename(args[0]))[0]
-        sys.argv = args
-        runpy.run_module(module_name)
-
-
-CmdRunScript.register()

wcs/ctl/shell.py

@@ -1,36 +0,0 @@
-# w.c.s. - web application for online forms
-# Copyright (C) 2005-2010  Entr'ouvert
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, see <http://www.gnu.org/licenses/>.
-
-# Most of this code is copied from the shell command [1] of the project Django
-# release 1.4. Many thanks to them.
-#
-# [1]: file django/core/management/commands/shell.py
-
-import sys
-
-from ..qommon.ctl import Command
-
-
-class CmdShell(Command):
-    '''(obsolete) Launch a shell and initialize a publisher on a given host'''
-
-    name = 'shell'
-
-    def execute(self, base_options, sub_options, args):
-        print('Error: use wcs-manage shell command', file=sys.stderr)
-
-
-CmdShell.register()

wcs/fields/file.py

@@ -126,7 +126,12 @@ class FileField(WidgetField):
             value = value.get_value()  # unbox
         if hasattr(value, 'base_filename'):
             upload = PicklableUpload(value.base_filename, value.content_type or 'application/octet-stream')
-            upload.receive([value.get_content()])
+            if hasattr(value, 'get_content'):
+                upload.receive([value.get_content()])
+            else:
+                # native quixote Upload object
+                upload.receive([value.fp.read()])
+                value.fp.seek(0)
             return upload
         from wcs.workflows import NamedAttachmentsSubstitutionProxy
 

wcs/fields/item.py

@@ -45,6 +45,15 @@ from .base import SetValueError, WidgetField, register_field_class
 from .map import MapOptionsMixin
 
 
+class UnknownCardValueError(ValueError):
+    def __init__(self, value):
+        super().__init__('unknown card value (%r)' % value)
+        self.value = value
+
+    def get_error_summary(self):
+        return _('unknown card value (%r)') % self.value
+
+
 class ItemWithImageFieldMixin:
     # images options
     image_desktop_size = 150
@@ -536,7 +545,7 @@ class ItemField(WidgetField, MapOptionsMixin, ItemFieldMixin, ItemWithImageField
                 if card_value:
                     value = str(card_value['id'])
                 else:
-                    raise ValueError('unknown card value (%r)' % value)
+                    raise UnknownCardValueError(value)
         return value
 
     def convert_value_from_str(self, value):

wcs/formdata.py

@@ -24,6 +24,7 @@ import json
 import re
 import urllib.parse
 
+import unidecode
 from django.utils.html import strip_tags
 from django.utils.timezone import localtime, make_naive
 from quixote import get_publisher, get_request, get_session
@@ -31,7 +32,7 @@ from quixote.errors import RequestError
 from quixote.html import htmltext
 from quixote.http_request import Upload
 
-from wcs.sql_criterias import And, Contains, Intersects
+from wcs.sql_criterias import And, Contains, Equal, Intersects
 
 from .qommon import _, misc
 from .qommon.evalutils import make_datetime
@@ -360,6 +361,15 @@ class FormData(StorableObject):
         value = self.get_natural_key()
         return str(value) if value is not None else None
 
+    @classmethod
+    def get_by_id(cls, value):
+        if cls._formdef.id_template:
+            try:
+                return cls.select([Equal('id_display', str(value))], limit=1)[0]
+            except IndexError:
+                raise KeyError(value)
+        return cls.get(value)
+
     def get_user(self):
         if self.user_id and self.user_id != 'ultra-user':
             return get_publisher().user_class.get(self.user_id, ignore_errors=True)
@@ -492,6 +502,13 @@ class FormData(StorableObject):
         self.evolution = [evo]
         evo.add_part(ContentSnapshotPart(formdata=self, old_data={}))
 
+    @classmethod
+    def force_valid_id_characters(cls, value):
+        value = unidecode.unidecode(value)
+        value = re.sub(r'[^\w\s\'\-_]', '', unidecode.unidecode(value)).strip()
+        value = re.sub(r'\s+', '-', value)
+        return value
+
     def set_auto_fields(self, *args, **kwargs):
         fields = {}
         for key, value in (self.formdef.digest_templates or {}).items():
@@ -574,6 +591,10 @@ class FormData(StorableObject):
                             exception=e,
                         )
                         new_value = 'ERROR'
+
+                if attribute == 'id_display' and self.formdef.id_template:
+                    new_value = self.force_valid_id_characters(new_value)
+
                 if attribute.startswith('template:'):
                     key = attribute[9:]
                     if new_value != (self.digests or {}).get(key):
@@ -874,14 +895,14 @@ class FormData(StorableObject):
     def get_url(self, backoffice=False, include_category=False, language=None):
         return '%s%s/' % (
             self.formdef.get_url(backoffice=backoffice, include_category=include_category, language=language),
-            self.id,
+            self.identifier,
         )
 
     def get_backoffice_url(self):
         return self.get_url(backoffice=True)
 
     def get_api_url(self):
-        return '%s%s/' % (self.formdef.get_api_url(), self.id)
+        return '%s%s/' % (self.formdef.get_api_url(), self.identifier)
 
     def get_file_base_url(self):
         return '%sdownload' % self.get_url()
@@ -1535,7 +1556,7 @@ class FormData(StorableObject):
         if hasattr(self, 'uuid'):
             data['uuid'] = self.uuid
         data['id'] = self.identifier
-        data['internal-id'] = str(self.id)
+        data['internal_id'] = str(self.id)
         data['display_id'] = self.get_display_id()
         data['display_name'] = self.get_display_name()
         data['digests'] = self.digests

wcs/formdef.py

@@ -1882,23 +1882,27 @@ class FormDef(StorableObject):
                     form_roles.append(x)
         return self.is_user_allowed_read(user, formdata=formdata)
 
+    @property
+    def publication_datetime(self):
+        try:
+            return get_as_datetime(self.publication_date)
+        except (TypeError, ValueError):
+            return None
+
+    @property
+    def expiration_datetime(self):
+        try:
+            return get_as_datetime(self.expiration_date)
+        except (TypeError, ValueError):
+            return None
+
     def is_disabled(self):
         if self.disabled:
             return True
-        if self.publication_date:
-            try:
-                publication_datetime = get_as_datetime(self.publication_date)
-            except ValueError:
-                return False
-            if publication_datetime > datetime.datetime.now():
-                return True
-        if self.expiration_date:
-            try:
-                expiration_datetime = get_as_datetime(self.expiration_date)
-            except ValueError:
-                return False
-            if expiration_datetime < datetime.datetime.now():
-                return True
+        if self.publication_datetime and self.publication_datetime > datetime.datetime.now():
+            return True
+        if self.expiration_datetime and self.expiration_datetime < datetime.datetime.now():
+            return True
         return False
 
     class _EmptyClass:  # helper for instance creation without calling __init__

wcs/forms/backoffice.py

@@ -344,7 +344,7 @@ class FormDefUI:
                     classes.append('criticality-level')
                     style = ' style="border-left-color: %s;"' % level.colour
 
-            link = str(filled.id) + '/'
+            link = str(filled.identifier) + '/'
             data = ' data-link="%s"' % link
             if filled.anonymised:
                 data += ' data-anonymised="true"'

wcs/forms/common.py

@@ -112,7 +112,7 @@ class FileDirectory(Directory):
 
         # force potential HTML upload to be used as-is (not decorated with theme)
         # and with minimal permissions
-        response.filter = {}
+        response.raw = True
         response.set_header(
             'Content-Security-Policy',
             'default-src \'none\'; img-src %s;' % get_request().build_absolute_uri(),
@@ -157,7 +157,7 @@ class FormTemplateMixin:
 
 
 class FormStatusPage(Directory, FormTemplateMixin):
-    _q_exports_orig = ['', 'download', 'json', 'action', 'live', 'tempfile']
+    _q_exports_orig = ['', 'download', 'json', 'action', 'live', 'tempfile', 'tsupdate']
     _q_extra_exports = []
     form_page_class = None
 
@@ -244,6 +244,12 @@ class FormStatusPage(Directory, FormTemplateMixin):
             values_at=values_at,
         )
 
+    def tsupdate(self):
+        # return new timestamp value used for replay protection
+        get_request().ignore_session = True
+        get_response().set_content_type('application/json')
+        return json.dumps({'ts': str(self.filled.last_update_time.timestamp())})
+
     def tempfile(self):
         # allow for file uploaded via a file widget in a workflow form
         # to be downloaded back from widget
@@ -734,6 +740,9 @@ class FormStatusPage(Directory, FormTemplateMixin):
         try:
             next_url = self.filled.handle_workflow_form(user, form)
         except ReplayException:
+            widget = form.get_widget('_ts')
+            # update with new timestamp as the page is refreshed
+            widget.set_value(str(self.filled.last_update_time.timestamp()))
             raise RedisplayFormException(
                 form=form,
                 error={
@@ -1065,7 +1074,7 @@ class TempfileDirectoryMixin:
 
         # force potential HTML upload to be used as-is (not decorated with theme)
         # and with minimal permissions
-        response.filter = {}
+        response.raw = True
         response.set_header(
             'Content-Security-Policy',
             'default-src \'none\'; img-src %s;' % get_request().build_absolute_uri(),

wcs/forms/root.py

@@ -31,6 +31,7 @@ from django.utils.timezone import localtime
 from quixote import get_publisher, get_request, get_response, get_session, get_session_manager, redirect
 from quixote.directory import AccessControlled, Directory
 from quixote.errors import MethodNotAllowedError, RequestError
+from quixote.form import FormTokenWidget
 from quixote.html import TemplateIO, htmltext
 from quixote.util import randbytes
 
@@ -621,6 +622,9 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
         try:
             with get_publisher().substitutions.temporary_feed(transient_formdata, force_mode='lazy'):
                 form = self.create_form(page, displayed_fields, transient_formdata=transient_formdata)
+                if page_change is False and page_error_messages:
+                    # ignore form token when there are other errors
+                    form._names.pop('_form_id', None)
         except MissingBlockFieldError as e:
             logged_error = get_publisher().record_error(
                 str(e), exception=e, notify=True, formdef=self.formdef
@@ -1002,6 +1006,10 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
 
     def create_form(self, *args, **kwargs):
         form = self.formdef.create_form(*args, **kwargs)
+        if len(self.pages) == 1 and not self.formdef.confirmation:
+            # if there's a form with a single page, no confirmation, add native quixote
+            # CSRF protection.
+            form.add(FormTokenWidget, form.TOKEN_NAME)
         form.attrs['data-live-url'] = self.formdef.get_url(language=get_publisher().current_language) + 'live'
         form.attrs['data-live-validation-url'] = (
             self.formdef.get_url(language=get_publisher().current_language) + 'live-validation'
@@ -1951,6 +1959,7 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
                         user_id = user.id
                 wf_status = item.get_target_status(self.edited_data)
                 if wf_status:
+                    item.handle_markers_stack(self.edited_data)
                     if self.edited_data.jump_status(wf_status[0].id, user_id=user_id):
                         self.edited_data.record_workflow_event('edit-action', action_item_id=item.id)
                         url = self.edited_data.perform_workflow()
@@ -2409,7 +2418,7 @@ class RootDirectory(AccessControlled, Directory):
 
 
 class PublicFormStatusPage(FormStatusPage):
-    _q_exports_orig = ['', 'download', 'status', 'live', 'tempfile']
+    _q_exports_orig = ['', 'download', 'status', 'live', 'tempfile', 'tsupdate']
     form_page_class = FormPage
     history_templates = ['wcs/front/formdata_history.html', 'wcs/formdata_history.html']
     status_templates = ['wcs/front/formdata_status.html', 'wcs/formdata_status.html']

wcs/locale/fr/LC_MESSAGES/django.po

@@ -4,8 +4,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: wcs 0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-02-13 12:28+0100\n"
-"PO-Revision-Date: 2024-02-13 12:36+0100\n"
+"POT-Creation-Date: 2024-03-04 13:46+0100\n"
+"PO-Revision-Date: 2024-03-04 13:47+0100\n"
 "Last-Translator: Thomas Noël <tnoel@entrouvert.com>\n"
 "Language-Team: french\n"
 "Language: fr\n"
@@ -313,7 +313,7 @@ msgstr "Nouveau bloc de champs"
 #: admin/blocks.py admin/categories.py admin/comment_templates.py
 #: admin/data_sources.py admin/forms.py admin/mail_templates.py
 #: admin/settings.py admin/tests.py admin/workflows.py admin/wscalls.py
-#: backoffice/data_management.py backoffice/i18n.py
+#: backoffice/data_management.py backoffice/i18n.py wf/export_to_model.py
 msgid "File"
 msgstr "Fichier"
 
@@ -725,11 +725,11 @@ msgstr "Attribut du texte (text)"
 msgid "Name of the attribute containing the label of an entry (default: text)"
 msgstr "Nom de l’attribut contenant le texte d’une donnée (par défaut : text)"
 
-#: admin/data_sources.py admin/wscalls.py wf/wscall.py
+#: admin/data_sources.py admin/wscalls.py
 msgid "Notify on errors"
 msgstr "Notifier en cas d’erreur"
 
-#: admin/data_sources.py admin/wscalls.py wf/wscall.py
+#: admin/data_sources.py admin/wscalls.py
 msgid "Record on errors"
 msgstr "Enregistrer les erreurs"
 
@@ -934,9 +934,10 @@ msgstr "Ce formulaire contient %d champs."
 msgid "This form contains more than %d fields."
 msgstr "Ce formulaire contient plus de %d champs."
 
-#: admin/fields.py fields/page.py
-msgid "Page"
-msgstr "Page"
+#: admin/fields.py
+#, python-format
+msgid "Page \"%s\""
+msgstr "Page « %s »"
 
 #: admin/fields.py templates/wcs/backoffice/block-inspect.html
 #: templates/wcs/backoffice/carddef.html
@@ -1737,6 +1738,7 @@ msgid "Django Expression"
 msgstr "Expression Django"
 
 #: admin/logged_errors.py backoffice/management.py qommon/form.py
+#: wf/export_to_model.py
 msgid "Template"
 msgstr "Gabarit"
 
@@ -1758,8 +1760,8 @@ msgid "Stack trace (most recent call first)"
 msgstr "Trace (appels les plus récents en premier)"
 
 #: admin/logged_errors.py api_export_import.py backoffice/management.py
-#: formdata.py formdef.py statistics/views.py wf/create_formdata.py wf/form.py
-#: wf/resubmit.py
+#: backoffice/submission.py formdata.py formdef.py statistics/views.py
+#: wf/create_formdata.py wf/form.py wf/resubmit.py
 msgid "Form"
 msgstr "Formulaire"
 
@@ -2146,12 +2148,12 @@ msgid "Configure geolocation and geocoding"
 msgstr "Configurer la géolocalisation et le géocodage"
 
 #: admin/settings.py
-msgid "Submission channels"
-msgstr "Canaux de saisie"
+msgid "Backoffice Submission"
+msgstr "Saisie backoffice"
 
 #: admin/settings.py
-msgid "Configure submission channels related options"
-msgstr "Configurer les options en rapport avec les canaux de saisie"
+msgid "Configure backoffice submission related options"
+msgstr "Configurer les options en rapport avec la saisie par les agents"
 
 #: admin/settings.py
 msgid "Configure users"
@@ -2454,10 +2456,36 @@ msgstr ""
 "Si complété, envoie tous les courriels à cette adresse au lieu des vrais "
 "destinataires"
 
+#: admin/settings.py
+msgid "Sidebar menu entry"
+msgstr "Icône dans le menu latéral gauche"
+
+#: admin/settings.py
+msgctxt "sidebar_menu_entry"
+msgid "Visible"
+msgstr "Visible"
+
+#: admin/settings.py
+msgctxt "sidebar_menu_entry"
+msgid "Hidden"
+msgstr "Cachée"
+
+#: admin/settings.py
+msgid "URL for backoffice submission"
+msgstr "URL pour la saisie backoffice"
+
+#: admin/settings.py
+msgid "Leave empty to use native screen."
+msgstr "Laisser vide pour utiliser l’écran par défaut."
+
 #: admin/settings.py
 msgid "Include submission channel column in global listing"
 msgstr "Inclure une colonne avec le canal de saisie dans la vue globale"
 
+#: admin/settings.py
+msgid "Backoffice submission settings"
+msgstr "Paramètres de saisie backoffice"
+
 #: admin/settings.py
 msgid "Back to settings"
 msgstr "Retourner au paramètres"
@@ -2591,6 +2619,15 @@ msgstr "Lancement manuel."
 msgid "Workflow error: %s"
 msgstr "Erreur du workflow : %s"
 
+#: admin/tests.py
+#, python-format
+msgid "Invalid JSON: %s"
+msgstr "JSON invalide : %s"
+
+#: admin/tests.py
+msgid "Response payload (JSON)"
+msgstr "Contenu de la réponse (JSON)"
+
 #: admin/tests.py
 msgid "Restrict to query string data"
 msgstr "Limiter aux paramètres de l’URL"
@@ -2627,15 +2664,6 @@ msgstr "Limiter à la méthode"
 msgid "Restrict to POST data"
 msgstr "Limiter au données contenues dans le corps de la requête"
 
-#: admin/tests.py
-#, python-format
-msgid "Invalid JSON: %s"
-msgstr "JSON invalide : %s"
-
-#: admin/tests.py
-msgid "Response payload (JSON)"
-msgstr "Contenu de la réponse (JSON)"
-
 #: admin/tests.py
 msgid "Edit webservice response"
 msgstr "Modifier la réponse webservice"
@@ -2762,8 +2790,8 @@ msgstr "Filtrer"
 msgid "Last Modification:"
 msgstr "Dernière modification "
 
-#: admin/utils.py wf/attachment.py wf/comment.py wf/editable.py wf/form.py
-#: wf/resubmit.py workflows.py
+#: admin/utils.py wf/attachment.py wf/comment.py wf/form.py wf/resubmit.py
+#: workflows.py
 #, python-format
 msgid "by %s"
 msgstr "par %s"
@@ -3972,6 +4000,14 @@ msgstr "Types de champ dépréciés"
 msgid "Obsolete action types"
 msgstr "Types d’actions dépréciés"
 
+#: backoffice/deprecations.py
+msgid "CSV connector"
+msgstr "Connecteur « Fichier tableur »"
+
+#: backoffice/deprecations.py
+msgid "JSON Data Store connector"
+msgstr "Connecteur « Stockage de données JSON »"
+
 #: backoffice/deprecations.py
 msgid "Use Django templates."
 msgstr "Utiliser des gabarits Django."
@@ -4250,7 +4286,7 @@ msgstr[1] "%(total)s éléments"
 msgid "Reference"
 msgstr "Référence"
 
-#: backoffice/management.py
+#: backoffice/management.py backoffice/submission.py
 msgid "Created"
 msgstr "Date de création"
 
@@ -4323,7 +4359,7 @@ msgstr "Fin"
 msgid "Current User Function"
 msgstr "Fonction de l’utilisateur connecté"
 
-#: backoffice/management.py
+#: backoffice/management.py backoffice/submission.py
 msgid "Submission Agent"
 msgstr "Agent à la saisie"
 
@@ -4835,6 +4871,7 @@ msgid "Per page: "
 msgstr "Par page : "
 
 #: backoffice/root.py backoffice/submission.py
+#: templates/wcs/backoffice/submission.html
 #: templates/wcs/backoffice/test_edit_sidebar.html
 msgid "Submission"
 msgstr "Saisie"
@@ -4991,26 +5028,9 @@ msgstr ""
 msgid "Discard this form"
 msgstr "Abandonner la saisie"
 
-#: backoffice/submission.py
-msgid "New submission"
-msgstr "Nouvelle demande"
-
-#: backoffice/submission.py
-msgid "Running submission"
-msgstr "Saisie entamée"
-
-#: backoffice/submission.py
-msgid "Submission to complete"
-msgstr "Prédemande"
-
-#: backoffice/submission.py
-#, python-format
-msgid "#%(id)s, %(time)s"
-msgstr "n°%(id)s, %(time)s"
-
-#: backoffice/submission.py
-msgid "unknown date"
-msgstr "date inconnue"
+#: backoffice/submission.py templates/wcs/backoffice/submission.html
+msgid "Pending submissions"
+msgstr "Saisies entamées"
 
 #: blocks.py
 msgid "Field block"
@@ -5540,6 +5560,11 @@ msgstr ""
 msgid "File storage system"
 msgstr "Système de stockage de fichier"
 
+#: fields/item.py
+#, python-format
+msgid "unknown card value (%r)"
+msgstr "valeur de fiche inconnue (%r)"
+
 #: fields/item.py
 msgid "Image size on desktop"
 msgstr "Taille des images sur ordinateur"
@@ -5798,6 +5823,10 @@ msgstr "Message d’erreur si condition non satisfaite"
 msgid "Both condition and error message are required."
 msgstr "La condition et le message d’erreur sont requis."
 
+#: fields/page.py
+msgid "Page"
+msgstr "Page"
+
 #: fields/page.py
 msgid "Post Conditions"
 msgstr "Conditions de sortie"
@@ -6831,14 +6860,6 @@ msgstr "erreur"
 msgid "completed"
 msgstr "complétée"
 
-#: qommon/ctl.py
-msgid "use a non default configuration file"
-msgstr "utilise un fichier de configuration autre que celui par défaut"
-
-#: qommon/ctl.py
-msgid "Display this help and exit"
-msgstr "Afficher cette aide et quitter"
-
 #: qommon/emails.py
 msgid "Failed to connect to SMTP server (timeout)"
 msgstr ""
@@ -8626,7 +8647,7 @@ msgstr "liste"
 msgid "string"
 msgstr "texte"
 
-#: qommon/misc.py
+#: qommon/misc.py qommon/templatetags/qommon.py
 msgid "file"
 msgstr "fichier"
 
@@ -8962,6 +8983,16 @@ msgstr "Réessayer"
 msgid "Failed to apply unaccent filter on value (%s)"
 msgstr "Erreur à l’application du filter unaccent sur la valeur (%s)"
 
+#: qommon/templatetags/qommon.py
+#, python-format
+msgid "{%% action_button %%} requires a label parameter"
+msgstr "{%% action_button %%} nécessite un paramètre « label »"
+
+#: qommon/templatetags/qommon.py
+#, python-format
+msgid "{%% temporary_action_button %%} requires a label parameter"
+msgstr "{%% temporary_action_button %%} nécessite un paramètre « label »"
+
 #: qommon/templatetags/qommon.py
 #, python-format
 msgid "|%s used on invalid queryset (%r)"
@@ -8977,11 +9008,38 @@ msgstr "|objects appelé sur une source invalide (%r)"
 msgid "|objects with invalid reference (%r)"
 msgstr "|objects utilisé avec une référence invalide (%r)"
 
+#: qommon/templatetags/qommon.py
+#, python-format
+msgid "|convert_image_format: unknown format (must be one of %s)"
+msgstr "|convert_image_format: format inconnu (doit être un de %s)"
+
+#: qommon/templatetags/qommon.py
+msgid "|convert_image_format: missing input"
+msgstr "|convert_image_format: données manquantes"
+
+#: qommon/templatetags/qommon.py
+msgid "|convert_image_format: not supported"
+msgstr "|convert_image_format: pas pris en charge"
+
+#: qommon/templatetags/qommon.py
+#, python-format
+msgid "|convert_image_format: conversion error (%s)"
+msgstr "|convert_image_format: erreur de conversion (%s)"
+
 #: qommon/templatetags/qommon.py
 #, python-format
 msgid "|check_no_duplicates not used on a list (%s)"
 msgstr "|check_no_duplicates pas utilisé sur une liste (%s)"
 
+#: qommon/templatetags/qommon.py
+msgid "|details_format called without specifying a format"
+msgstr "|details_format appelé sans préciser de format"
+
+#: qommon/templatetags/qommon.py
+#, python-format
+msgid "|details_format called with unknown format (%s)"
+msgstr "|details_format appelé avec un format inconnu (%s)"
+
 #: roles.py
 msgid "Logged Users"
 msgstr "Utilisateurs identifiés"
@@ -9064,6 +9122,15 @@ msgstr "Six derniers mois"
 msgid "Last twelve months"
 msgstr "Douze derniers mois"
 
+#: statistics/views.py
+msgid "Simplified status"
+msgstr "Statut simplifié"
+
+#: statistics/views.py
+#, python-format
+msgid "Ignore forms where \"%s\" is empty."
+msgstr "Ignorer les demandes/fiches où « %s » est vide."
+
 #: statistics/views.py
 msgctxt "statistics"
 msgid "Open"
@@ -9074,15 +9141,6 @@ msgctxt "statistics"
 msgid "Done"
 msgstr "Terminés"
 
-#: statistics/views.py
-msgid "Simplified status"
-msgstr "Statut simplifié"
-
-#: statistics/views.py
-#, python-format
-msgid "Ignore forms where \"%s\" is empty."
-msgstr "Ignorer les demandes/fiches où « %s » est vide."
-
 #: statistics/views.py
 msgid "In progress"
 msgstr "En cours"
@@ -9874,6 +9932,7 @@ msgid "XML"
 msgstr "XML"
 
 #: templates/wcs/backoffice/snapshots_compare.html
+#: templates/wcs/backoffice/test-result.html
 #: templates/wcs/backoffice/test_sidebar.html
 msgid "Inspect"
 msgstr "Inspecteur"
@@ -9930,6 +9989,10 @@ msgstr "Voir toutes les erreurs"
 msgid "No errors, congratulations!"
 msgstr "Aucune erreur, bravo !"
 
+#: templates/wcs/backoffice/test-result-detail.html
+msgid "Field linked to error:"
+msgstr "Champ lié à l’erreur :"
+
 #: templates/wcs/backoffice/test-result-detail.html
 msgid "Test action:"
 msgstr "Action de test :"
@@ -9978,6 +10041,10 @@ msgstr "Date :"
 msgid "Success!"
 msgstr "Succès !"
 
+#: templates/wcs/backoffice/test-result.html
+msgid "Display inspect"
+msgstr "Voir l’inspecteur"
+
 #: templates/wcs/backoffice/test-result.html
 msgid "Display details"
 msgstr "Afficher les détails"
@@ -9997,7 +10064,7 @@ msgstr "Pas encore de résultats des tests."
 
 #: templates/wcs/backoffice/test-webservice-responses.html
 #: wf/assign_carddata.py wf/create_formdata.py wf/redirect_to_url.py
-#: wf/roles.py
+#: wf/roles.py workflow_tests.py
 msgid "not configured"
 msgstr "non configurée"
 
@@ -10465,6 +10532,10 @@ msgid ""
 msgstr ""
 "Erreur inattendue lors de l’appel webservice vers l’URL %(url)s : %(error)s."
 
+#: testdef.py
+msgid "method must be GET"
+msgstr "la méthode doit être GET"
+
 #: users.py
 #, python-format
 msgid "Session User Field: %s"
@@ -10789,6 +10860,11 @@ msgstr "Données de traitement « %s »"
 msgid "Fields Update"
 msgstr "Modifier les données de traitement"
 
+#: wf/backoffice_fields.py
+#, python-format
+msgid "Failed to assign field (%(id)s): %(summary)s"
+msgstr "Erreur d’assignation au champ (%(id)s) : %(summary)s"
+
 #: wf/backoffice_fields.py
 #, python-format
 msgid "Failed to convert %(class)s value to %(kind)s field (%(id)s)"
@@ -11163,6 +11239,11 @@ msgstr "Préciser la liste des fiches sur laquelle l’action va s’exécuter"
 msgid "Edition"
 msgstr "Édition"
 
+#: wf/editable.py
+#, python-format
+msgid "\"%(button_label)s\", by %(by)s"
+msgstr "« %(button_label)s », par %(by)s"
+
 #: wf/editable.py
 msgid "Edit Form"
 msgstr "Modifier le formulaire"
@@ -11187,6 +11268,10 @@ msgstr "À partir d’une page"
 msgid "Page Identifier"
 msgstr "Identifiant de page"
 
+#: wf/editable.py wf/wscall.py workflows.py
+msgid "Set marker to jump back to current status"
+msgstr "Poser un marqueur qui permettra de revenir au statut actuel"
+
 #: wf/export_to_model.py
 msgid "Templating Error"
 msgstr "Erreur de traitement d’un modèle"
@@ -11204,6 +11289,10 @@ msgstr "Création de document"
 msgid "with model named %(file_name)s of %(size)s"
 msgstr "avec le modèle %(file_name)s de %(size)s"
 
+#: wf/export_to_model.py
+msgid "with model from template"
+msgstr "avec un modèle issu d’un gabarit"
+
 #: wf/export_to_model.py
 msgid "no model set"
 msgstr "aucun modèle défini"
@@ -11232,10 +11321,6 @@ msgstr "Interactive (bouton)"
 msgid "Non interactive"
 msgstr "Non interactive"
 
-#: wf/export_to_model.py
-msgid "Available variables"
-msgstr "Variables disponibles"
-
 #: wf/export_to_model.py
 msgid ""
 "You can use variables in your model using the {{variable}} syntax, available "
@@ -11244,13 +11329,17 @@ msgstr ""
 "Vous pouvez utiliser des variables dans votre modèle avec la syntaxe "
 "{{variable}}. Les variables disponibles dépendent du formulaire."
 
+#: wf/export_to_model.py
+msgid "Model"
+msgstr "Modèle"
+
 #: wf/export_to_model.py
 msgid "Current value"
 msgstr "Valeur actuelle"
 
 #: wf/export_to_model.py
-msgid "Model"
-msgstr "Modèle"
+msgid "Template to obtain model file"
+msgstr "Gabarit pour obtenir le ficher servant de modèle"
 
 #: wf/export_to_model.py
 msgid "Convert generated file to PDF"
@@ -11289,6 +11378,15 @@ msgstr "Erreur dans un gabarit lors de la création de document"
 msgid "Error in template: %s"
 msgstr "erreur dans le document modèle : %s"
 
+#: wf/export_to_model.py
+msgid "Failed to evaluate template for action"
+msgstr "Erreur à l’évaluation du gabarit pour l’action"
+
+#: wf/export_to_model.py
+#, python-format
+msgid "Invalid value obtained for model file (%r)"
+msgstr "Valeur invalide obtenue pour le fichier modèle (%r)"
+
 #: wf/external_workflow.py
 #, python-format
 msgid "Running external actions on \"%(label)s\" (%(count)s processed)"
@@ -11772,9 +11870,19 @@ msgstr "Raison"
 msgid "Request Signature Key"
 msgstr "Clé de signature de la requête"
 
-#: wf/wscall.py wscalls.py
-msgid "Post formdata"
-msgstr "Envoyer les données du formulaire"
+#: wf/wscall.py
+msgid "Post complete card/form data"
+msgstr "Transmettre toutes les données de la demande/fiche"
+
+#: wf/wscall.py
+msgid ""
+"Warning: this option sends the full content of the card/form, with "
+"additional POST data in an additional \"extra\" key. It is often not "
+"necessary."
+msgstr ""
+"Attention : cette option envoie l’intégralité des donneés de la demande ou "
+"fiche concernée, avec les données POST supplémentaires dans une clé "
+"« extra ». Cette option ne devrait généralement pas être utilisée."
 
 #: wf/wscall.py wscalls.py
 msgid "POST data"
@@ -11828,12 +11936,23 @@ msgid "Action on network errors"
 msgstr "Action en cas d’erreur réseau"
 
 #: wf/wscall.py
-msgid "Record errors in the log"
-msgstr "Enregistrer les erreurs dans l’historique"
+msgid "Notify errors by email"
+msgstr "Notifier les erreurs par courriel "
 
-#: wf/wscall.py workflows.py
-msgid "Set marker to jump back to current status"
-msgstr "Poser un marqueur qui permettra de revenir au statut actuel"
+#: wf/wscall.py
+#, python-format
+msgid "Error traces will be sent to %s"
+msgstr "Les traces des erreurs seront envoyées à %s."
+
+#: wf/wscall.py
+msgid ""
+"Record errors in the central error screen, for management by administrators"
+msgstr ""
+" Enregistrer les erreurs dans l’écran de centralisation des erreurs, pour traitement par l’équipe d’administration"
+
+#: wf/wscall.py
+msgid "Record errors in card/form history log, for agents"
+msgstr "Enregistrer les erreurs dans l’historique de la demande ou fiche, pour visualisation par les agents"
 
 #: wf/wscall.py
 msgctxt "wscall-parameter"
@@ -11877,6 +11996,15 @@ msgstr "Statut de la demande quand l’erreur s’est produite : %s"
 msgid "Simulate click on action button"
 msgstr "Clic sur un bouton d’action"
 
+#: workflow_tests.py
+msgid "Workflow has no action that displays a button."
+msgstr "Le workflow ne contient pas d’action qui affiche un bouton."
+
+#: workflow_tests.py
+#, python-format
+msgid "Click on \"%s\""
+msgstr "Clic sur « %s »"
+
 #: workflow_tests.py
 #, python-format
 msgid "Button \"%s\" is not displayed."
@@ -11894,6 +12022,11 @@ msgstr "Texte du bouton"
 msgid "Assert form status"
 msgstr "Vérifier le statut de la demande"
 
+#: workflow_tests.py
+#, python-format
+msgid "Status is \"%s\""
+msgstr "Le statut est « %s »"
+
 #: workflow_tests.py
 #, python-format
 msgid ""
@@ -11977,6 +12110,43 @@ msgstr ""
 "Mauvaise valeur pour la donnée de traitement « %(field)s » (devait valoir "
 "« %(expected_value)s » mais valait « %(value)s »."
 
+#: workflow_tests.py
+msgid "Assert webservice call"
+msgstr "Vérifier l’appel d’un webservice"
+
+#: workflow_tests.py
+msgid "Broken, missing webservice response"
+msgstr "Cassé, réponse webservice manquante"
+
+#: workflow_tests.py
+msgid ""
+"In order to assert a webservice is called, you must define corresponding "
+"webservice response."
+msgstr ""
+"Afin de vérifier l’appel d’un webservice, vous devez d’abord définir la "
+"réponse webservice correspondante."
+
+#: workflow_tests.py
+msgid "Add webservice response"
+msgstr "Ajouter une réponse webservice"
+
+#: workflow_tests.py
+#, python-format
+msgid ""
+"Webservice response %(name)s was used %(count)s times (instead of "
+"%(expected_count)s)."
+msgstr ""
+"La réponse webservice %(name)s a été utilisée %(count)s fois (au lieu de "
+"%(expected_count)s)."
+
+#: workflow_tests.py
+msgid "Webservice response"
+msgstr "Réponse webservice"
+
+#: workflow_tests.py
+msgid "Call count"
+msgstr "Nombre d’appels"
+
 #: workflow_traces.py
 msgid "Created (by API)"
 msgstr "Création (par l’API)"
@@ -12005,6 +12175,10 @@ msgstr "Suite"
 msgid "Created (by CSV import)"
 msgstr "Création (par importation CSV)"
 
+#: workflow_traces.py
+msgid "Updated (by CSV import)"
+msgstr "Mise à jour (par importation CSV)"
+
 #: workflow_traces.py
 msgid "Actions after edit action"
 msgstr "Actions après une édition"
@@ -12412,6 +12586,10 @@ msgstr "(conditionné)"
 msgid "Webservice call"
 msgstr "Appel de webservice"
 
+#: wscalls.py
+msgid "Post formdata"
+msgstr "Envoyer les données du formulaire"
+
 #: wscalls.py
 msgid "Timeout must be empty or a number."
 msgstr "L’expiration doit être un nombre, ou laissée vide."

wcs/portfolio.py

@@ -62,13 +62,12 @@ class fargo_post_json_async:
 def push_document(user, filename, stream):
     if not user:
         return
-    publisher = get_publisher()
     payload = {}
     if user.name_identifiers:
         payload['user_nameid'] = force_str(user.name_identifiers[0], 'ascii')
     elif user.email:
         payload['user_email'] = force_str(user.email, 'ascii')
-    payload['origin'] = urllib.parse.urlparse(publisher.get_frontoffice_url()).netloc
+    payload['origin'] = urllib.parse.urlparse(get_publisher().get_frontoffice_url()).netloc
     payload['file_name'] = filename
     stream.seek(0)
     payload['file_b64_content'] = force_str(base64.b64encode(stream.read()))
@@ -78,9 +77,10 @@ def push_document(user, filename, stream):
         status = 0
         status, payload = async_post()
         if status != 200:
-            publisher.record_error(
+            get_publisher().record_error(
                 _(
-                    'file %(filename)r failed to be pushed to portfolio of %(display_name)r [status: %(status)d, payload: %(payload)r]'
+                    'file %(filename)r failed to be pushed to portfolio of %(display_name)r '
+                    '[status: %(status)d, payload: %(payload)r]'
                 )
                 % {
                     'filename': filename,

wcs/publisher.py

@@ -171,6 +171,11 @@ class WcsPublisher(QommonPublisher):
 
         cls.register_cronjob(CronJob(Audit.clean, name='clean_audit', hours=[3], minutes=[0]))
 
+        # once a day clean old test results
+        from .testdef import TestResult
+
+        cls.register_cronjob(CronJob(TestResult.clean, name='clean_test_result', hours=[4], minutes=[0]))
+
         # other jobs
         data_sources.register_cronjob()
         formdef.register_cronjobs()
@@ -310,6 +315,7 @@ class WcsPublisher(QommonPublisher):
                             ('misc', ('default-position', 'default-zoom-level')),
                             ('sms', '*'),
                             ('submission-channels', '*'),
+                            ('backoffice-submission', '*'),
                             ('texts', '*'),
                             ('users', ('*_template',)),
                         ):
@@ -592,11 +598,11 @@ class WcsPublisher(QommonPublisher):
 
     @contextmanager
     def complex_data(self):
-        self.complex_data_cache = {}
+        old_complex_data_cache, self.complex_data_cache = self.complex_data_cache, {}
         try:
             yield True
         finally:
-            self.complex_data_cache = None
+            self.complex_data_cache = old_complex_data_cache
 
     def cache_complex_data(self, value, rendered_value):
         # Keep a temporary cache of assocations between a complex data value

wcs/qommon/afterjobs.py

@@ -72,6 +72,9 @@ class AfterJob(StorableObject):
         self.status = N_('registered')
         self.kwargs = kwargs
 
+    def __repr__(self):
+        return '<AfterJob id:%s cmd:%r>' % (self.id, self.job_cmd)
+
     def done_action_label(self):
         return self.done_action_label_arg
 
@@ -128,7 +131,7 @@ class AfterJob(StorableObject):
             if getattr(self, 'raise_exception', False):
                 raise
             get_publisher().capture_exception(sys.exc_info())
-            get_publisher().record_error(exception=e, notify=True)
+            get_publisher().record_error(exception=e, record=False, notify=True)
             self.exception = traceback.format_exc()
             self.status = N_('failed')
         else:

wcs/qommon/ctl.py

@@ -1,161 +0,0 @@
-# w.c.s. - web application for online forms
-# Copyright (C) 2005-2010  Entr'ouvert
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, see <http://www.gnu.org/licenses/>.
-
-import configparser
-import optparse  # noqa pylint: disable=deprecated-module
-import os
-import sys
-from optparse import make_option  # noqa pylint: disable=deprecated-module
-
-__all__ = [
-    'Command',
-]
-
-from wcs import qommon
-
-from . import _
-
-qommon._commands = {}
-
-
-class Command:
-    doc = ''
-    name = None
-    usage_args = '[ options ... ]'
-
-    def __init__(self, options=None):
-        options = options or []
-        self.config = configparser.ConfigParser()
-        self.options = options + [
-            make_option('--app-dir', metavar='DIR', action='store', dest='app_dir', default=None),
-            make_option('--data-dir', metavar='DIR', action='store', dest='data_dir', default=None),
-        ]
-
-    def run(self, args, base_options):
-        if base_options.configfile:
-            if not os.path.exists(base_options.configfile):
-                print('Missing configuration file %s' % base_options.configfile, file=sys.stderr)
-                sys.exit(1)
-            try:
-                self.config.read(base_options.configfile)
-            except configparser.ParsingError as e:
-                print('Invalid configuration file %s' % base_options.configfile, file=sys.stderr)
-                print(e, file=sys.stderr)
-                sys.exit(1)
-        if not self.config.has_section('main'):
-            self.config.add_section('main')
-        sub_options, args = self.parse_args(args)
-        if sub_options.app_dir:
-            self.config.set('main', 'app_dir', sub_options.app_dir)
-        if sub_options.data_dir:
-            self.config.set('main', 'data_dir', sub_options.data_dir)
-        return self.execute(base_options, sub_options, args)
-
-    def parse_args(self, args):
-        self.parser = optparse.OptionParser(
-            usage='%%prog %s %s' % (self.name, _(self.usage_args)), description=self.doc
-        )
-        self.parser.add_options(self.options)
-        return self.parser.parse_args(args)
-
-    def execute(self, base_options, sub_options, args):
-        """The body of the command"""
-        raise NotImplementedError
-
-    @classmethod
-    def register(cls):
-        qommon._commands[cls.name] = cls
-
-
-class Ctl:
-    def __init__(self, cmd_prefixes=None):
-        self.cmd_prefixes = cmd_prefixes or []
-        self.parser = optparse.OptionParser(
-            usage='%prog [ -f config ] command [ options ... ]', add_help_option=False
-        )
-        self.parser.disable_interspersed_args()
-        self.parser.add_option(
-            '-f',
-            '--file',
-            action='store',
-            metavar='CONFIG',
-            type='string',
-            dest='configfile',
-            help=_('use a non default configuration file'),
-        )
-        self.parser.add_option(
-            '--help', action='callback', callback=self.print_help, help=_('Display this help and exit')
-        )
-
-    def load_all_commands(self, ignore_errors=True):
-        for cmd_prefix in self.cmd_prefixes:
-            if not cmd_prefix in sys.modules:
-                __import__(cmd_prefix)
-            mod = sys.modules.get(cmd_prefix)
-            if not mod:
-                continue
-            if os.path.isdir(mod.__file__):
-                cmddir = os.path.abspath(mod.__file__)
-            else:
-                cmddir = os.path.abspath(os.path.dirname(mod.__file__))
-            for fname in os.listdir(os.path.join(cmddir)):
-                name, ext = os.path.splitext(fname)
-                if not ext == '.py':
-                    continue
-                if name.startswith('_'):
-                    continue
-                try:
-                    __import__('%s.%s' % (cmd_prefix, name))
-                except ImportError as e:
-                    if not ignore_errors:
-                        raise e
-
-    def get_commands(self):
-        return qommon._commands
-
-    def print_help(self, *args):
-        self.parser.print_help()
-        self.load_all_commands()
-        print()
-
-        commands = sorted((x.name, x.doc) for x in self.get_commands().values())
-        print('Available commands:')
-        for name, description in commands:
-            print('  %-15s %s' % (name, description))
-        sys.exit(0)
-
-    def run(self, args):
-        options, args = self.parser.parse_args(args)
-        if not args:
-            self.parser.error('You must use a command')
-        command, args = args[0], args[1:]
-
-        if command not in self.get_commands():
-            # load a module named like the command, this is the common case
-            for cmd_prefix in self.cmd_prefixes:
-                try:
-                    __import__('%s.%s' % (cmd_prefix, command))
-                except ImportError:
-                    pass
-
-        if command not in self.get_commands():
-            # if the command could not be loaded from a same-name module,
-            # go over all modules
-            self.load_all_commands()
-
-        command_class = self.get_commands()[command]
-        cmd = command_class()
-        return cmd.run(args, options)

wcs/qommon/evalutils.py

@@ -180,3 +180,9 @@ def dict_from_prefix(prefix, in_dict):
              b64_content  = wf_data['akey_b64_content']
     """
     return {k[len(prefix) :]: v for k, v in in_dict.items() if k.startswith('%s' % prefix)}
+
+
+def details_format(value, format=None):
+    # render form_details as plain text
+    # (for now this is the only possible output so it just returns the value as is)
+    return str(value)

wcs/qommon/form.py

@@ -3636,6 +3636,7 @@ class MapWidget(CompositeWidget):
                         # lat;lon
                         self.map_attributes['data-def-lat'] = position.split(';')[0]
                         self.map_attributes['data-def-lng'] = position.split(';')[1]
+                        self.map_attributes['data-def-template'] = 'true'
                     else:
                         # address?
                         from wcs.wf.geolocate import GeolocateWorkflowStatusItem

wcs/qommon/http_response.py

@@ -28,6 +28,7 @@ class HTTPResponse(quixote.http_response.HTTPResponse):
     javascript_code_parts = None
     css_includes = None
     after_jobs = None
+    raw = False  # in case of html content, send result as is (True) or embedded in page template (False)
 
     def __init__(self, charset=None, **kwargs):
         quixote.http_response.HTTPResponse.__init__(self, charset=charset, **kwargs)

wcs/qommon/management/commands/cron.py

@@ -47,8 +47,10 @@ class Command(BaseCommand):
                 print('Command is ignored because DISABLE_CRON_JOBS is set in settings')
             return
         if domain:
+            single_tenant = True
             domains = [domain]
         else:
+            single_tenant = False
             domains = [x.hostname for x in get_publisher_class().get_tenants()]
         if not job_name and verbosity > 2:
             print('cron start')
@@ -63,8 +65,8 @@ class Command(BaseCommand):
             # exit early if maximum number of workers has been reached
             running = 0
             stalled_tenants = []
-            for hostname in domains:
-                publisher.set_tenant_by_hostname(hostname)
+            for domain in domains:
+                publisher.set_tenant_by_hostname(domain)
                 if not publisher.has_postgresql_config():
                     continue
                 publisher.set_sql_application_name('wcs-cron')
@@ -72,8 +74,8 @@ class Command(BaseCommand):
                 if status == 'running':
                     running += 1
                     if now() - timestamp > datetime.timedelta(hours=6):
-                        stalled_tenants.append(hostname)
-                        CronJob.log('stalled tenant: %s' % hostname)
+                        stalled_tenants.append(domain)
+                        CronJob.log('stalled tenant: %s' % domain)
                         sql.mark_cron_status('done')
 
             if stalled_tenants:
@@ -82,29 +84,29 @@ class Command(BaseCommand):
                 CronJob.log('skipped, too many workers')
                 return
 
-        for hostname in domains:
-            publisher.set_tenant_by_hostname(hostname)
+        for domain in domains:
+            publisher.set_tenant_by_hostname(domain)
             if publisher.get_site_option('disable_cron_jobs', 'variables'):
                 if verbosity > 1:
-                    print('cron ignored on %s because DISABLE_CRON_JOBS is set' % hostname)
+                    print('cron ignored on %s because DISABLE_CRON_JOBS is set' % domain)
                 continue
             if not publisher.has_postgresql_config():
                 if verbosity > 1:
-                    print('cron ignored on %s because it has no PostgreSQL configuration' % hostname)
+                    print('cron ignored on %s because it has no PostgreSQL configuration' % domain)
                 continue
             publisher.set_sql_application_name('wcs-cron')
-            if domain:
+            if single_tenant:
                 cron_status, timestamp = 'ignored', now()
             else:
                 cron_status, timestamp = sql.get_and_update_cron_status()
             if not options['force_job']:
                 if cron_status == 'running':
                     if verbosity > 1:
-                        print(hostname, 'skip running, already handled')
+                        print(domain, 'skip running, already handled')
                     continue
-            setproctitle.setproctitle(sys.argv[0] + ' cron [%s]' % hostname)
+            setproctitle.setproctitle(sys.argv[0] + ' cron [%s]' % domain)
             if verbosity > 1:
-                print('cron work on %s' % hostname)
+                print('cron work on %s' % domain)
             CronJob.log('start')
             try:
                 cron_worker(publisher, timestamp, job_name=job_name)
@@ -113,7 +115,7 @@ class Command(BaseCommand):
                 publisher.capture_exception(sys.exc_info())
                 raise e
             finally:
-                if not domain:
+                if not single_tenant:
                     sql.mark_cron_status('done')
 
         if verbosity > 1:

wcs/qommon/misc.py

@@ -275,11 +275,11 @@ def site_encode(s):
     return force_str(s)
 
 
-def ellipsize(s, length=30):
+def ellipsize(s, length=30, truncate='(…)'):
     s = force_str(s)
     if s and len(s) > length:
         if length > 3:
-            s = Truncator(s).chars(length, truncate='(…)')
+            s = Truncator(s).chars(length, truncate=truncate)
         else:
             s = s[:length]
     return force_str(s)
@@ -1346,7 +1346,7 @@ def get_dependencies_from_template(string):
 
 def parse_decimal(value, do_raise=False, keep_none=False):
     value = unlazy(value)
-    if keep_none and value is None:
+    if keep_none and (value is None or value == ''):
         return None
     if isinstance(value, bool):
         # treat all booleans as 0 (contrary to Python behaviour where

wcs/qommon/static/css/dc2/admin.scss

@@ -3096,13 +3096,10 @@ ul.objects-list.single-links li.loading-list-item {
 	}
 }
 
-ul.objects-list.single-links li.loading-list-item span {
-	padding: 0 0.5ex 0 2ex;
-}
-
+ul.objects-list.single-links li.loading-list-item p,
 ul.objects-list.single-links li.list-item-no-usage p {
+	margin: 0.5em 0 0 0;
 	padding: 0 0.5ex 0 2ex;
-	margin: 0;
 }
 
 .inline-hint-message {
@@ -3131,3 +3128,11 @@ ul.objects-list.single-links li a.link-action-icon.duplicate {
 		content: "\f24d"; /* clone */
 	}
 }
+
+div[role="tabpanel"] > div.infonotice:first-child {
+	margin-top: 0;
+}
+
+form div.widget[data-widget-name="model_file_mode"] {
+	margin-bottom: 0;
+}

wcs/qommon/static/js/qommon.forms.js

@@ -478,7 +478,13 @@ $(function() {
     if ($form.hasClass('download-button-clicked')) {
       /* form cannot be disabled for download buttons as the user will stay on
        * the same page; enable it back after a few seconds. */
-      setTimeout(function() { $form.removeClass('disabled-during-submit'); }, 3000);
+      setTimeout(function() {
+        /* request new _ts value */
+        $.getJSON(window.location.pathname + 'tsupdate', function(data) {
+          $form.find('[type="hidden"][name="_ts"]').val(data.ts);
+          $form.removeClass('disabled-during-submit');
+        });
+      }, 3000);
     }
     if ($form[0].wait_for_changes) {
       var waited = 0;
@@ -563,7 +569,7 @@ $(function() {
           if (value.visible) {
             var was_visible = $widget.is(':visible');
             $widget.css('display', '');
-            if ($widget.hasClass('MapWidget') && !was_visible) {
+            if (($widget.hasClass('MapWidget') || $widget.hasClass('BlockWidget')) && !was_visible) {
               // add mini-delay to workaround wrong width calculation bug
               setTimeout(function() { $widget.find('.qommon-map').trigger('qommon:invalidate'); }, 10);
             }

wcs/qommon/substitution.py

@@ -190,7 +190,8 @@ class CompatibilityNamesDict(dict):
             item = self[base]
             flat_keys[base] = item
             if hasattr(item, 'inspect_keys'):
-                sub_keys = item.inspect_keys()
+                sub_keys = list(item.inspect_keys())
+                assert len(sub_keys) == len(set(sub_keys))
             elif isinstance(item, dict):
                 sub_keys = [x for x in item.keys() if self.valid_key_regex.match(x)]
             else:

wcs/qommon/template.py

@@ -78,12 +78,17 @@ def error_page(error_message, error_title=None, location_hint=None):
 def get_decorate_vars(body, response, generate_breadcrumb=True, **kwargs):
     from .publisher import get_cfg
 
-    if response.content_type != 'text/html':
+    if response.content_type != 'text/html' or response.raw:
         return {'body': body}
 
-    body = str(body)
     if get_request().get_header('x-popup') == 'true':
-        return {'body': body}
+        if isinstance(body, QommonTemplateResponse):
+            body.add_media()
+            if body.is_django_native:
+                body = render(body.templates, body.context)
+        return {'body': str(body)}
+
+    body = str(body)
 
     kwargs = {}
     for k, v in response.filter.items():

wcs/qommon/templates/qommon/forms/widgets/mini-rich-text.html

@@ -1,9 +1,16 @@
 {% extends "qommon/forms/widget.html" %}
 {% block widget-control %}
-  <textarea style="width: 100%" id="form_{{widget.get_name_for_id}}" name="{{widget.name}}"
-            {% for attr in widget.attrs.items %}{{attr.0}}="{{attr.1}}" {% endfor %}
-            {% if widget.live_condition_source %}data-godo-instant-update="true"{% endif %}
-            data-godo-schema="{{widget.EDITION_MODE}}"
-            data-godo-update-event="wcs:live-update">{{widget.value|default:""}}</textarea>
+  <textarea hidden id="form_{{widget.get_name_for_id}}" name="{{widget.name}}">
+    {{widget.value|default:""}}
+  </textarea>
+  <godo-editor
+    style="width: 100%"
+    {% for attr in widget.attrs.items %}{{attr.0}}="{{attr.1}}" {% endfor %}
+    {% if widget.live_condition_source %}instant-update="true"{% endif %}
+    schema="{{widget.EDITION_MODE}}"
+    update-event="wcs:live-update"
+    linked-source="form_{{widget.get_name_for_id}}"
+  >
+  </godo-editor>
   <script type="module" src="/static/xstatic/js/godo.js?{{version_hash}}"></script>
 {% endblock %}

wcs/qommon/templatetags/qommon.py

@@ -24,6 +24,7 @@ import math
 import os
 import random
 import string
+import subprocess
 import urllib.parse
 from decimal import Decimal
 from decimal import DivisionByZero as DecimalDivisionByZero
@@ -58,6 +59,7 @@ from wcs.qommon import _, calendar, evalutils, upload_storage
 from wcs.qommon.admin.texts import TextsDirectory
 from wcs.qommon.humantime import seconds2humanduration
 from wcs.qommon.misc import parse_decimal, strip_some_tags, unlazy, validate_phone_fr
+from wcs.qommon.template import TemplateError
 
 register = template.Library()
 
@@ -506,12 +508,14 @@ def add_javascript(js_id):
 
 
 @register.simple_tag(takes_context=True)
-def action_button(context, action_id, label, delay=3, message=None, done_message=None):
+def action_button(context, action_id, label=None, delay=3, message=None, done_message=None):
     formdata_id = context.get('form_number_raw')
     formdef_urlname = context.get('form_slug')
     formdef_type = context.get('form_type')
     if not (formdef_urlname and formdata_id):
         return ''
+    if not label:
+        raise TemplateError(_('{%% action_button %%} requires a label parameter'))
     token = get_publisher().token_class(expiration_delay=delay * 86400, size=64)
     token.type = 'action'
     token.context = {
@@ -529,8 +533,10 @@ def action_button(context, action_id, label, delay=3, message=None, done_message
 
 @register.simple_tag(takes_context=True)
 def temporary_access_button(
-    context, label, days=None, hours=None, minutes=None, seconds=None, bypass_checks=False
+    context, label=None, days=None, hours=None, minutes=None, seconds=None, bypass_checks=False
 ):
+    if not label:
+        raise TemplateError(_('{%% temporary_action_button %%} requires a label parameter'))
     url = temporary_access_url(context, days=days, hours=hours, minutes=minutes, bypass_checks=bypass_checks)
     if not url:
         return ''
@@ -1081,6 +1087,58 @@ def rename_file(value, new_name):
     return file_object
 
 
+@register.filter
+def convert_image_format(value, new_format):
+    from wcs.fields import FileField
+
+    formats = {
+        'jpeg': 'image/jpeg',
+        'pdf': 'application/pdf',
+        'png': 'image/png',
+    }
+    if new_format not in formats:
+        get_publisher().record_error(
+            _('|convert_image_format: unknown format (must be one of %s)') % ', '.join(formats.keys())
+        )
+        return None
+
+    try:
+        file_object = FileField.convert_value_from_anything(value)
+    except ValueError:
+        file_object = None
+    if not file_object:
+        get_publisher().record_error(_('|convert_image_format: missing input'))
+        return None
+
+    if file_object.base_filename:
+        current_name, current_format = os.path.splitext(file_object.base_filename)
+        if current_format == f'.{new_format}':
+            return file_object
+        new_name = f'{current_name}.{new_format}'
+    else:
+        new_name = '%s.%s' % (_('file'), new_format)
+
+    try:
+        proc = subprocess.run(
+            ['gm', 'convert', '-', f'{new_format}:-'],
+            input=file_object.get_content(),
+            capture_output=True,
+            check=True,
+        )
+    except FileNotFoundError:
+        get_publisher().record_error(_('|convert_image_format: not supported'))
+        return None
+    except subprocess.CalledProcessError as e:
+        get_publisher().record_error(_('|convert_image_format: conversion error (%s)' % e.stderr.decode()))
+        return None
+
+    new_file_object = FileField.convert_value_from_anything(
+        {'content': proc.stdout, 'filename': new_name, 'content_type': formats[new_format]}
+    )
+
+    return new_file_object
+
+
 @register.filter
 def first(value):
     try:
@@ -1272,7 +1330,7 @@ def json_dumps(value):
 
 
 @register.simple_tag(takes_context=True)
-def make_public_url(context, url):
+def make_public_url(context, url=None):
     if not url:
         return ''
     token = get_session().create_token('sign-url-token', {'url': url})
@@ -1298,3 +1356,14 @@ def check_no_duplicates(value):
         get_publisher().record_error(_('|check_no_duplicates not used on a list (%s)') % value)
         return False
     return bool(len(value or []) == len(set(value or [])))
+
+
+@register.filter
+def details_format(value, format=None):
+    if format is None:
+        get_publisher().record_error(_('|details_format called without specifying a format'))
+        return ''
+    if format not in ('text',):
+        get_publisher().record_error(_('|details_format called with unknown format (%s)') % format)
+        return ''
+    return evalutils.details_format(value, format=format)

wcs/sql.py

@@ -4086,6 +4086,7 @@ class TestResult(SqlMixin):
         ('reason', 'varchar'),
         ('results', 'jsonb[]'),
     ]
+    _table_select_skipped_fields = ['results']
 
     id = None
 
@@ -4161,7 +4162,9 @@ class TestResult(SqlMixin):
 
     @classmethod
     def migrate_legacy(cls):
-        for test_result in TestResult.select():
+        skipped_fields = cls._table_select_skipped_fields.copy()
+        cls._table_select_skipped_fields = []
+        for test_result in cls.select():
             store = False
             for result in test_result.results:
                 if 'details' not in result:
@@ -4176,6 +4179,8 @@ class TestResult(SqlMixin):
             if store:
                 test_result.store()
 
+        cls._table_select_skipped_fields = skipped_fields
+
 
 class WorkflowTrace(SqlMixin):
     _table_name = 'workflow_traces'

wcs/statistics/views.py

@@ -232,8 +232,8 @@ class FormsCountView(RestrictedView):
         }
         group_by = request.GET.get('group-by')
         group_labels = {}
-        subfilters = self.get_common_subfilters(time_interval)
 
+        formdefs = []
         slugs = request.GET.getlist('form', ['_all'] if self.has_global_count_support else ['_nothing'])
         if slugs != ['_all']:
             formdef_slugs = [x for x in slugs if not x.startswith('category:')]
@@ -244,22 +244,17 @@ class FormsCountView(RestrictedView):
             if not formdefs:
                 raise TraversalError()
 
+        subfilters = self.get_common_subfilters(time_interval, formdefs)
+
+        if formdefs:
             for formdef in formdefs:
                 formdef.form_page = self.formpage_class(formdef=formdef, update_breadcrumbs=False)
 
             self.set_formdef_parameters(totals_kwargs, formdefs)
             totals_kwargs['criterias'].extend(self.get_filters_criterias(formdefs))
-            self.set_group_by_parameters(group_by, formdefs, totals_kwargs, group_labels)
-            subfilters += self.get_formdefs_subfilters(formdefs, group_by, time_interval)
-        else:
-            subfilters += [
-                {'id': 'group-by', 'label': _('Group by'), 'options': [{'id': 'form', 'label': _('Form')}]}
-            ]
-            if group_by == 'form':
-                totals_kwargs['group_by'] = 'formdef_id'
-                group_labels = {
-                    int(x.id): x.name for x in self.formdef_class.select(lightweight=True, order_by='name')
-                }
+            self.add_formdefs_subfilters(subfilters, formdefs, group_by, time_interval)
+
+        self.set_group_by_parameters(group_by, formdefs, totals_kwargs, group_labels)
 
         channel = request.GET.get('channel', '_all')
         if channel in ('web', 'backoffice'):
@@ -366,7 +361,7 @@ class FormsCountView(RestrictedView):
 
         return criterias
 
-    def get_common_subfilters(self, time_interval):
+    def get_common_subfilters(self, time_interval, formdefs):
         subfilters = []
 
         if time_interval == 'month':
@@ -384,12 +379,27 @@ class FormsCountView(RestrictedView):
                 }
             )
 
+        group_by_filter = {
+            'id': 'group-by',
+            'label': _('Group by'),
+            'has_subfilters': True,
+            'options': [
+                {'id': 'channel', 'label': _('Channel')},
+            ],
+        }
+
+        if len(formdefs) != 1:
+            # allow grouping by form only if no form is selected or many are selected
+            group_by_filter['options'].append({'id': 'form', 'label': _('Form')})
+
+        subfilters.append(group_by_filter)
+
         return subfilters
 
-    def get_formdefs_subfilters(self, formdefs, group_by, time_interval):
+    def add_formdefs_subfilters(self, common_subfilters, formdefs, group_by, time_interval):
         subfilters = None
         for formdef in formdefs:
-            new_subfilters = self.get_form_subfilters(formdef.form_page, group_by)
+            new_subfilters = self.get_form_subfilters(formdef.form_page)
 
             if not subfilters:
                 subfilters = new_subfilters
@@ -400,7 +410,7 @@ class FormsCountView(RestrictedView):
 
             for filter_id, subfilter in subfilters.copy().items():
                 if subfilter['options'] != new_subfilters[filter_id]['options']:
-                    if filter_id in ('filter-status', 'group-by'):
+                    if filter_id in ('filter-status'):
                         # keep only common options
                         subfilter['options'] = {
                             k: v
@@ -421,11 +431,29 @@ class FormsCountView(RestrictedView):
             if needs_sorting:
                 subfilter['options'].sort(key=lambda x: x['label'])
 
-        return subfilters
+        group_by_filter = [x for x in common_subfilters if x['id'] == 'group-by'][0]
+        group_by_filter['options'].append({'id': 'simple-status', 'label': _('Simplified status')})
+        group_by_filter['options'].extend(
+            [{'id': x['id'].removeprefix('filter-'), 'label': x['label']} for x in subfilters]
+        )
 
-    def get_form_subfilters(self, form_page, group_by):
+        if group_by not in (None, 'channel', 'simple-status', 'status'):
+            group_by_field = self.get_group_by_field(formdefs[0].form_page, group_by)
+            if group_by_field:
+                common_subfilters.append(
+                    {
+                        'id': 'hide_none_label',
+                        'label': _('Ignore forms where "%s" is empty.') % group_by_field.label,
+                        'options': [{'id': 'true', 'label': _('Yes')}, {'id': 'false', 'label': _('No')}],
+                        'required': True,
+                        'default': 'false',
+                    }
+                )
+
+        common_subfilters.extend(subfilters)
+
+    def get_form_subfilters(self, form_page):
         subfilters = []
-        field_choices = []
         for field in form_page.get_formdef_fields(include_block_items_fields=True):
             if not getattr(field, 'include_in_statistics', False) or not field.contextual_varname:
                 continue
@@ -466,36 +494,6 @@ class FormsCountView(RestrictedView):
                 filter_description['default'] = field.default_filter_value
 
             subfilters.append(filter_description)
-            field_choices.append((field.contextual_varname, field.label))
-
-        if field_choices:
-            additionnal_filters = [
-                {
-                    'id': 'group-by',
-                    'label': _('Group by'),
-                    'options': {
-                        'channel': _('Channel'),
-                        'simple-status': _('Simplified status'),
-                        **{x[0]: x[1] for x in field_choices},
-                    },
-                    'has_subfilters': True,
-                }
-            ]
-
-            if group_by not in (None, 'channel', 'simple-status', 'status'):
-                group_by_field = self.get_group_by_field(form_page, group_by)
-                if group_by_field:
-                    additionnal_filters.append(
-                        {
-                            'id': 'hide_none_label',
-                            'label': _('Ignore forms where "%s" is empty.') % group_by_field.label,
-                            'options': {'true': _('Yes'), 'false': _('No')},
-                            'required': True,
-                            'default': 'false',
-                        }
-                    )
-
-            subfilters = additionnal_filters + subfilters
 
         subfilters = {x['id']: x for x in subfilters}
         return subfilters
@@ -536,6 +534,13 @@ class FormsCountView(RestrictedView):
         if not group_by:
             return
 
+        if group_by == 'form':
+            totals_kwargs['group_by'] = 'formdef_id'
+            group_labels.update(
+                {int(x.id): x.name for x in self.formdef_class.select(lightweight=True, order_by='name')}
+            )
+            return
+
         if group_by == 'channel':
             totals_kwargs['group_by'] = 'submission_channel_new'
             totals_kwargs['group_by_clause'] = (

wcs/templates/wcs/backoffice/includes/inspect-field.html

@@ -1,6 +1,6 @@
 <div class="inspect-field inspect-field--{{ field.key }} {% if forloop.first %}inspect-field--first{% endif %}"
      data-field-id="{{ field.id }}">
-  <h4><a href="{{ path|default:"fields/" }}{{ field.id }}/">{{ field.ellipsized_label }}</a>
+  <h4><a href="{{ path|default:"fields/" }}{% if field.on_page %}pages/{{ field.on_page.id }}/{% endif %}{{ field.id }}/">{{ field.ellipsized_label }}</a>
     <span class="inspect-field-type">-
       {% if field.key == 'block' %}<a href="{{ field.block.get_admin_url }}inspect">{% endif %}
       {{ field.get_type_label }}

wcs/templates/wcs/backoffice/submission.html

@@ -0,0 +1,29 @@
+{% extends "wcs/backoffice.html" %}
+{% load i18n %}
+
+{% block appbar-title %}{% trans "Submission" %}{% endblock %}
+
+{% block appbar-actions %}
+  <a href="pending">{% trans "Pending submissions" %}</a>
+{% endblock %}
+
+{% block content %}
+  {{ block.super }}
+
+  {% for category in categories %}
+    {% if category.formdefs %}
+      {% with section_folded_pref_name="folded-submission"|add:"-category-"|add:category.id %}
+        <div class="section foldable {% if user|get_preference:section_folded_pref_name %}folded{% endif %}"
+             data-section-folded-pref-name="{{ section_folded_pref_name }}">
+          <h2>{{ category.name }}</h2>
+          <ul class="objects-list single-links">
+            {% for formdef in category.formdefs %}
+              <li><a href="{{ formdef.url_name }}/">{{ formdef.name }}</a></li>
+            {% endfor %}
+          </ul>
+        </div>
+      {% endwith %}
+    {% endif %}
+  {% endfor %}
+
+{% endblock %}

wcs/templates/wcs/backoffice/test-result-detail.html

@@ -8,6 +8,16 @@
 {% block body %}
   <div class="section">
     <ul>
+      {% if result.error_field_id %}
+        <li>
+          {% trans "Field linked to error:" %}
+          {% if error_field %}
+            <a href="{{ error_field.url }}">{{ error_field.label }}</a>
+          {% else %}
+            {% trans "deleted" %}
+          {% endif %}
+        </li>
+      {% endif %}
       {% if result.workflow_test_action_uuid %}
         <li id='test-action'>
           {% trans "Test action:" %}

wcs/templates/wcs/backoffice/test-result.html

@@ -20,6 +20,7 @@
         <thead>
           <th>{% trans "Name" %}</th>
           <th>{% trans "Result" %}</th>
+          <th>{% trans "Inspect" %}</th>
           <th>{% trans "Details" %}</th>
         </thead>
         <tbody>
@@ -27,6 +28,11 @@
             <tr>
               <td><a {% if test.url %}href="{{ test.url }}"{% else %}disabled{% endif %}>{{ test.name }}</a></td>
               <td>{% firstof test.error _("Success!") %}</td>
+              <td>
+                {% if test.formdata %}
+                  <a href="{{ forloop.counter0 }}/inspect">{% trans "Display inspect" %}</a>
+                {% endif %}
+              </td>
               <td>
                 {% if test.has_details %}
                   <a rel="popup" data-selector="div.section" href="{{ forloop.counter0 }}/">{% trans "Display details" %}</a>

wcs/templates/wcs/backoffice/wscall.html

@@ -50,7 +50,7 @@
   <div class="section">
     <h3>{% trans "Usage" %}</h3>
     <ul class="objects-list single-links" data-async-url="{{ publisher.get_request.get_path }}usage">
-      <li class="loading-list-item"><span>{% trans "Searching..." %}</span></li>
+      <li class="loading-list-item"><p>{% trans "Searching..." %}</p></li>
     </ul>
   </div>
 {% endblock %}

wcs/testdef.py

@@ -14,6 +14,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+import collections
+import datetime
 import http
 import io
 import json
@@ -24,6 +26,7 @@ from contextlib import contextmanager
 
 import requests
 from django.core.handlers.wsgi import WSGIRequest
+from django.utils.timezone import now
 from quixote import get_publisher, get_session_manager
 from urllib3 import HTTPResponse
 
@@ -42,10 +45,11 @@ from .qommon import _
 class TestError(Exception):
     action_uuid = None
 
-    def __init__(self, msg, error=None, details=None):
+    def __init__(self, msg, error=None, details=None, field_id=None):
         self.msg = msg
         self.error = error or msg
         self.details = details or []
+        self.field_id = field_id
 
     # prevent pytest from trying to collect this class (#75521)
     __test__ = False
@@ -139,6 +143,7 @@ class TestDef(sql.TestDef):
 
         workflow_tests_list = WorkflowTests.select([Equal('testdef_id', self.id)])
         self._workflow_tests = workflow_tests_list[0] if workflow_tests_list else WorkflowTests()
+        self._workflow_tests.testdef = self
         return self._workflow_tests
 
     @workflow_tests.setter
@@ -157,6 +162,7 @@ class TestDef(sql.TestDef):
         super().store(*args, **kwargs)
 
         self.workflow_tests.testdef_id = self.id
+        self.workflow_tests.testdef = self
         self.workflow_tests.store()
 
     @classmethod
@@ -213,6 +219,8 @@ class TestDef(sql.TestDef):
         formdata = objectdef.data_class()()
         formdata.just_created()
 
+        formdata.workflow_traces = []
+
         if self.data['user']:
             formdata.set_user_from_json(self.data['user'])
 
@@ -255,6 +263,7 @@ class TestDef(sql.TestDef):
     def run(self, objectdef):
         self.exception = None
         self.sent_requests = []
+        self.used_webservice_responses = []
         self.recorded_errors = []
         self.missing_required_fields = []
         with self.fake_request():
@@ -267,7 +276,8 @@ class TestDef(sql.TestDef):
                 if e.error != self.expected_error:
                     raise TestError(
                         _('Expected error "%(expected_error)s" but got error "%(error)s" instead.')
-                        % {'expected_error': self.expected_error, 'error': e.error}
+                        % {'expected_error': self.expected_error, 'error': e.error},
+                        field_id=e.field_id,
                     )
             else:
                 if self.expected_error:
@@ -277,12 +287,12 @@ class TestDef(sql.TestDef):
 
     def _run(self, objectdef):
         formdata = self.run_form_fill(objectdef)
-        if self.agent_id:
+        if self.agent_id and self.workflow_tests.actions:
             agent_user = get_publisher().user_class.get(self.agent_id)
             self.workflow_tests.run(formdata, agent_user)
 
     def run_form_fill(self, objectdef):
-        formdata = self.build_formdata(objectdef)
+        self.formdata = formdata = self.build_formdata(objectdef)
 
         get_publisher().reset_formdata_state()
         get_publisher().substitutions.feed(objectdef)
@@ -315,7 +325,8 @@ class TestDef(sql.TestDef):
                 if fields_with_data:
                     raise TestError(
                         _('Tried to fill field "%(label)s" on page %(no)d but page was not shown.')
-                        % {'label': fields_with_data[0].label, 'no': page.index}
+                        % {'label': fields_with_data[0].label, 'no': page.index},
+                        field_id=page.id,
                     )
                 continue
 
@@ -337,7 +348,8 @@ class TestDef(sql.TestDef):
                 if self.data['fields'].get(field.id) is not None:
                     raise TestError(
                         _('Tried to fill field "%(label)s" on page %(no)d but it is hidden.')
-                        % {'label': field.label, 'no': page.index}
+                        % {'label': field.label, 'no': page.index},
+                        field_id=field.id,
                     )
                 continue
 
@@ -386,9 +398,12 @@ class TestDef(sql.TestDef):
                         _('Page %(no)d post condition was not met (%(condition)s).')
                         % {'no': page.index, 'condition': condition.get('value')},
                         error=post_condition.get('error_message'),
+                        field_id=page.id,
                     )
             except RuntimeError:
-                raise TestError(_('Failed to evaluate page %d post condition.') % page.index)
+                raise TestError(
+                    _('Failed to evaluate page %d post condition.') % page.index, field_id=page.id
+                )
 
     def run_widget_validation(self, field, value):
         widget = field.add_to_form(self.form)
@@ -428,6 +443,7 @@ class TestDef(sql.TestDef):
                 'details': widget.error,
             },
             error=widget.error,
+            field_id=field.id,
         )
 
     def handle_computed_fields(self, fields, formdata, exclude_frozen=False):
@@ -539,6 +555,41 @@ class TestResult(sql.TestResult):
         objects_dir = 'forms' if self.object_type == 'formdefs' else 'cards'
         return '%s/%s/%s/tests/results/%s/' % (base_url, objects_dir, self.object_id, self.id)
 
+    @classmethod
+    def clean(cls, publisher=None, **kwargs):
+        test_results_by_formdef = collections.defaultdict(list)
+        for test_result in cls.select(order_by='-timestamp'):
+            test_results_by_formdef[(test_result.object_id, test_result.object_type)].append(test_result)
+
+        deletion_timestamp_by_formdef = {}
+        for formdef_key, test_results in test_results_by_formdef.items():
+            success = False
+            test_results_count = 0
+            for test_result in test_results:
+                test_results_count += 1
+
+                if (
+                    success
+                    and test_results_count > 10
+                    and test_result.timestamp < now() - datetime.timedelta(days=14)
+                ):
+                    break
+
+                success |= test_result.success
+            else:
+                continue
+
+            deletion_timestamp_by_formdef[formdef_key] = test_result.timestamp
+
+        for (object_id, object_type), deletion_timestamp in deletion_timestamp_by_formdef.items():
+            TestResult.wipe(
+                clause=[
+                    sql.LessOrEqual('timestamp', deletion_timestamp),
+                    sql.Equal('object_id', object_id),
+                    sql.Equal('object_type', object_type),
+                ]
+            )
+
 
 class WebserviceResponseError(Exception):
     pass
@@ -552,8 +603,8 @@ class MockWebserviceResponseAdapter(requests.adapters.HTTPAdapter):
     def send(self, request, *args, **kwargs):
         try:
             return self._send(request, *args, **kwargs)
-        except WebserviceResponseError:
-            raise requests.exceptions.RequestError
+        except WebserviceResponseError as e:
+            raise requests.RequestException(str(e))
         except Exception as e:
             # Webservice call can happen through templates which catch all exceptions.
             # Record error to ensure we have a trace nonetheless.
@@ -578,10 +629,11 @@ class MockWebserviceResponseAdapter(requests.adapters.HTTPAdapter):
         else:
             if request.method != 'GET':
                 request_info['forbidden_method'] = True
-                raise WebserviceResponseError
+                raise WebserviceResponseError(str(_('method must be GET')))
             return super().send(request, *args, **kwargs)
 
         request_info['webservice_response_id'] = response.id
+        self.testdef.used_webservice_responses.append(response)
 
         headers = {
             'Content-Type': 'application/json',

wcs/variables.py

@@ -756,6 +756,18 @@ class LazyFormDef:
     def frontoffice_submission_url(self):
         return self._formdef.get_url()
 
+    @property
+    def publication_disabled(self):
+        return self._formdef.is_disabled()
+
+    @property
+    def publication_datetime(self):
+        return self._formdef.publication_datetime
+
+    @property
+    def publication_expiration_datetime(self):
+        return self._formdef.expiration_datetime
+
 
 class LazyFormData(LazyFormDef):
     # noqa pylint: disable=too-many-public-methods

wcs/wf/backoffice_fields.py

@@ -187,11 +187,17 @@ class SetBackofficeFieldsWorkflowStatusItem(WorkflowStatusItem):
                 try:
                     new_value = formdef_field.convert_value_from_anything(new_value)
                 except ValueError as e:
-                    summary = _('Failed to convert %(class)s value to %(kind)s field (%(id)s)') % {
-                        'class': type(new_value),
-                        'kind': formdef_field.get_type_label(),
-                        'id': field['field_id'],
-                    }
+                    if hasattr(e, 'get_error_summary'):
+                        summary = _('Failed to assign field (%(id)s): %(summary)s') % {
+                            'id': field['field_id'],
+                            'summary': e.get_error_summary(),
+                        }
+                    else:
+                        summary = _('Failed to convert %(class)s value to %(kind)s field (%(id)s)') % {
+                            'class': type(new_value),
+                            'kind': formdef_field.get_type_label(),
+                            'id': field['field_id'],
+                        }
                     expression_dict = self.get_expression(field['value'])
                     get_publisher().record_error(
                         summary,

wcs/wf/editable.py

@@ -18,6 +18,7 @@ from quixote import get_publisher, get_request
 
 from wcs.qommon import _
 from wcs.qommon.form import (
+    CheckboxWidget,
     RadiobuttonsWidget,
     SingleSelectWidget,
     StringWidget,
@@ -41,10 +42,14 @@ class EditableWorkflowStatusItem(WorkflowStatusItem):
     backoffice_info_text = None
     operation_mode = 'full'  # or 'single' or 'partial'
     page_identifier = None
+    set_marker_on_status = False
 
     def get_line_details(self):
         if self.by:
-            return _('by %s') % self.render_list_of_roles(self.by)
+            return _('"%(button_label)s", by %(by)s') % {
+                'button_label': self.get_button_label(),
+                'by': self.render_list_of_roles(self.by),
+            }
         else:
             return _('not completed')
 
@@ -57,12 +62,13 @@ class EditableWorkflowStatusItem(WorkflowStatusItem):
         if self.label:
             yield location, None, self.label
 
-    def fill_form(self, form, formdata, user, **kwargs):
+    def get_button_label(self):
         if self.label:
-            label = get_publisher().translate(self.label)
-        else:
-            label = _('Edit Form')
-        widget = form.add_submit('button%s' % self.id, label)
+            return get_publisher().translate(self.label)
+        return _('Edit Form')
+
+    def fill_form(self, form, formdata, user, **kwargs):
+        widget = form.add_submit('button%s' % self.id, self.get_button_label())
         widget.backoffice_info_text = self.backoffice_info_text
         widget.ignore_form_errors = True
         widget.prevent_jump_on_submit = True
@@ -146,6 +152,14 @@ class EditableWorkflowStatusItem(WorkflowStatusItem):
                     'data-dynamic-display-value-in': 'single|partial',
                 },
             )
+        if 'set_marker_on_status' in parameters:
+            form.add(
+                CheckboxWidget,
+                '%sset_marker_on_status' % prefix,
+                title=_('Set marker to jump back to current status'),
+                value=self.set_marker_on_status,
+                advanced=True,
+            )
 
     def get_parameters(self):
         return (
@@ -156,6 +170,7 @@ class EditableWorkflowStatusItem(WorkflowStatusItem):
             'condition',
             'operation_mode',
             'page_identifier',
+            'set_marker_on_status',
         )
 
 

wcs/wf/export_to_model.py

@@ -34,7 +34,7 @@ from quixote.errors import TraversalError
 from quixote.html import htmltext
 from quixote.http_request import Upload
 
-from wcs.fields import CommentField, PageField, SubtitleField, TitleField
+from wcs.fields import FileField
 from wcs.portfolio import has_portfolio, push_document
 from wcs.workflows import (
     AttachmentEvolutionPart,
@@ -51,6 +51,7 @@ from ..qommon.form import (
     CheckboxWidget,
     ComputedExpressionWidget,
     FileWidget,
+    HtmlWidget,
     RadiobuttonsWidget,
     SingleSelectWidget,
     StringWidget,
@@ -198,24 +199,6 @@ class TemplatingError(TraversalError):
         self.description = description
 
 
-def get_varnames(fields):
-    """Extract variable names for helping people fill their templates.
-
-    Prefer to variable name to the numeric field name.
-    """
-    varnames = []
-    for field in fields:
-        if isinstance(field, (SubtitleField, TitleField, CommentField, PageField)):
-            continue
-        # add it as f$n$
-        label = field.label
-        if field.varname:
-            varnames.append(('var_%s' % field.varname, label))
-        else:
-            varnames.append(('f%s' % field.id, label))
-    return varnames
-
-
 class ExportToModelDirectory(Directory):
     _q_exports = ['']
 
@@ -224,21 +207,22 @@ class ExportToModelDirectory(Directory):
         self.wfstatusitem = wfstatusitem
 
     def _q_index(self):
-        if not self.wfstatusitem.model_file:
+        if not (self.wfstatusitem.model_file or self.wfstatusitem.model_file_template):
             raise TemplatingError(_('No model defined for this action'))
         response = get_response()
+        model_file = self.wfstatusitem.get_model_file()
         if self.wfstatusitem.convert_to_pdf:
             response.content_type = 'application/pdf'
         else:
-            response.content_type = self.wfstatusitem.model_file.content_type
+            response.content_type = model_file.content_type
         response.set_header('location', '..')
 
-        filename = self.wfstatusitem.get_filename()
+        filename = self.wfstatusitem.get_filename(model_file)
         if self.wfstatusitem.convert_to_pdf:
             filename = filename.rsplit('.', 1)[0] + '.pdf'
         if response.content_type != 'text/html':
             response.set_header('content-disposition', 'attachment; filename="%s"' % filename)
-        return self.wfstatusitem.apply_template_to_formdata(self.formdata).read()
+        return self.wfstatusitem.apply_template_to_formdata(self.formdata, model_file).read()
 
 
 class UploadValidationError(Exception):
@@ -279,7 +263,9 @@ class ExportToModel(WorkflowStatusItem):
     waitpoint = True
 
     label = None
+    model_file_mode = 'file'  # or 'template'
     model_file = None
+    model_file_template = None
     attach_to_history = False
     directory_class = ExportToModelDirectory
     by = ['_receiver']
@@ -291,19 +277,26 @@ class ExportToModel(WorkflowStatusItem):
     backoffice_filefield_id = None
 
     def get_line_details(self):
-        if self.model_file:
+        if self.model_file and self.model_file_mode == 'file':
             return _('with model named %(file_name)s of %(size)s') % {
                 'file_name': self.model_file.base_filename,
                 'size': filesizeformat(self.model_file.size),
             }
+        elif self.model_file_template and self.model_file_mode == 'template':
+            return _('with model from template')
         else:
             return _('no model set')
 
     def is_interactive(self):
         return bool(self.method == 'interactive')
 
+    def has_configured_model_file(self):
+        return (self.model_file_mode == 'file' and self.model_file) or (
+            self.model_file_mode == 'template' and self.model_file_template
+        )
+
     def fill_form(self, form, formdata, user, **kwargs):
-        if self.method != 'interactive' or not self.model_file:
+        if self.method != 'interactive' or not self.has_configured_model_file():
             return
         label = self.label
         if not label:
@@ -317,7 +310,7 @@ class ExportToModel(WorkflowStatusItem):
     def submit_form(self, form, formdata, user, evo):
         if self.method != 'interactive':
             return
-        if not self.model_file:
+        if not self.has_configured_model_file():
             return
         if form.get_submit() == 'button%s' % self.id:
             if not evo.comment:
@@ -387,7 +380,7 @@ class ExportToModel(WorkflowStatusItem):
         upload.fp.seek(0)
 
     def get_parameters(self):
-        parameters = ('model_file',)
+        parameters = ('model_file_mode', 'model_file', 'model_file_template')
         if transform_to_pdf is not None:
             parameters += ('convert_to_pdf',)
         parameters += ('varname', 'backoffice_filefield_id', 'attach_to_history')
@@ -402,6 +395,10 @@ class ExportToModel(WorkflowStatusItem):
             parameters.remove('by')
             parameters.remove('label')
             parameters.remove('backoffice_info_text')
+        if self.model_file_mode == 'file':
+            parameters.remove('model_file_template')
+        elif self.model_file_mode == 'template':
+            parameters.remove('model_file')
         return parameters
 
     def add_parameters_widgets(self, form, parameters, prefix='', formdef=None, **kwargs):
@@ -410,46 +407,66 @@ class ExportToModel(WorkflowStatusItem):
         methods = collections.OrderedDict(
             [('interactive', _('Interactive (button)')), ('non-interactive', _('Non interactive'))]
         )
-        if 'model_file' in parameters:
-            ids = (self.get_workflow().id, self.parent.id, self.id)
-            if formdef:
-                hint = htmltext('%s: <ul class="varnames">') % _('Available variables')
-                varnames = get_varnames(formdef.fields)
-                for pair in varnames:
-                    hint += (
-                        htmltext('<li><tt class="varname">{{ %s }}</tt> <label>%s</label></span></li>') % pair
-                    )
-                hint += htmltext('</ul>')
-                ids = (formdef.id,) + ids
-                filename = 'export_to_model-%s-%s-%s-%s.upload' % ids
-            else:
-                hint = _(
+        if 'model_file_mode' in parameters:
+            form.add(
+                HtmlWidget,
+                name='note',
+                title=htmltext('<div class="infonotice">%s</div>')
+                % _(
                     'You can use variables in your model using '
                     'the {{variable}} syntax, available variables '
                     'depends on the form.'
-                )
-                filename = 'export_to_model-%s-%s-%s.upload' % ids
+                ),
+            )
+            form.add(
+                RadiobuttonsWidget,
+                '%smodel_file_mode' % prefix,
+                title=_('Model'),
+                options=[('file', _('File'), 'file'), ('template', _('Template'), 'template')],
+                value=self.model_file_mode,
+                default_value=self.__class__.model_file_mode,
+                attrs={'data-dynamic-display-parent': 'true'},
+                extra_css_class='widget-inline-radio',
+            )
+        if 'model_file' in parameters:
+            ids = (self.get_workflow().id, self.parent.id, self.id)
+            filename = 'export_to_model-%s-%s-%s.upload' % ids
             widget_name = '%smodel_file' % prefix
             if formdef and formdef.workflow_options and formdef.workflow_options.get(widget_name) is not None:
                 value = formdef.workflow_options.get(widget_name)
             else:
                 value = self.model_file
             if value:
-                hint_prefix = htmltext('<div>%s: <a href="?file=%s">%s</a></div>') % (
+                hint = htmltext('<div>%s: <a href="?file=%s">%s</a></div>') % (
                     _('Current value'),
                     widget_name,
                     value.base_filename,
                 )
-                hint = hint_prefix + force_str(hint)
+            else:
+                hint = None
             form.add(
                 ModelFileWidget,
                 widget_name,
                 directory='models',
                 filename=filename,
-                title=_('Model'),
                 hint=hint,
                 validation=self.model_file_content_validation,
                 value=value,
+                attrs={
+                    'data-dynamic-display-child-of': '%smodel_file_mode' % prefix,
+                    'data-dynamic-display-value': 'file',
+                },
+            )
+        if 'model_file_template' in parameters:
+            form.add(
+                ComputedExpressionWidget,
+                name='%smodel_file_template' % prefix,
+                title=_('Template to obtain model file'),
+                value=self.model_file_template,
+                attrs={
+                    'data-dynamic-display-child-of': '%smodel_file_mode' % prefix,
+                    'data-dynamic-display-value': 'template',
+                },
             )
         if 'convert_to_pdf' in parameters:
             form.add(
@@ -563,12 +580,12 @@ class ExportToModel(WorkflowStatusItem):
             self.model_file.base_filename,
         )
 
-    def get_filename(self):
+    def get_filename(self, model_file):
         filename = None
         if self.filename:
             filename = self.compute(self.filename)
         if not filename:
-            filename = self.model_file.base_filename
+            filename = model_file.base_filename
         filename = filename.replace('/', '-')
         return filename
 
@@ -577,14 +594,14 @@ class ExportToModel(WorkflowStatusItem):
 
     directory_name = property(get_directory_name)
 
-    def apply_template_to_formdata(self, formdata):
-        kind = self.model_file_validation(self.model_file)
+    def apply_template_to_formdata(self, formdata, model_file):
+        kind = self.model_file_validation(model_file)
         if kind == 'rtf' and not get_publisher().has_site_option('disable-rtf-support'):
-            outstream = self.apply_rtf_template_to_formdata(formdata)
+            outstream = self.apply_rtf_template_to_formdata(formdata, model_file)
         elif kind == 'opendocument':
-            outstream = self.apply_od_template_to_formdata(formdata)
+            outstream = self.apply_od_template_to_formdata(formdata, model_file)
         elif kind == 'xml':
-            outstream = self.apply_text_template_to_formdata(formdata)
+            outstream = self.apply_text_template_to_formdata(formdata, model_file)
         else:
             raise Exception('unsupported model kind %r' % kind)
         if kind == 'xml':
@@ -606,17 +623,17 @@ class ExportToModel(WorkflowStatusItem):
                 return transform_to_pdf(outstream)
         return outstream
 
-    def apply_text_template_to_formdata(self, formdata):
+    def apply_text_template_to_formdata(self, formdata, model_file):
         return io.BytesIO(
             force_bytes(
                 template_on_formdata(
                     formdata,
-                    self.model_file.get_file().read().decode(errors='surrogateescape'),
+                    model_file.get_file().read().decode(errors='surrogateescape'),
                 )
             )
         )
 
-    def apply_rtf_template_to_formdata(self, formdata):
+    def apply_rtf_template_to_formdata(self, formdata, model_file):
         try:
             # force ezt_only=True because an RTF file may contain {{ characters
             # and would be seen as a Django template
@@ -624,7 +641,7 @@ class ExportToModel(WorkflowStatusItem):
                 force_bytes(
                     template_on_formdata(
                         formdata,
-                        force_str(self.model_file.get_file().read()),
+                        force_str(model_file.get_file().read()),
                         ezt_format=ezt.FORMAT_RTF,
                         ezt_only=True,
                     )
@@ -636,7 +653,7 @@ class ExportToModel(WorkflowStatusItem):
             )
             raise TemplatingError(_('Error in template: %s') % str(e))
 
-    def apply_od_template_to_formdata(self, formdata):
+    def apply_od_template_to_formdata(self, formdata, model_file):
         context = get_formdata_template_context(formdata)
 
         def process_styles(root):
@@ -750,7 +767,7 @@ class ExportToModel(WorkflowStatusItem):
                     node.tail = current_tail
 
         outstream = io.BytesIO()
-        transform_opendocument(self.model_file.get_file(), outstream, process_root)
+        transform_opendocument(model_file.get_file(), outstream, process_root)
         outstream.seek(0)
         return outstream
 
@@ -871,10 +888,18 @@ class ExportToModel(WorkflowStatusItem):
             with zipfile.ZipFile(model_file_fp, mode='r') as zin:
                 content = zin.read('content.xml')
                 root = ET.fromstring(content)
+                fields_in_use = [
+                    x.attrib.get('{%s}name' % OO_TEXT_NS)
+                    for x in root.findall('.//{%s}user-field-get' % OO_TEXT_NS)
+                ]
                 for node in root.iter():
                     if node.tag == DRAW_FRAME:
                         yield node.attrib.get(DRAW_NAME)
-                    elif node.tag == USER_FIELD_DECL and STRING_VALUE in node.attrib:
+                    elif (
+                        node.tag == USER_FIELD_DECL
+                        and STRING_VALUE in node.attrib
+                        and node.attrib.get('{%s}name' % OO_TEXT_NS) in fields_in_use
+                    ):
                         yield node.attrib[STRING_VALUE]
                     yield getattr(node, 'text', None)
                     yield getattr(node, 'tail', None)
@@ -884,12 +909,38 @@ class ExportToModel(WorkflowStatusItem):
             return
         self.perform_real(formdata, formdata.evolution[-1])
 
+    def get_model_file(self):
+        if self.model_file_mode == 'file':
+            return self.model_file
+        with get_publisher().complex_data():
+            try:
+                model_file = self.compute(
+                    self.model_file_template, allow_complex=True, record_errors=False, raises=True
+                )
+            except Exception as e:
+                get_publisher().record_error(
+                    _('Failed to evaluate template for action'), exception=e, status_item=self
+                )
+                return None
+            model_file = get_publisher().get_cached_complex_data(model_file)
+        try:
+            model_file = FileField.convert_value_from_anything(model_file)
+        except ValueError:
+            get_publisher().record_error(
+                _('Invalid value obtained for model file (%r)') % model_file, status_item=self
+            )
+            return None
+        return model_file
+
     def perform_real(self, formdata, evo):
-        if not self.model_file:
+        if not self.has_configured_model_file():
             return
-        outstream = self.apply_template_to_formdata(formdata)
-        filename = self.get_filename()
-        content_type = self.model_file.content_type
+        model_file = self.get_model_file()
+        if not model_file:
+            return
+        outstream = self.apply_template_to_formdata(formdata, model_file)
+        filename = self.get_filename(model_file)
+        content_type = model_file.content_type
         if self.convert_to_pdf:
             filename = filename.rsplit('.', 1)[0] + '.pdf'
             content_type = 'application/pdf'

wcs/wf/form.py

@@ -411,9 +411,11 @@ class LazyFormDataWorkflowForms:
         raise AttributeError(varname)
 
     def inspect_keys(self):
+        varnames = set()
         for part in self._formdata.iter_evolution_parts(WorkflowFormEvolutionPart):
             if part.varname and part.data:
-                yield part.varname
+                varnames.add(part.varname)
+        yield from varnames
 
 
 class LazyFormDataWorkflowFormsItems:

wcs/wf/jump.py

@@ -379,9 +379,7 @@ def get_min_jumps_delay(jump_actions):
             break
         delay = min(delay, int(jump_action.timeout))
     # limit delay to minimal delay
-    if delay < JUMP_TIMEOUT_INTERVAL * 60:
-        delay = JUMP_TIMEOUT_INTERVAL * 60
-
+    delay = max(delay, JUMP_TIMEOUT_INTERVAL * 60)
     return delay
 
 
@@ -462,9 +460,11 @@ class LazyFormDataWorkflowTriggers:
         raise AttributeError(trigger_name)
 
     def inspect_keys(self):
+        varnames = set()
         for part in self._formdata.iter_evolution_parts(WorkflowTriggeredEvolutionPart):
             if part.trigger_name:
-                yield part.trigger_name_key
+                varnames.add(part.trigger_name_key)
+        yield from varnames
 
 
 class LazyFormDataWorkflowTriggersItems:

wcs/wf/sendmail.py

@@ -457,9 +457,11 @@ class LazyFormDataEmailsBase:
         raise AttributeError(varname)
 
     def inspect_keys(self):
+        varnames = set()
         for part in self._formdata.iter_evolution_parts(EmailEvolutionPart):
             if part.varname:
-                yield part.varname
+                varnames.add(part.varname)
+        yield from varnames
 
 
 class LazyFormDataEmails:

wcs/wf/wscall.py

@@ -185,9 +185,9 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
             'action_on_5xx',
             'action_on_bad_data',
             'action_on_network_errors',
-            'notify_on_errors',
-            'record_on_errors',
             'record_errors',
+            'record_on_errors',
+            'notify_on_errors',
             'condition',
             'set_marker_on_status',
         )
@@ -258,7 +258,12 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
             form.add(
                 CheckboxWidget,
                 '%spost' % prefix,
-                title=_('Post formdata'),
+                title=_('Post complete card/form data'),
+                hint=_(
+                    'Warning: this option sends the full content of the card/form, '
+                    'with additional POST data in an additional "extra" key. It is often '
+                    'not necessary.'
+                ),
                 value=self.post,
                 attrs={
                     'data-dynamic-display-child-of': '%smethod' % prefix,
@@ -271,6 +276,7 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
                         ]
                     ),
                 },
+                advanced=True,
             )
         if 'post_data' in parameters:
             form.add(
@@ -372,11 +378,12 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
                 default_value=getattr(self.__class__, attribute),
             )
 
-        if 'notify_on_errors' in parameters:
+        if 'notify_on_errors' in parameters and get_publisher().logger.error_email:
             form.add(
                 CheckboxWidget,
                 '%snotify_on_errors' % prefix,
-                title=_('Notify on errors'),
+                title=_('Notify errors by email'),
+                hint=_('Error traces will be sent to %s') % get_publisher().logger.error_email,
                 value=self.notify_on_errors,
                 tab=('error', _('Error Handling')),
             )
@@ -385,7 +392,7 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
             form.add(
                 CheckboxWidget,
                 '%srecord_on_errors' % prefix,
-                title=_('Record on errors'),
+                title=_('Record errors in the central error screen, for management by administrators'),
                 value=self.record_on_errors,
                 tab=('error', _('Error Handling')),
                 default_value=self.__class__.record_on_errors,
@@ -395,7 +402,7 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
             form.add(
                 CheckboxWidget,
                 '%srecord_errors' % prefix,
-                title=_('Record errors in the log'),
+                title=_('Record errors in card/form history log, for agents'),
                 value=self.record_errors,
                 tab=('error', _('Error Handling')),
             )
@@ -669,6 +676,9 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
             else:
                 return _('Error calling webservice')
 
+    def get_workflow_test_action(self, *args, **kwargs):
+        return self
+
     def _kv_data_export_to_xml(self, xml_item, include_id, attribute):
         assert attribute
         if not getattr(self, attribute):

wcs/workflow_tests.py

@@ -19,10 +19,10 @@ import uuid
 
 from wcs import wf
 from wcs.qommon import _
-from wcs.qommon.form import SingleSelectWidget, StringWidget, WidgetList
+from wcs.qommon.form import EmailWidget, IntWidget, SingleSelectWidget, StringWidget, WidgetList
 from wcs.qommon.humantime import humanduration2seconds, seconds2humanduration, timewords
 from wcs.qommon.xml_storage import XmlStorableObject
-from wcs.testdef import TestError
+from wcs.testdef import TestError, WebserviceResponse
 from wcs.wf.backoffice_fields import SetBackofficeFieldRowWidget, SetBackofficeFieldsTableWidget
 from wcs.wf.profile import FieldNode
 
@@ -32,7 +32,12 @@ class WorkflowTestError(TestError):
 
 
 def get_test_action_options():
-    return [(x.key, x.label, x.key) for x in WorkflowTestAction.__subclasses__()]
+    actions = sorted(WorkflowTestAction.__subclasses__(), key=lambda x: x.label)
+
+    assertion_options = [(x.key, x.label, x.key) for x in actions if x.is_assertion]
+    other_options = [(x.key, x.label, x.key) for x in actions if not x.is_assertion]
+
+    return assertion_options + [('', '—', '')] + other_options
 
 
 def get_test_action_class_by_type(action_type):
@@ -59,21 +64,23 @@ class WorkflowTests(XmlStorableObject):
         self.actions = []
 
     def run(self, formdata, agent_user):
-        # mock methods so nothing is stored
-        formdata.record_workflow_event = lambda *args, **kwargs: None
-        formdata.record_workflow_action = lambda *args, **kwargs: None
-        formdata.store = lambda *args, **kwargs: None
+        self.mock_formdata_methods(formdata)
 
         # mark formdata as running workflow tests
         formdata.workflow_test = True
 
         formdata.frozen_receipt_time = formdata.receipt_time
         formdata.sent_emails = []
+        formdata.used_webservice_responses = self.testdef.used_webservice_responses = []
 
         formdata.perform_workflow()
         for action in self.actions:
             status = formdata.get_status()
 
+            if not action.is_assertion:
+                formdata.sent_emails.clear()
+                formdata.used_webservice_responses.clear()
+
             try:
                 action.perform(formdata, agent_user)
             except WorkflowTestError as e:
@@ -81,6 +88,20 @@ class WorkflowTests(XmlStorableObject):
                 e.details.append(_('Form status when error occured: %s') % status.name)
                 raise e
 
+    def mock_formdata_methods(self, formdata):
+        from wcs.workflow_traces import WorkflowTrace
+
+        def record_workflow_event(event, **kwargs):
+            formdata.workflow_traces.append(WorkflowTrace(formdata=formdata, event=event, event_args=kwargs))
+
+        def record_workflow_action(action):
+            formdata.workflow_traces.append(WorkflowTrace(formdata=formdata, action=action))
+
+        formdata.record_workflow_event = record_workflow_event
+        formdata.record_workflow_action = record_workflow_action
+
+        formdata.store = lambda *args, **kwargs: None
+
     def get_new_action_id(self):
         if not self.actions:
             return '1'
@@ -90,6 +111,11 @@ class WorkflowTests(XmlStorableObject):
     def add_action(self, action_class):
         self.actions.append(action_class(id=self.get_new_action_id()))
 
+    def store(self, *args, **kwargs):
+        super().store(*args, **kwargs)
+        for action in self.actions:
+            action.parent = self
+
     def export_actions_to_xml(self, element, attribute_name, **kwargs):
         for action in self.actions:
             element.append(action.export_to_xml())
@@ -104,7 +130,9 @@ class WorkflowTests(XmlStorableObject):
             except KeyError:
                 continue
 
-            actions.append(klass.import_from_xml_tree(sub))
+            action = klass.import_from_xml_tree(sub)
+            action.parent = self
+            actions.append(action)
 
         return actions
 
@@ -115,6 +143,7 @@ class WorkflowTestAction(XmlStorableObject):
     uuid = None
 
     optional_fields = []
+    is_assertion = True
 
     XML_NODES = [
         ('id', 'str'),
@@ -136,24 +165,27 @@ class WorkflowTestAction(XmlStorableObject):
     def render_as_line(self):
         for field, dummy in self.XML_NODES:
             if field not in self.optional_fields and not getattr(self, field):
-                return 'not configured'
+                return _('not configured')
 
         return self.details_label
 
 
 class ButtonClick(WorkflowTestAction):
     label = _('Simulate click on action button')
+    empty_form_error = _('Workflow has no action that displays a button.')
 
     key = 'button-click'
     button_name = None
 
+    is_assertion = False
+
     XML_NODES = WorkflowTestAction.XML_NODES + [
         ('button_name', 'str'),
     ]
 
     @property
     def details_label(self):
-        return 'Click on "%s"' % self.button_name
+        return _('Click on "%s"') % self.button_name
 
     def perform(self, formdata, user):
         status = formdata.get_status()
@@ -172,6 +204,9 @@ class ButtonClick(WorkflowTestAction):
             if isinstance(item, wf.choice.ChoiceWorkflowStatusItem) and item.status:
                 possible_button_names.add(item.label)
 
+        if not possible_button_names:
+            return
+
         possible_button_names = sorted(possible_button_names)
 
         value = self.button_name
@@ -201,7 +236,7 @@ class AssertStatus(WorkflowTestAction):
 
     @property
     def details_label(self):
-        return 'Status is "%s"' % self.status_name
+        return _('Status is "%s"') % self.status_name
 
     def perform(self, formdata, user):
         status = formdata.get_status()
@@ -233,24 +268,35 @@ class AssertEmail(WorkflowTestAction):
     label = _('Assert email is sent')
 
     key = 'assert-email'
+    addresses = None
     subject_strings = None
     body_strings = None
 
-    optional_fields = ['subject_strings', 'body_strings']
+    optional_fields = ['addresses', 'subject_strings', 'body_strings']
 
     XML_NODES = WorkflowTestAction.XML_NODES + [
+        ('addresses', 'str_list'),
         ('subject_strings', 'str_list'),
         ('body_strings', 'str_list'),
     ]
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self.addresses = self.addresses or []
         self.subject_strings = self.subject_strings or []
         self.body_strings = self.body_strings or []
 
     @property
     def details_label(self):
-        return ''
+        if not self.addresses:
+            return ''
+
+        label = _('Email to "%s"') % self.addresses[0]
+
+        if len(self.addresses) > 1:
+            label = '%s (+%s)' % (label, len(self.addresses) - 1)
+
+        return label
 
     def perform(self, formdata, user):
         try:
@@ -258,6 +304,11 @@ class AssertEmail(WorkflowTestAction):
         except IndexError:
             raise WorkflowTestError(_('No email was sent.'))
 
+        for address in self.addresses:
+            details = [_('Email addresses: %s') % ', '.join(email.email_msg.to)]
+            if address not in email.email_msg.to:
+                raise WorkflowTestError(_('Email was not sent to address "%s".') % address, details=details)
+
         for subject in self.subject_strings:
             details = [_('Email subject: %s') % email.email_msg.subject]
             if subject not in email.email_msg.subject:
@@ -269,6 +320,15 @@ class AssertEmail(WorkflowTestAction):
                 raise WorkflowTestError(_('Email body does not contain "%s".') % body, details=details)
 
     def fill_admin_form(self, form, formdef):
+        form.add(
+            WidgetList,
+            'addresses',
+            element_type=EmailWidget,
+            title=_('Email addresses'),
+            value=self.addresses,
+            add_element_label=_('Add address'),
+            element_kwargs={'render_br': False, 'size': 50},
+        )
         form.add(
             WidgetList,
             'subject_strings',
@@ -295,6 +355,8 @@ class SkipTime(WorkflowTestAction):
     key = 'skip-time'
     seconds = None
 
+    is_assertion = False
+
     XML_NODES = WorkflowTestAction.XML_NODES + [
         ('seconds', 'int'),
     ]
@@ -319,7 +381,7 @@ class SkipTime(WorkflowTestAction):
             if hasattr(item, 'has_valid_timeout') and item.has_valid_timeout():
                 jump_actions.append(item)
 
-        delay = wf.jump.get_min_jumps_delay(status.items)
+        delay = wf.jump.get_min_jumps_delay(jump_actions)
 
         if formdata.last_update_time > formdata.frozen_receipt_time - datetime.timedelta(seconds=delay):
             return
@@ -421,3 +483,70 @@ class AssertBackofficeFieldValues(WorkflowTestAction):
             fields.append(field_node.as_dict())
 
         return fields
+
+
+class AssertWebserviceCall(WorkflowTestAction):
+    label = _('Assert webservice call')
+
+    key = 'assert-webservice-call'
+    webservice_response_id = None
+    call_count = 1
+
+    XML_NODES = WorkflowTestAction.XML_NODES + [
+        ('webservice_response_id', 'str'),
+        ('call_count', 'int'),
+    ]
+
+    @property
+    def details_label(self):
+        webservice_responses = [
+            x for x in self.parent.testdef.get_webservice_responses() if x.id == self.webservice_response_id
+        ]
+        if webservice_responses:
+            return webservice_responses[0].name
+        else:
+            return _('Broken, missing webservice response')
+
+    @property
+    def empty_form_error(self):
+        r = '<p>%s</p>' % _(
+            'In order to assert a webservice is called, you must define corresponding webservice response.'
+        )
+        r += '<p><a href="%swebservice-responses/">%s</a><p>' % (
+            self.parent.testdef.get_admin_url(),
+            _('Add webservice response'),
+        )
+        return r
+
+    def perform(self, formdata, user):
+        call_count = 0
+        for response in formdata.used_webservice_responses.copy():
+            if response.id == self.webservice_response_id:
+                formdata.used_webservice_responses.remove(response)
+                call_count += 1
+
+        if call_count != self.call_count:
+            response = WebserviceResponse.get(self.webservice_response_id)
+            raise WorkflowTestError(
+                _('Webservice response %(name)s was used %(count)s times (instead of %(expected_count)s).')
+                % {'name': response.name, 'count': call_count, 'expected_count': self.call_count}
+            )
+
+    def fill_admin_form(self, form, formdef):
+        webservice_response_options = [
+            (response.id, response.name, response.id)
+            for response in self.parent.testdef.get_webservice_responses()
+        ]
+
+        if not webservice_response_options:
+            return
+
+        form.add(
+            SingleSelectWidget,
+            'webservice_response_id',
+            title=_('Webservice response'),
+            options=webservice_response_options,
+            required=True,
+            value=self.webservice_response_id,
+        )
+        form.add(IntWidget, 'call_count', title=_('Call count'), required=True, value=self.call_count)

wcs/workflow_traces.py

@@ -17,7 +17,7 @@
 from quixote.html import TemplateIO, htmltext
 
 from wcs import sql
-from wcs.qommon import _
+from wcs.qommon import _, misc
 
 
 class WorkflowTrace(sql.WorkflowTrace):
@@ -51,6 +51,7 @@ class WorkflowTrace(sql.WorkflowTrace):
             'button': _('Action button'),
             'continuation': _('Continuation'),
             'csv-import-created': _('Created (by CSV import)'),
+            'csv-import-updated': _('Updated (by CSV import)'),
             'edit-action': _('Actions after edit action'),
             'email-button': _('Email action button'),
             'frontoffice-created': _('Created (frontoffice submission)'),
@@ -283,3 +284,19 @@ class WorkflowTrace(sql.WorkflowTrace):
             status_admin_base_url,
             status_label,
         )
+
+    def get_json_export_dict(self):
+        return {field: getattr(self, field) for field, _ in self._table_static_fields}
+
+    @classmethod
+    def import_from_json_dict(cls, data):
+        workflow_trace = cls.__new__(cls)
+
+        for field, kind in cls._table_static_fields:
+            value = data.get(field)
+            if value and kind == 'timestamptz':
+                value = misc.get_as_datetime(value)
+
+            setattr(workflow_trace, field, value)
+
+        return workflow_trace

wcs/workflows.py

@@ -2664,6 +2664,7 @@ class WorkflowStatus(SerieOfActionsMixin):
         if form is None:
             form = Form(enctype='multipart/form-data', use_tokens=False)
             form.attrs['id'] = 'wf-actions'
+            form.add_hidden('_ts', str(filled.last_update_time.timestamp()))
 
         for action in filled.formdef.workflow.get_global_actions_for_user(filled, user):
             form.add_submit('button-action-%s' % action.id, get_publisher().translate(action.name))
@@ -2693,6 +2694,8 @@ class WorkflowStatus(SerieOfActionsMixin):
 
     def handle_form(self, form, filled, user, check_replay=True):
         # check for global actions
+        if check_replay and form.get('_ts') != str(filled.last_update_time.timestamp()):
+            raise ReplayException()
         for action in filled.formdef.workflow.get_global_actions_for_user(filled, user):
             if 'button-action-%s' % action.id in get_request().form:
                 if action.is_interactive():
@@ -3173,7 +3176,8 @@ class WorkflowStatusItem(XmlSerialisable):
             if not widget:
                 continue
             r += htmltext('<li class="parameter-%s">' % parameter)
-            r += htmltext('<span class="parameter">%s</span> ') % _('%s:') % widget.get_title()
+            if widget.get_title():
+                r += htmltext('<span class="parameter">%s</span> ') % _('%s:') % widget.get_title()
             r += self.get_parameter_view_value(widget, parameter)
             r += htmltext('</li>')
         r += htmltext('</ul>')

wcsctl.py

@@ -1,11 +0,0 @@
-#! /usr/bin/env python3
-
-import os
-import sys
-
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'wcs.settings')
-
-import wcs.qommon.ctl
-
-ctl = wcs.qommon.ctl.Ctl(cmd_prefixes=['wcs.ctl'])
-ctl.run(sys.argv[1:])