pre-commit

debian/control

@@ -43,7 +43,8 @@ Depends: graphviz,
          uwsgi-plugin-python3,
          ${misc:Depends},
          ${python3:Depends},
-Recommends: libreoffice-writer-nogui | libreoffice-writer,
+Recommends: graphicsmagick,
+            libreoffice-writer-nogui | libreoffice-writer,
             poppler-utils,
             python3-docutils,
             python3-langdetect,

tests/admin_pages/test_deprecations.py

@@ -183,6 +183,9 @@ def test_deprecations(pub):
     data_source = NamedDataSource(name='ds_jsonp')
     data_source.data_source = {'type': 'jsonp', 'value': 'xxx'}
     data_source.store()
+    data_source = NamedDataSource(name='ds_csv')
+    data_source.data_source = {'type': 'json', 'value': 'http://example.net/csvdatasource/plop/test'}
+    data_source.store()
 
     NamedWsCall.wipe()
     wscall = NamedWsCall()
@@ -190,6 +193,16 @@ def test_deprecations(pub):
     wscall.request = {'url': 'http://example.net', 'qs_data': {'a': '=1+2'}}
     wscall.store()
 
+    wscall = NamedWsCall()
+    wscall.name = 'Hello CSV'
+    wscall.request = {'url': 'http://example.net/csvdatasource/plop/test'}
+    wscall.store()
+
+    wscall = NamedWsCall()
+    wscall.name = 'Hello json data store'
+    wscall.request = {'url': 'http://example.net/jsondatastore/plop'}
+    wscall.store()
+
     MailTemplate.wipe()
     mail_template1 = MailTemplate()
     mail_template1.name = 'Hello1'
@@ -258,6 +271,13 @@ def test_deprecations(pub):
     assert [x.text for x in resp.pyquery('.section--actions li a')] == [
         'test / Daily Summary Email',
     ]
+    assert [x.text for x in resp.pyquery('.section--csv-connector li a')] == [
+        'Data source "ds_csv"',
+        'Webservice "Hello CSV"',
+    ]
+    assert [x.text for x in resp.pyquery('.section--json-data-store li a')] == [
+        'Webservice "Hello json data store"',
+    ]
     # check all links are ok
     for link in resp.pyquery('.section li a'):
         resp.click(href=link.attrib['href'], index=0)

tests/admin_pages/test_settings.py

@@ -1084,17 +1084,31 @@ def test_i18n(pub):
 def test_submission_channels(pub):
     create_superuser(pub)
     app = login(get_app(pub))
-    resp = app.get('/backoffice/settings/submission-channels')
+    resp = app.get('/backoffice/settings/backoffice-submission')
     resp.form['include-in-global-listing'].checked = True
     resp = resp.form.submit('submit')
 
     pub.reload_cfg()
     assert pub.cfg['submission-channels']['include-in-global-listing'] is True
 
-    resp = app.get('/backoffice/settings/submission-channels')
+    resp = app.get('/backoffice/settings/backoffice-submission')
     assert resp.form['include-in-global-listing'].checked
 
 
+def test_backoffice_submission(pub):
+    create_superuser(pub)
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/settings/backoffice-submission')
+    resp.form['redirect'] = 'https://example.net'
+    resp = resp.form.submit('submit')
+
+    pub.reload_cfg()
+    assert pub.cfg['backoffice-submission']['redirect'] == 'https://example.net'
+
+    resp = app.get('/backoffice/settings/backoffice-submission')
+    assert resp.form['redirect'].value == 'https://example.net'
+
+
 def test_hobo_locked_settings(pub):
     create_superuser(pub)
     app = login(get_app(pub))

tests/admin_pages/test_tests.py

@@ -950,7 +950,8 @@ def test_tests_result_sent_requests(pub, http_requests):
     assert 'Sent requests:' in resp.text
     assert 'POST http://remote.example.net/json' in resp.text
     assert 'Request was blocked since it is not a GET request.' in resp.text
-    assert 'Recorded errors:' not in resp.text
+    assert 'Recorded errors:' in resp.text
+    assert 'error in HTTP request to remote.example.net (method must be GET)' in resp.text
 
     resp = resp.click('You can create corresponding webservice response here.')
     assert 'Webservice responses' in resp.text
@@ -1023,6 +1024,11 @@ def test_tests_result_inspect(pub):
     jump.status = new_status.id
     jump.by = [role.id]
 
+    wscall = new_status.add_action('webservice_call')
+    wscall.url = 'http://example.com/json'
+    wscall.varname = 'test_webservice'
+    wscall.qs_data = {'a': 'b'}
+
     workflow.store()
 
     formdef = FormDef()
@@ -1045,6 +1051,13 @@ def test_tests_result_inspect(pub):
     ]
     testdef.store()
 
+    response = WebserviceResponse()
+    response.testdef_id = testdef.id
+    response.name = 'Fake response'
+    response.url = 'http://example.com/json'
+    response.payload = '{"foo": "bar"}'
+    response.store()
+
     app = login(get_app(pub))
     resp = app.get('/backoffice/forms/1/tests/results/run')
     result_url = resp.location
@@ -1053,12 +1066,15 @@ def test_tests_result_inspect(pub):
 
     assert 'form_var_text' in resp.text
     assert 'form_var_text_bo' in resp.text
+    assert 'form_workflow_data_test_webservice_response_foo' in resp.text
 
     assert [x.text_content() for x in resp.pyquery('div#inspect-timeline a')] == [
         'New status',
         'Backoffice Data',
+        'Webservice',
         'Action button - Manual Jump Loop on status',
         'Backoffice Data',
+        'Webservice',
     ]
 
     resp.form['django-condition'] = 'form_var_text == "hello"'

tests/admin_pages/test_workflow_tests.py

@@ -144,6 +144,15 @@ def test_workflow_tests_edit_actions(pub):
     assert 'There are no workflow test actions yet.' in resp.text
     assert len(resp.pyquery('.biglist li')) == 0
 
+    option_labels = [x[2] for x in resp.form['type'].options]
+    assert (
+        option_labels.index('Assert email is sent')
+        < option_labels.index('Assert form status')
+        < option_labels.index('—')
+        < option_labels.index('Move forward in time')
+        < option_labels.index('Simulate click on action button')
+    )
+
     # add workflow test action through sidebar form
     resp.form['type'] = 'button-click'
     resp = resp.form.submit().follow()
@@ -189,18 +198,6 @@ def test_workflow_tests_action_button_click(pub):
 
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')
-
-    jump = new_status.add_action('choice')
-    jump.label = 'Button 1'
-    jump.status = new_status.id
-
-    jump = new_status.add_action('choice')
-    jump.label = 'Button 2'
-    jump.status = new_status.id
-
-    jump = new_status.add_action('choice')
-    jump.label = 'Button no target status'
-
     workflow.store()
 
     formdef = FormDef()
@@ -219,7 +216,22 @@ def test_workflow_tests_action_button_click(pub):
 
     app = login(get_app(pub))
     resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
+    assert 'Workflow has no action that displays a button.' in resp.text
 
+    jump = new_status.add_action('choice')
+    jump.label = 'Button 1'
+    jump.status = new_status.id
+
+    jump = new_status.add_action('choice')
+    jump.label = 'Button 2'
+    jump.status = new_status.id
+
+    jump = new_status.add_action('choice')
+    jump.label = 'Button no target status'
+
+    workflow.store()
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
     assert resp.form['button_name'].options == [
         ('Button 1', False, 'Button 1'),
         ('Button 2', False, 'Button 2'),
@@ -388,6 +400,14 @@ def test_workflow_tests_action_assert_webservice_call(pub):
     ]
     testdef.store()
 
+    app = login(get_app(pub))
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
+    assert 'you must define corresponding webservice response' in resp.text
+
+    resp = resp.click('Add webservice response')
+    assert 'There are no webservice responses yet.' in resp.text
+
     response = WebserviceResponse()
     response.testdef_id = testdef.id
     response.name = 'Fake response'
@@ -403,8 +423,6 @@ def test_workflow_tests_action_assert_webservice_call(pub):
     response3.name = 'Other response'
     response3.store()
 
-    app = login(get_app(pub))
-
     resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
     assert resp.form['webservice_response_id'].options == [
         ('1', False, 'Fake response'),

tests/backoffice_pages/test_submission.py

@@ -161,6 +161,52 @@ def test_backoffice_submission(pub):
     assert resp.location == 'http://www.example.org/'
 
 
+def test_backoffice_submission_menu_entry(pub):
+    user = create_user(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = []
+    formdef.workflow_roles = {'_receiver': 1}
+    formdef.backoffice_submission_roles = user.roles[:]
+    formdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/forms')
+    assert resp.pyquery('#sidepage-menu .icon-submission')
+
+    pub.cfg['backoffice-submission'] = {}
+    pub.cfg['backoffice-submission']['sidebar_menu_entry'] = 'visible'
+    pub.write_cfg()
+    resp = app.get('/backoffice/submission/', status=200)
+    assert resp.pyquery('#sidepage-menu .icon-submission')
+
+    pub.cfg['backoffice-submission']['sidebar_menu_entry'] = 'redirect'
+    pub.cfg['backoffice-submission']['redirect'] = 'https://example.net/'
+    pub.write_cfg()
+    resp = app.get('/backoffice/management/forms')
+    assert resp.pyquery('#sidepage-menu .icon-submission')
+    resp = app.get('/backoffice/submission/', status=302)
+    assert resp.location == 'https://example.net/'
+
+    pub.cfg['backoffice-submission']['sidebar_menu_entry'] = 'hidden'
+    pub.write_cfg()
+    resp = app.get('/backoffice/management/forms')
+    assert not resp.pyquery('#sidepage-menu .icon-submission')
+    resp = app.get('/backoffice/submission/', status=302)
+    assert resp.location == 'https://example.net/'
+
+    pub.cfg['backoffice-submission'][
+        'redirect'
+    ] = '{% if session_user_email == "admin@localhost" %}https://example.net/{% endif %}'
+    pub.write_cfg()
+    app.get('/backoffice/submission/', status=302)  # redirection
+    user.email = 'admin2@localhost'
+    user.store()
+    app.get('/backoffice/submission/', status=200)  # native screen
+
+
 def test_backoffice_submission_with_tracking_code(pub):
     user = create_user(pub)
 
@@ -389,8 +435,8 @@ def test_backoffice_parallel_submission(pub, autosave):
     formdata.store()
 
     app = login(get_app(pub))
-    resp = app.get('/backoffice/submission/')
-    assert 'Submission to complete' in resp.text
+    resp = app.get('/backoffice/submission/pending')
+    assert resp.pyquery('tbody tr')
     resp1 = app.get('/backoffice/submission/form-title/%s/' % formdata.id)
     resp1 = resp1.follow()
     resp2 = app.get('/backoffice/submission/form-title/%s/' % formdata.id)
@@ -600,13 +646,16 @@ def test_backoffice_submission_drafts(pub):
     tracking_code = data_class.select()[0].tracking_code
 
     # stop here, go back to index
+    pub.cfg['submission-channels'] = {'include-in-global-listing': True}
+    pub.write_cfg()
     resp = app.get('/backoffice/submission/')
-    assert '%s/%s' % (formdef.url_name, formdata_no) in resp.text
-    assert '>#%s' % formdata_no in resp.text
+    resp = resp.click('Pending submissions')
+    assert resp.pyquery('tbody tr a').text() == formdata.get_display_name()
+    assert resp.pyquery('tbody tr a')[0].attrib['href'] == f'{formdef.url_name}/{formdata_no}/'
     formdata.submission_channel = 'mail'
     formdata.store()
-    resp = app.get('/backoffice/submission/')
-    assert '>Mail #%s' % formdata_no in resp.text
+    resp = app.get('/backoffice/submission/pending')
+    assert resp.pyquery('tbody td:nth-child(1)').text() == 'Mail'
 
     # check it can also be accessed using its final URL
     resp2 = app.get('/backoffice/management/%s/%s/' % (formdef.url_name, formdata_no))
@@ -628,6 +677,33 @@ def test_backoffice_submission_drafts(pub):
     assert resp.location == 'http://example.net/backoffice/management/form-title/%s/' % formdata_no
 
 
+def test_backoffice_draft_with_digest(pub):
+    user = create_user(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [
+        fields.StringField(id='1', label='1st field', varname='foo'),
+    ]
+    formdef.backoffice_submission_roles = user.roles[:]
+    formdef.digest_templates = {'default': 'digest: {{ form_var_foo }}'}
+    formdef.workflow_roles = {'_receiver': 1}
+    formdef.store()
+
+    formdef.data_class().wipe()
+    formdata = formdef.data_class()()
+    formdata.data = {'1': 'bar'}
+    formdata.status = 'draft'
+    formdata.backoffice_submission = True
+    formdata.submission_agent_id = str(user.id)
+    formdata.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/submission/pending')
+    assert resp.pyquery('tbody td:nth-child(1)').text() == 'form title #1-1 digest: bar'
+
+
 def test_backoffice_submission_remove_drafts(pub):
     user = create_user(pub)
 
@@ -661,7 +737,7 @@ def test_backoffice_submission_remove_drafts(pub):
     formdata_no = formdata.id
 
     # stop here, go back to the index
-    resp = app.get('/backoffice/submission/')
+    resp = app.get('/backoffice/submission/pending')
     resp = resp.click('#%s' % formdata_no)
     resp = resp.follow()
 
@@ -673,7 +749,7 @@ def test_backoffice_submission_remove_drafts(pub):
     assert pub.tracking_code_class().count() == 1
 
     # and this time for real
-    resp = app.get('/backoffice/submission/')
+    resp = app.get('/backoffice/submission/pending')
     resp = resp.click('#%s' % formdata_no)
     resp = resp.follow()
     resp = resp.click('Discard this form')
@@ -951,48 +1027,6 @@ def test_backoffice_submission_conditional_jump_based_on_bo_field(pub):
     assert formdef.data_class().select()[0].status == 'wf-st1'
 
 
-def test_backoffice_submission_sections(pub):
-    user = create_user(pub)
-
-    FormDef.wipe()
-    formdef = FormDef()
-    formdef.name = 'form title'
-    formdef.fields = [
-        fields.StringField(
-            id='1', label='1st field', display_locations=['validation', 'summary', 'listings']
-        ),
-    ]
-    formdef.backoffice_submission_roles = user.roles[:]
-    formdef.store()
-
-    data_class = formdef.data_class()
-    data_class.wipe()
-
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/submission/')
-    assert 'Submission to complete' not in resp.text
-    assert 'Running submission' not in resp.text
-
-    formdata = data_class()
-    formdata.data = {}
-    formdata.status = 'draft'
-    formdata.backoffice_submission = True
-    formdata.receipt_time = make_aware(datetime.datetime(2015, 1, 1))
-    formdata.store()
-
-    resp = app.get('/backoffice/submission/')
-    assert 'Submission to complete' in resp.text
-    assert 'Running submission' not in resp.text
-    assert '>#%s' % formdata.id in resp.text
-
-    formdata.data = {'1': 'xxx'}
-    formdata.store()
-    resp = app.get('/backoffice/submission/')
-    assert 'Submission to complete' not in resp.text
-    assert 'Running submission' in resp.text
-    assert '>#%s' % formdata.id in resp.text
-
-
 def test_backoffice_submission_drafts_order(pub):
     user = create_user(pub)
 
@@ -1011,28 +1045,39 @@ def test_backoffice_submission_drafts_order(pub):
     data_class.wipe()
 
     formdata_ids = []
-    for i in range(10):
+    for i in range(25):
         formdata = data_class()
         formdata.data = {}
         formdata.status = 'draft'
         formdata.backoffice_submission = True
-        formdata.receipt_time = make_aware(datetime.datetime(2023, 11, 20 - i))
+        formdata.receipt_time = make_aware(datetime.datetime(2023, 11, 30 - i))
         formdata.store()
         formdata_ids.append(formdata.id)
 
     app = login(get_app(pub))
-    resp = app.get('/backoffice/submission/')
-    assert [x.attrib['href'] for x in resp.pyquery('.biglist.empty a:not(.fake)')] == [
-        f'form-title/{x}/' for x in reversed(formdata_ids)
+    resp = app.get('/backoffice/submission/pending')
+    assert [x.attrib['data-link'] for x in resp.pyquery('tbody tr')] == [
+        f'form-title/{x}/' for x in formdata_ids[:20]
     ]
 
     formdata.receipt_time = None  # check a missing receipt_time is ok
     formdata.store()
-    resp = app.get('/backoffice/submission/')
-    assert [x.attrib['href'] for x in resp.pyquery('.biglist.empty a:not(.fake)')] == [
-        f'form-title/{x}/' for x in reversed(formdata_ids)
+    new_order = [formdata.id] + [x for x in formdata_ids if x != formdata.id]
+    resp = app.get('/backoffice/submission/pending')
+    assert [x.attrib['data-link'] for x in resp.pyquery('tbody tr')] == [
+        f'form-title/{x}/' for x in new_order[:20]
     ]
-    assert 'unknown date' in resp.pyquery('li.smallitem:first').text()
+
+    resp = resp.click('<!--Next Page-->')
+    assert [x.attrib['data-link'] for x in resp.pyquery('tbody tr')] == [
+        f'form-title/{x}/' for x in new_order[20:]
+    ]
+
+    # check ajax call result
+    resp = app.get('/backoffice/submission/pending?ajax=true')
+    assert 'appbar' not in resp.text
+    assert '<table' in resp.text
+    assert 'page-links' in resp.text
 
 
 def test_backoffice_submission_prefill_user(pub):
@@ -1267,7 +1312,7 @@ def test_backoffice_submission_multiple_page_restore_on_validation(pub):
     assert formdef.data_class().count() == 1
     formdata = formdef.data_class().select()[0]
     # restore draft
-    resp = app.get('/backoffice/submission/')
+    resp = app.get('/backoffice/submission/pending')
     resp = resp.click(href='form-title/%s' % formdata.id)
     resp = resp.follow()
     assert 'Check values then click submit.' in resp.text

tests/form_pages/test_all.py

@@ -6011,3 +6011,65 @@ def test_form_errors_summary(pub):
     resp = resp.forms[0].submit('submit')
     assert 'The following field has an error: testblock' in resp.pyquery('.errornotice').text()
     assert resp.pyquery('.error').text() == 'required field required field '
+
+
+def test_form_submit_no_csrf(pub):
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [fields.StringField(id='0', label='string')]
+    formdef.confirmation = False
+    formdef.store()
+    formdef.data_class().wipe()
+
+    create_user(pub)
+    app = get_app(pub)
+    login(app, username='foo', password='foo')
+    resp = app.get(formdef.get_url())
+    resp.form['f0'] = 'hello'
+    # get expected data
+    form_data = {x: y for x, y in resp.form.submit_fields('submit')}
+    # remove token values
+    form_data['_form_id'] = 'xxx'
+    form_data['_ajax_form_token'] = 'xxx'
+    form_data['magictoken'] = 'xxx'
+    # simulate call from remote/attacker site (form token prevents this)
+    resp = app.post(formdef.get_url(), params=form_data)
+    assert 'The form you have submitted is invalid.' in resp.text
+
+    # with confirmation page
+    formdef.confirmation = True
+    formdef.store()
+    resp = app.get(formdef.get_url())
+    resp.form['f0'] = 'hello'
+    resp = resp.form.submit('submit')
+    # get expected data
+    form_data = {x: y for x, y in resp.form.submit_fields('submit')}
+    # remove token values
+    form_data['_form_id'] = 'xxx'
+    form_data['_ajax_form_token'] = 'xxx'
+    form_data['magictoken'] = 'xxx'
+    # simulate call from remote/attacker site (magictoken prevents this)
+    resp = app.post(formdef.get_url(), params=form_data, status=302)
+    assert resp.location == formdef.get_url()
+
+    # with multiple pages
+    formdef.confirmation = False
+    formdef.fields = [
+        fields.PageField(id='1', label='page1'),
+        fields.PageField(id='2', label='page2'),
+        fields.StringField(id='3', label='string'),
+    ]
+    formdef.store()
+    resp = app.get(formdef.get_url())
+    resp = resp.form.submit('submit')
+    resp.form['f3'] = 'hello'
+    # get expected data
+    form_data = {x: y for x, y in resp.form.submit_fields('submit')}
+    # remove token values
+    form_data['_form_id'] = 'xxx'
+    form_data['_ajax_form_token'] = 'xxx'
+    form_data['magictoken'] = 'xxx'
+
+    # simulate call from remote/attacker site (magictokens prevents this)
+    resp = app.post(formdef.get_url(), params=form_data, status=302)
+    assert resp.location == formdef.get_url()

tests/form_pages/test_file_field.py

@@ -238,6 +238,42 @@ def test_form_file_field_image_submit(pub):
     assert '<img alt="" src="tempfile?' not in resp.text
 
 
+def test_form_file_field_html_submit(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [fields.FileField(id='0', label='file')]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    html_content = b'<html><body>hello</body></html>'
+    upload = Upload('test.html', html_content, 'text/html')
+
+    app = get_app(pub)
+    resp = app.get('/test/')
+    resp.forms[0]['f0$file'] = upload
+    resp = resp.forms[0].submit('submit')
+    assert 'Check values then click submit.' in resp.text
+    tempfile_id = resp.pyquery('.fileinfo .filename a').attr.href.split('=')[1]
+
+    resp_tempfile = app.get('/test/tempfile?t=%s' % tempfile_id)
+    assert resp_tempfile.body == html_content
+
+    resp = resp.form.submit('submit').follow()
+    assert resp.click('test.html').follow().content_type == 'text/html'
+    assert resp.click('test.html').follow().body == html_content
+
+    # check it's also served raw from backoffice
+    user = create_user(pub)
+    user.is_admin = True
+    user.store()
+    app = get_app(pub)
+    login(app, username='foo', password='foo')
+    resp = app.get(formdef.data_class().select()[0].get_backoffice_url())
+    assert resp.click('test.html').follow().content_type == 'text/html'
+    assert resp.click('test.html').follow().body == html_content
+
+
 def test_form_file_field_submit_document_type(pub):
     FormDef.wipe()
     formdef = FormDef()

tests/form_pages/test_map_field.py

@@ -233,6 +233,7 @@ def test_form_map_field_default_position(pub):
     resp.form['f1'] = '169 rue du chateau, paris'
     resp = resp.form.submit('submit')
     assert resp.pyquery('.qommon-map').attr('data-def-lat') == '13'
+    assert resp.pyquery('.qommon-map').attr('data-def-template')
 
     formdef.fields[3].initial_position = 'template'
     formdef.fields[3].position_template = '{{ form_var_address }}'
@@ -244,3 +245,4 @@ def test_form_map_field_default_position(pub):
         rsps.get('https://nominatim.entrouvert.org/search', json=[{'lat': '48.8337085', 'lon': '2.3233693'}])
         resp = resp.form.submit('submit')
         assert resp.pyquery('.qommon-map').attr('data-def-lat') == '48.83370850'
+        assert resp.pyquery('.qommon-map').attr('data-def-template')

tests/test_fields.py

@@ -153,11 +153,11 @@ def test_text(pub):
 
     form = Form(use_tokens=False)
     fields.TextField(display_mode='rich').add_to_form(form)
-    assert 'data-godo-schema="full"' in str(form.render())
+    assert PyQuery(str(form.render()))('godo-editor[schema=full]')
 
     form = Form(use_tokens=False)
     fields.TextField(display_mode='basic-rich').add_to_form(form)
-    assert 'data-godo-schema="basic"' in str(form.render())
+    assert PyQuery(str(form.render()))('godo-editor[schema=basic]')
 
 
 def test_text_anonymise(pub):

tests/test_templates.py

@@ -2,6 +2,8 @@ import datetime
 import html
 import os
 import string
+import subprocess
+from unittest import mock
 
 import pytest
 from django.test import override_settings
@@ -1696,3 +1698,83 @@ def test_details_format(pub):
     pub.loggederror_class.wipe()
     assert tmpl.render(context) == 'String:\n  foo'
     assert pub.loggederror_class.count() == 0
+
+
+@pytest.mark.parametrize('image_format', ['jpeg', 'png', 'pdf'])
+def test_convert_image_format(pub, image_format):
+    with pub.complex_data():
+        img = Template('{{ url|qrcode|convert_image_format:"%s" }}' % image_format).render(
+            {'url': 'http://example.com/', 'allow_complex': True}
+        )
+        assert pub.has_cached_complex_data(img)
+        value = pub.get_cached_complex_data(img)
+        assert value.orig_filename == 'qrcode.%s' % image_format
+        assert value.content_type == {'jpeg': 'image/jpeg', 'png': 'image/png', 'pdf': 'application/pdf'}.get(
+            image_format
+        )
+        with value.get_file_pointer() as fp:
+            if image_format in ('jpeg', 'png'):
+                img = PIL.Image.open(fp)
+                assert img.format == image_format.upper()
+                assert img.size == (330, 330)
+                assert (
+                    zbar_decode_qrcode(img, symbols=[ZBarSymbol.QRCODE])[0].data.decode()
+                    == 'http://example.com/'
+                )
+            else:
+                assert b'%PDF-' in fp.read()[:200]
+
+
+def test_convert_image_format_no_name(pub):
+    with pub.complex_data():
+        img = Template('{{ url|qrcode|rename_file:""|convert_image_format:"jpeg" }}').render(
+            {'url': 'http://example.com/', 'allow_complex': True}
+        )
+        assert pub.has_cached_complex_data(img)
+        value = pub.get_cached_complex_data(img)
+        assert value.orig_filename == 'file.jpeg'
+
+
+def test_convert_image_format_errors(pub):
+    pub.loggederror_class.wipe()
+    with pub.complex_data():
+        img = Template('{{ "xxx"|convert_image_format:"gif" }}').render({'allow_complex': True})
+        assert pub.has_cached_complex_data(img)
+        assert pub.get_cached_complex_data(img) is None
+    assert pub.loggederror_class.count() == 1
+    assert (
+        pub.loggederror_class.select()[0].summary
+        == '|convert_image_format: unknown format (must be one of jpeg, pdf, png)'
+    )
+
+    pub.loggederror_class.wipe()
+    with pub.complex_data():
+        img = Template('{{ "xxx"|convert_image_format:"jpeg" }}').render({'allow_complex': True})
+        assert pub.has_cached_complex_data(img)
+        assert pub.get_cached_complex_data(img) is None
+    assert pub.loggederror_class.count() == 1
+    assert pub.loggederror_class.select()[0].summary == '|convert_image_format: missing input'
+
+    pub.loggederror_class.wipe()
+    with mock.patch('subprocess.run', side_effect=FileNotFoundError()):
+        with pub.complex_data():
+            img = Template('{{ url|qrcode|convert_image_format:"jpeg" }}').render(
+                {'url': 'http://example.com/', 'allow_complex': True}
+            )
+            assert pub.has_cached_complex_data(img)
+            assert pub.get_cached_complex_data(img) is None
+    assert pub.loggederror_class.count() == 1
+    assert pub.loggederror_class.select()[0].summary == '|convert_image_format: not supported'
+
+    pub.loggederror_class.wipe()
+    with mock.patch(
+        'subprocess.run', side_effect=subprocess.CalledProcessError(returncode=-1, cmd='xx', stderr=b'xxx')
+    ):
+        with pub.complex_data():
+            img = Template('{{ url|qrcode|convert_image_format:"jpeg" }}').render(
+                {'url': 'http://example.com/', 'allow_complex': True}
+            )
+            assert pub.has_cached_complex_data(img)
+            assert pub.get_cached_complex_data(img) is None
+    assert pub.loggederror_class.count() == 1
+    assert pub.loggederror_class.select()[0].summary == '|convert_image_format: conversion error (xxx)'

tests/test_widgets.py

@@ -595,7 +595,7 @@ def test_wysiwygwidget_img():
 def test_mini_rich_text_widget():
     widget = MiniRichTextWidget('test')
     form = MockHtmlForm(widget)
-    assert 'data-godo-schema="basic"' in form.as_html
+    assert PyQuery(form.as_html)('godo-editor[schema=basic]')
 
 
 def test_mini_rich_text_widget_maxlength():
@@ -613,7 +613,7 @@ def test_mini_rich_text_widget_maxlength():
 def test_rich_text_widget():
     widget = RichTextWidget('test')
     form = MockHtmlForm(widget)
-    assert 'data-godo-schema="full"' in form.as_html
+    assert PyQuery(form.as_html)('godo-editor[schema=full]')
 
 
 def test_select_hint_widget():

tests/workflow/test_display_message.py

@@ -58,7 +58,7 @@ def test_display_message_rich_text(pub):
 
     app = login(get_app(pub))
     resp = app.get(display_message.get_admin_url())
-    assert resp.pyquery('textarea[data-godo-schema]')  # godo
+    assert resp.pyquery('godo-editor')  # godo
 
     display_message.message = '<table><tr><td>hello world</td></tr></table>'
     workflow.store()
@@ -76,13 +76,13 @@ def test_display_message_rich_text(pub):
     display_message.message = '<ul>{% for item in lists %}<li>{{ item }}</li>{% endfor %}</ul>'
     workflow.store()
     resp = app.get(display_message.get_admin_url())
-    assert resp.pyquery('textarea:not([data-config]):not([data-godo-schema])')  # plain textarea
+    assert resp.pyquery('textarea:not([data-config])')  # plain textarea
 
     pub.site_options.set('options', 'rich-text-wf-displaymsg', 'auto-textarea')
     with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
         pub.site_options.write(fd)
     resp = app.get(display_message.get_admin_url())
-    assert resp.pyquery('textarea:not([data-config]):not([data-godo-schema])')  # plain textarea
+    assert resp.pyquery('textarea:not([data-config])')  # plain textarea
 
     pub.site_options.set('options', 'rich-text-wf-displaymsg', 'auto-ckeditor')
     with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
@@ -102,10 +102,10 @@ def test_display_message_rich_text(pub):
     with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
         pub.site_options.write(fd)
     resp = app.get(display_message.get_admin_url())
-    assert resp.pyquery('textarea[data-godo-schema]')  # godo
+    assert resp.pyquery('godo-editor')  # godo
 
     pub.site_options.set('options', 'rich-text-wf-displaymsg', 'textarea')
     with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
         pub.site_options.write(fd)
     resp = app.get(display_message.get_admin_url())
-    assert resp.pyquery('textarea:not([data-config]):not([data-godo-schema])')  # plain textarea
+    assert resp.pyquery('textarea:not([data-config])')  # plain textarea

wcs/admin/data_sources.py

@@ -388,7 +388,7 @@ class NamedDataSourcePage(Directory):
     def preview_block(self):
         get_request().disable_error_notifications = True
         get_request().ignore_session = True
-        get_response().filter = {'raw': True}
+        get_response().raw = True
         data_source = self.datasource.extended_data_source
         try:
             items = get_structured_items(data_source)

wcs/admin/settings.py

@@ -39,7 +39,7 @@ from wcs.carddef import CardDef
 from wcs.data_sources import NamedDataSource
 from wcs.fields.map import MapOptionsMixin
 from wcs.formdef import FormDef, FormdefImportError, get_formdefs_of_all_kinds
-from wcs.qommon import _, audit, errors, get_cfg, ident, misc, template
+from wcs.qommon import _, audit, errors, get_cfg, ident, misc, pgettext_lazy, template
 from wcs.qommon.admin.cfg import cfg_submit, hobo_kwargs
 from wcs.qommon.admin.emails import EmailsDirectory
 from wcs.qommon.admin.texts import TextsDirectory
@@ -487,7 +487,7 @@ class SettingsDirectory(AccessControlled, Directory):
         ('data-sources', 'data_sources'),
         'wscalls',
         ('api-access', 'api_access'),
-        ('submission-channels', 'submission_channels'),
+        ('backoffice-submission', 'backoffice_submission'),
     ]
 
     emails = EmailsDirectory()
@@ -636,10 +636,10 @@ class SettingsDirectory(AccessControlled, Directory):
                 _('Geolocation'),
                 _('Configure geolocation and geocoding'),
             )
-        if enabled('submission-channels'):
-            r += htmltext('<dt><a href="submission-channels">%s</a></dt> <dd>%s</dd>') % (
-                _('Submission channels'),
-                _('Configure submission channels related options'),
+        if enabled('backoffice-submission'):
+            r += htmltext('<dt><a href="backoffice-submission">%s</a></dt> <dd>%s</dd>') % (
+                _('Backoffice Submission'),
+                _('Configure backoffice submission related options'),
             )
         if enabled('users'):
             r += htmltext('<dt><a href="users/">%s</a></dt> <dd>%s</dd>') % (_('Users'), _('Configure users'))
@@ -1272,9 +1272,29 @@ $('#form_default-zoom-level').on('change', function() {
             )
             return redirect('.')
 
-    def submission_channels(self):
+    def backoffice_submission(self):
         form = Form(enctype='multipart/form-data')
         submission_channels_cfg = get_cfg('submission-channels', {})
+        backoffice_submission_cfg = get_cfg('backoffice-submission', {})
+        form.add(
+            RadiobuttonsWidget,
+            'sidebar_menu_entry',
+            title=_('Sidebar menu entry'),
+            value=backoffice_submission_cfg.get('sidebar_menu_entry', 'visible'),
+            options=[
+                ('visible', pgettext_lazy('sidebar_menu_entry', 'Visible'), 'visible'),
+                ('hidden', pgettext_lazy('sidebar_menu_entry', 'Hidden'), 'hidden'),
+            ],
+            extra_css_class='widget-inline-radio',
+        )
+        form.add(
+            StringWidget,
+            'redirect',
+            title=_('URL for backoffice submission'),
+            hint=_('Leave empty to use native screen.'),
+            value=backoffice_submission_cfg.get('redirect', ''),
+            size=80,
+        )
         form.add(
             CheckboxWidget,
             'include-in-global-listing',
@@ -1288,18 +1308,15 @@ $('#form_default-zoom-level').on('change', function() {
             return redirect('.')
 
         if not form.is_submitted() or form.has_errors():
-            get_response().breadcrumb.append(('submission-channels', _('Submission channels')))
-            get_response().set_title(_('Submission channels'))
+            get_response().breadcrumb.append(('backoffice-submission', _('Backoffice Submission')))
+            get_response().set_title(_('Backoffice submission settings'))
             r = TemplateIO(html=True)
-            r += htmltext('<h2>%s</h2>') % _('Submission channels')
+            r += htmltext('<h2>%s</h2>') % _('Backoffice submission settings')
             r += form.render()
             return r.getvalue()
         else:
-            cfg_submit(
-                form,
-                'submission-channels',
-                ('include-in-global-listing',),
-            )
+            cfg_submit(form, 'submission-channels', ('include-in-global-listing',))
+            cfg_submit(form, 'backoffice-submission', ('sidebar_menu_entry', 'redirect'))
             return redirect('.')
 
 

wcs/admin/tests.py

@@ -535,6 +535,7 @@ class TestResultDetailPage(Directory):
         formdata.geolocations = formdata_json.get('geolocations')
         formdata.criticality_level = formdata_json['criticality_level']
         formdata.anonymised = formdata_json['anonymised']
+        formdata.workflow_data = formdata_json.get('workflow', {}).get('data', {})
         formdata.set_auto_fields()
 
         # load fields

wcs/admin/users.py

@@ -401,7 +401,7 @@ class UsersDirectory(Directory):
         r += htmltext('</div>')
 
         if get_request().form.get('ajax') == 'true':
-            get_response().filter = {'raw': True}
+            get_response().raw = True
             return r.getvalue()
 
         ident_methods = get_cfg('identification', {}).get('methods', [])

wcs/admin/workflow_tests.py

@@ -43,7 +43,10 @@ class WorkflowTestActionPage(Directory):
 
         self.action.fill_admin_form(form, self.formdef)
 
-        form.add_submit('submit', _('Submit'))
+        if not form.widgets:
+            form.add_global_errors([htmltext(self.action.empty_form_error)])
+        else:
+            form.add_submit('submit', _('Submit'))
         form.add_submit('cancel', _('Cancel'))
 
         if form.get_widget('cancel').parse():

wcs/admin/wscalls.py

@@ -161,7 +161,7 @@ class NamedWsCallPage(Directory):
     def usage(self):
         get_request().disable_error_notifications = True
         get_request().ignore_session = True
-        get_response().filter = {'raw': True}
+        get_response().raw = True
 
         usage = {}
 

wcs/backoffice/deprecations.py

@@ -88,6 +88,8 @@ class DeprecationsDirectory(Directory):
             'script': _('Filesystem Script'),
             'fields': _('Obsolete field types'),
             'actions': _('Obsolete action types'),
+            'csv-connector': _('CSV connector'),
+            'json-data-store': _('JSON Data Store connector'),
         }
 
     @property
@@ -99,6 +101,8 @@ class DeprecationsDirectory(Directory):
             'python-expression': _('Use Django templates.'),
             'python-prefill': _('Use Django templates.'),
             'python-data-source': _('Use cards.'),
+            'csv-connector': _('Use cards.'),
+            'json-data-store': _('Use cards.'),
             'rtf': _('Use OpenDocument format.'),
             'script': _('Use a dedicated template tags application.'),
             'fields': _('Use block fields to replace tables and ranked order fields.'),
@@ -118,6 +122,8 @@ class DeprecationsDirectory(Directory):
             'script': 'https://doc-publik.entrouvert.com/admin-fonctionnel/elements-deprecies/',
             'fields': 'https://doc-publik.entrouvert.com/admin-fonctionnel/elements-deprecies/',
             'actions': 'https://doc-publik.entrouvert.com/admin-fonctionnel/elements-deprecies/',
+            'csv-connector': 'https://doc-publik.entrouvert.com/admin-fonctionnel/elements-deprecies/',
+            'json-data-store': 'https://doc-publik.entrouvert.com/admin-fonctionnel/elements-deprecies/',
         }
 
 
@@ -253,6 +259,10 @@ class DeprecationsScanAfterJob(AfterJob):
                                 source=source,
                             )
                             break
+                if action.key == 'webservice_call':
+                    self.check_remote_call_url(
+                        action.url, location_label=location_label, url=url, source=source
+                    )
 
             for global_action in workflow.global_actions or []:
                 location_label = '%s / %s' % (workflow.name, _('trigger in %s') % global_action.name)
@@ -294,6 +304,10 @@ class DeprecationsScanAfterJob(AfterJob):
             url = named_ws_call.get_admin_url()
             for string in named_ws_call.get_computed_strings():
                 self.check_string(string, location_label=location_label, url=url, source=source)
+            if named_ws_call.request and named_ws_call.request.get('url'):
+                self.check_remote_call_url(
+                    named_ws_call.request['url'], location_label=location_label, url=url, source=source
+                )
             self.increment_count()
 
         for mail_template in mail_templates:
@@ -337,6 +351,7 @@ class DeprecationsScanAfterJob(AfterJob):
             self.check_string(
                 data_source.get('value'), location_label, url, python_check=False, source=source
             )
+            self.check_remote_call_url(data_source.get('value'), location_label, url, source=source)
 
     def check_string(self, string, location_label, url, source, python_check=True):
         if not isinstance(string, str):
@@ -361,6 +376,16 @@ class DeprecationsScanAfterJob(AfterJob):
             if re.findall(r'\Wscript\.\w', string):
                 self.add_report_line(location_label=location_label, url=url, category='script', source=source)
 
+    def check_remote_call_url(self, wscall_url, location_label, url, source):
+        if 'csvdatasource/' in (wscall_url or ''):
+            self.add_report_line(
+                location_label=location_label, url=url, category='csv-connector', source=source
+            )
+        if 'jsondatastore/' in (wscall_url or ''):
+            self.add_report_line(
+                location_label=location_label, url=url, category='json-data-store', source=source
+            )
+
     def add_report_line(self, **kwargs):
         if kwargs not in self.report_lines:
             self.report_lines.append(kwargs)

wcs/backoffice/management.py

@@ -717,7 +717,7 @@ class ManagementDirectory(Directory):
 
         if get_request().form.get('ajax') == 'true':
             get_request().ignore_session = True
-            get_response().filter = {'raw': True}
+            get_response().raw = True
             return r.getvalue()
 
         get_response().filter['sidebar'] = self.get_global_listing_sidebar(
@@ -1240,15 +1240,16 @@ class FormPage(Directory, TempfileDirectoryMixin):
             result = htmltext('<div class="widget operator-and-value-widget">')
             result += htmltext('<div class="title-and-operator">')
             result += filter_widget.render_title(filter_widget.get_title())
-            result += htmltext('<div class="operator">')
-            operator_widget = SingleSelectWidget(
-                filter_field_operator_key,
-                options=[(o[0], o[1], o[0]) for o in operators],
-                value=filter_field_operator,
-                render_br=False,
-            )
-            result += operator_widget.render_content()
-            result += htmltext('</div>')
+            if operators:
+                result += htmltext('<div class="operator">')
+                operator_widget = SingleSelectWidget(
+                    filter_field_operator_key,
+                    options=[(o[0], o[1], o[0]) for o in operators],
+                    value=filter_field_operator,
+                    render_br=False,
+                )
+                result += operator_widget.render_content()
+                result += htmltext('</div>')
             result += htmltext('</div>')
             result += htmltext('<div class="value">')
             result += filter_widget.render_content()
@@ -1330,13 +1331,14 @@ class FormPage(Directory, TempfileDirectoryMixin):
                         filtered_user = None
                     filtered_user_value = filtered_user.display_name if filtered_user else _('Unknown')
                     options += [(filter_field_value, filtered_user_value, filter_field_value)]
-                r += SingleSelectWidget(
+                widget = SingleSelectWidget(
                     filter_field_key,
                     title=filter_field.label,
                     options=options,
                     value=filter_field_value,
                     render_br=False,
-                ).render()
+                )
+                r += render_widget(widget, operators=[])
 
             elif filter_field.key == 'submission-agent-id':
                 r += HiddenWidget(filter_field_key, value=filter_field_value).render()
@@ -1356,13 +1358,14 @@ class FormPage(Directory, TempfileDirectoryMixin):
                 options = [('', '', '')] + [
                     (x[0], x[1], x[0]) for x in self.formdef.workflow.get_sorted_functions()
                 ]
-                r += SingleSelectWidget(
+                widget = SingleSelectWidget(
                     filter_field_key,
                     title=filter_field.label,
                     options=options,
                     value=filter_field_value,
                     render_br=False,
-                ).render()
+                )
+                r += render_widget(widget, operators=[])
 
             elif filter_field.key == 'internal-id':
                 widget = StringWidget(
@@ -2442,7 +2445,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
 
         if get_request().form.get('ajax') == 'true':
             get_request().ignore_session = True
-            get_response().filter = {'raw': True}
+            get_response().raw = True
             r = TemplateIO(html=True)
             r += multi_form.render()
             r += get_session().display_message()
@@ -3129,7 +3132,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
 
         if get_request().form.get('ajax') == 'true':
             get_request().ignore_session = True
-            get_response().filter = {'raw': True}
+            get_response().raw = True
             return r.getvalue()
 
         page = TemplateIO(html=True)
@@ -3368,13 +3371,13 @@ class FormBackOfficeStatusPage(FormStatusPage):
 
     def lateral_block(self):
         self.check_receiver()
-        get_response().filter = {'raw': True}
+        get_response().raw = True
         response = self.get_lateral_block()
         return response
 
     def user_pending_forms(self):
         self.check_receiver()
-        get_response().filter = {'raw': True}
+        get_response().raw = True
         response = self.get_user_pending_forms()
 
         # preemptive locking of forms
@@ -4159,7 +4162,7 @@ class FormBackOfficeStatusPage(FormStatusPage):
             or get_publisher().get_backoffice_root().is_accessible('workflows')
         ):
             raise errors.AccessForbiddenError()
-        get_response().filter = {'raw': True}
+        get_response().raw = True
         return self.test_tool_result()
 
 

wcs/backoffice/submission.py

@@ -23,14 +23,15 @@ from quixote import get_publisher, get_request, get_response, get_session, redir
 from quixote.directory import Directory
 from quixote.html import TemplateIO, htmltext
 
+from wcs.backoffice.pagination import pagination_links
 from wcs.categories import Category
 from wcs.formdata import FormData
 from wcs.formdef import FormDef
 from wcs.forms.common import FormStatusPage
 from wcs.forms.root import FormPage as PublicFormFillPage
-from wcs.sql_criterias import Equal, StrictNotEqual
+from wcs.sql_criterias import Contains, Equal, StrictNotEqual
 
-from ..qommon import _, errors, misc
+from ..qommon import _, errors, get_cfg, misc, template
 from ..qommon.form import Form, HtmlWidget
 
 
@@ -174,7 +175,7 @@ class FormFillPage(PublicFormFillPage):
         return super()._q_index(*args, **kwargs)
 
     def lateral_block(self):
-        get_response().filter = {'raw': True}
+        get_response().raw = True
         response = self.get_lateral_block()
         return response
 
@@ -427,15 +428,18 @@ class FormFillPage(PublicFormFillPage):
 
 
 class SubmissionDirectory(Directory):
-    _q_exports = ['', 'count']
+    _q_exports = ['', 'pending', 'count']
 
     def _q_traverse(self, path):
         get_response().set_backoffice_section('submission')
+        get_response().breadcrumb.append(('submission/', _('Submission')))
         return super()._q_traverse(path)
 
     def is_accessible(self, user, traversal=False):
         if not user.can_go_in_backoffice():
             return False
+        if traversal is False and get_cfg('backoffice-submission', {}).get('sidebar_menu_entry') == 'hidden':
+            return False
         # check user has at least one role set for backoffice submission
         for role_id in user.roles or []:
             ids = FormDef.get_ids_with_indexed_value('backoffice_submission_roles', role_id)
@@ -443,7 +447,7 @@ class SubmissionDirectory(Directory):
                 return True
         return False
 
-    def get_submittable_formdefs(self):
+    def get_submittable_formdefs(self, prefetch=True):
         user = get_request().user
 
         agent_ids = set()
@@ -460,113 +464,144 @@ class SubmissionDirectory(Directory):
                 continue
             list_forms.append(formdef)
 
-            # prefetch formdatas
-            data_class = formdef.data_class()
-            formdef._formdatas = data_class.select(
-                [Equal('status', 'draft'), Equal('backoffice_submission', True)]
+            if prefetch:
+                # prefetch formdatas
+                data_class = formdef.data_class()
+                formdef._formdatas = data_class.select(
+                    [Equal('status', 'draft'), Equal('backoffice_submission', True)]
+                )
+                formdef._formdatas.sort(
+                    key=lambda x: x.receipt_time or make_aware(datetime.datetime(1900, 1, 1))
+                )
+                agent_ids.update([x.submission_agent_id for x in formdef._formdatas if x.submission_agent_id])
+
+        if prefetch:
+            # prefetch agents
+            self.prefetched_agents = {
+                str(x.id): x
+                for x in get_publisher().user_class.get_ids(list(agent_ids), ignore_errors=True)
+                if x is not None
+            }
+
+        return list_forms
+
+    def get_categories(self, list_formdefs):
+        cats = Category.select()
+        Category.sort_by_position(cats)
+        for cat in cats:
+            cat.formdefs = [x for x in list_formdefs if str(x.category_id) == str(cat.id)]
+        misc_cat = Category(name=_('Misc'))
+        misc_cat.formdefs = [x for x in list_formdefs if not x.category]
+        cats.append(misc_cat)
+        return cats
+
+    def _q_index(self):
+        redirect_url = get_cfg('backoffice-submission', {}).get('redirect')
+        if redirect_url:
+            redirect_url = misc.get_variadic_url(
+                redirect_url, get_publisher().substitutions.get_context_variables(mode='lazy')
             )
-            formdef._formdatas.sort(key=lambda x: x.receipt_time or make_aware(datetime.datetime(1900, 1, 1)))
-            agent_ids.update([x.submission_agent_id for x in formdef._formdatas if x.submission_agent_id])
+            if redirect_url:
+                return redirect(redirect_url)
+
+        get_response().breadcrumb.append(('submission/', _('Submission')))
+        get_response().set_title(_('Submission'))
+
+        list_forms = self.get_submittable_formdefs(prefetch=False)
+
+        context = {'categories': self.get_categories(list_forms)}
+        return template.QommonTemplateResponse(
+            templates=['wcs/backoffice/submission.html'], context=context, is_django_native=True
+        )
+
+    def pending(self):
+        get_response().breadcrumb.append(('pending', _('Pending submissions')))
+        get_response().set_title(_('Pending submissions'))
+        get_response().add_javascript(['wcs.listing.js'])
+
+        limit = misc.get_int_or_400(
+            get_request().form.get('limit', get_publisher().get_site_option('default-page-size') or 20)
+        )
+        offset = misc.get_int_or_400(get_request().form.get('offset', 0))
+        order_by = misc.get_order_by_or_400(
+            get_request().form.get(
+                'order_by', get_publisher().get_site_option('default-sort-order') or '-receipt_time'
+            )
+        )
+        include_submission_channel = misc.get_cfg('submission-channels', {}).get('include-in-global-listing')
+
+        list_formdefs = self.get_submittable_formdefs(prefetch=False)
+        criterias = [
+            Equal('status', 'draft'),
+            Equal('backoffice_submission', True),
+            Contains('formdef_id', [x.id for x in list_formdefs]),
+        ]
+
+        r = TemplateIO(html=True)
+
+        r += htmltext('<table id="listing" class="main">')
+        r += htmltext('<thead><tr>')
+        if include_submission_channel:
+            r += htmltext('<th data-field-sort-key="submission_channel"><span>%s</span></th>') % _('Channel')
+        r += htmltext('<th data-field-sort-key="formdef_name"><span>%s</span></th>') % _('Form')
+        r += htmltext('<th data-field-sort-key="receipt_time"><span>%s</span></th>') % _('Created')
+        r += htmltext('<th><span>%s</span></th>') % _('Submission Agent')
+        r += htmltext('</tr></thead>')
+        r += htmltext('<tbody>\n')
+
+        from wcs.sql import AnyFormData
+
+        total_count = AnyFormData.count(criterias)
+        formdatas = AnyFormData.select(criterias, order_by=order_by, limit=limit, offset=offset)
 
         # prefetch agents
+        agent_ids = set()
+        agent_ids.update([x.submission_agent_id for x in formdatas if x.submission_agent_id])
         self.prefetched_agents = {
             str(x.id): x
             for x in get_publisher().user_class.get_ids(list(agent_ids), ignore_errors=True)
             if x is not None
         }
 
-        return list_forms
-
-    def _q_index(self):
-        get_response().breadcrumb.append(('submission/', _('Submission')))
-        get_response().set_title(_('Submission'))
-
-        list_forms = self.get_submittable_formdefs()
-        cats = Category.select()
-        Category.sort_by_position(cats)
-        for cat in cats:
-            cat.formdefs = [x for x in list_forms if str(x.category_id) == str(cat.id)]
-        misc_cat = Category(name=_('Misc'))
-        misc_cat.formdefs = [x for x in list_forms if not x.category]
-        cats.append(misc_cat)
-
-        r = TemplateIO(html=True)
-        r += get_session().display_message()
-        modes = ['empty', 'create', 'existing']
-        for mode in modes:
-            list_content = TemplateIO()
-            for cat in cats:
-                if not cat.formdefs:
-                    continue
-                list_content += self.form_list(cat.formdefs, title=cat.name, mode=mode)
-            if not list_content.getvalue().strip():
-                continue
-            r += htmltext('<h2>%s</h2>') % {
-                'create': _('New submission'),
-                'existing': _('Running submission'),
-                'empty': _('Submission to complete'),
-            }.get(mode)
-            r += htmltext(f'<ul class="biglist {mode}">')
-            r += htmltext(list_content.getvalue())
-            r += htmltext('</ul>')
-
-        return r.getvalue()
-
-    def form_list(self, formdefs, title=None, mode='create'):
-        r = TemplateIO(html=True)
-        if mode != 'create':
-            skip = True
-            for formdef in formdefs:
-                skip &= not (bool(formdef._formdatas))
-            if skip:
-                return
-
-        first = True
-
-        for formdef in formdefs:
-            if mode != 'create':
-                formdatas = formdef._formdatas[:]
-                if mode == 'empty':
-                    formdatas = [x for x in formdatas if x.has_empty_data()]
-                elif mode == 'existing':
-                    formdatas = [x for x in formdatas if not x.has_empty_data()]
-                if not formdatas:
-                    continue
-
-            if first and title:
-                r += htmltext('<li><h3>%s</h3></li>') % title
-                first = False
-
-            r += htmltext('<li>')
-            if mode == 'create':
-                r += htmltext('<strong class="label"><a href="%s/">%s</a></strong>') % (
-                    formdef.url_name,
-                    formdef.name,
-                )
+        for formdata in formdatas:
+            url = f'{formdata.formdef.url_name}/{formdata.id}/'
+            r += htmltext(f'<tr data-link="{url}">')
+            if include_submission_channel:
+                r += htmltext('<td>%s</td>') % formdata.get_submission_channel_label()
+            r += htmltext(f'<td><a href="{url}">{formdata.get_display_name()}')
+            if formdata.default_digest:
+                r += htmltext(' <small>%s</small>') % formdata.default_digest
+            r += htmltext('</a></td>')
+            r += htmltext('<td class="cell-time">%s</td>') % misc.localstrftime(formdata.receipt_time)
+            agent_user = self.prefetched_agents.get(formdata.submission_agent_id)
+            if agent_user:
+                r += htmltext('<td class="cell-user">%s</td>') % agent_user.get_display_name()
             else:
-                r += htmltext('<strong class="label"><a class="fake">%s</a></strong>') % formdef.name
-            r += htmltext('</li>')
-            if mode == 'create':
-                continue
-            for formdata in formdatas:
-                r += htmltext('<li class="smallitem">')
-                label = ''
-                if formdata.submission_channel:
-                    label = '%s ' % formdata.get_submission_channel_label()
-                label += _('#%(id)s, %(time)s') % {
-                    'id': formdata.id,
-                    'time': misc.localstrftime(formdata.receipt_time)
-                    if formdata.receipt_time
-                    else _('unknown date'),
-                }
-                if formdata.submission_agent_id:
-                    agent_user = self.prefetched_agents.get(formdata.submission_agent_id)
-                    if agent_user:
-                        label += ' (%s)' % agent_user.display_name
-                r += htmltext('<a href="%s/%s/">%s</a>') % (formdef.url_name, formdata.id, label)
-                r += htmltext('</li>')
+                r += htmltext('<td class="cell-user cell-no-user">-</td>')
+            r += htmltext('</tr>\n')
 
-        return r.getvalue()
+        r += htmltext('</tbody></table>')
+
+        if (offset > 0) or (total_count > limit > 0):
+            r += pagination_links(offset, limit, total_count)
+
+        if get_request().form.get('ajax') == 'true':
+            get_request().ignore_session = True
+            get_response().filter = {'raw': True}
+            return r.getvalue()
+
+        rt = TemplateIO(html=True)
+        rt += htmltext('<div id="appbar">')
+        rt += htmltext('<h2>%s</h2>') % _('Pending submissions')
+        rt += htmltext('</div>')
+        rt += r.getvalue()
+        form = Form(use_tokens=False, id='listing-settings', method='get', action='pending')
+        form.add_hidden('offset', offset)
+        form.add_hidden('limit', limit)
+        form.add_hidden('order_by', order_by)
+        rt += form.render()
+
+        return rt.getvalue()
 
     def count(self):
         formdefs = self.get_submittable_formdefs()
@@ -582,5 +617,4 @@ class SubmissionDirectory(Directory):
         return misc.json_response({'count': count})
 
     def _q_lookup(self, component):
-        get_response().breadcrumb.append(('submission/', _('Submission')))
         return FormFillPage(component)

wcs/compat.py

@@ -92,7 +92,7 @@ class TemplateWithFallbackView(TemplateView):
             response.reason_phrase = self.quixote_response.reason_phrase
         elif request.headers.get('X-Popup') == 'true':
             response = HttpResponse('<div><div class="popup-content">%s</div></div>' % context['body'])
-        elif 'raw' in (getattr(self.quixote_response, 'filter') or {}):
+        elif self.quixote_response.raw:
             # used for raw HTML snippets (for example in the test tool
             # results in inspect page).
             response = HttpResponse(context['body'])
@@ -161,7 +161,7 @@ class CompatWcsPublisher(WcsPublisher):
         if response.status_code == 304:
             # clients don't like to receive content with a 304
             return ''
-        if response.content_type != 'text/html':
+        if response.content_type != 'text/html' or response.raw:
             return output
         if not hasattr(response, 'filter') or not response.filter:
             return output

wcs/forms/common.py

@@ -112,7 +112,7 @@ class FileDirectory(Directory):
 
         # force potential HTML upload to be used as-is (not decorated with theme)
         # and with minimal permissions
-        response.filter = {}
+        response.raw = True
         response.set_header(
             'Content-Security-Policy',
             'default-src \'none\'; img-src %s;' % get_request().build_absolute_uri(),
@@ -1074,7 +1074,7 @@ class TempfileDirectoryMixin:
 
         # force potential HTML upload to be used as-is (not decorated with theme)
         # and with minimal permissions
-        response.filter = {}
+        response.raw = True
         response.set_header(
             'Content-Security-Policy',
             'default-src \'none\'; img-src %s;' % get_request().build_absolute_uri(),

wcs/forms/root.py

@@ -31,6 +31,7 @@ from django.utils.timezone import localtime
 from quixote import get_publisher, get_request, get_response, get_session, get_session_manager, redirect
 from quixote.directory import AccessControlled, Directory
 from quixote.errors import MethodNotAllowedError, RequestError
+from quixote.form import FormTokenWidget
 from quixote.html import TemplateIO, htmltext
 from quixote.util import randbytes
 
@@ -621,6 +622,9 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
         try:
             with get_publisher().substitutions.temporary_feed(transient_formdata, force_mode='lazy'):
                 form = self.create_form(page, displayed_fields, transient_formdata=transient_formdata)
+                if page_change is False and page_error_messages:
+                    # ignore form token when there are other errors
+                    form._names.pop('_form_id', None)
         except MissingBlockFieldError as e:
             logged_error = get_publisher().record_error(
                 str(e), exception=e, notify=True, formdef=self.formdef
@@ -1002,6 +1006,10 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
 
     def create_form(self, *args, **kwargs):
         form = self.formdef.create_form(*args, **kwargs)
+        if len(self.pages) == 1 and not self.formdef.confirmation:
+            # if there's a form with a single page, no confirmation, add native quixote
+            # CSRF protection.
+            form.add(FormTokenWidget, form.TOKEN_NAME)
         form.attrs['data-live-url'] = self.formdef.get_url(language=get_publisher().current_language) + 'live'
         form.attrs['data-live-validation-url'] = (
             self.formdef.get_url(language=get_publisher().current_language) + 'live-validation'

wcs/locale/fr/LC_MESSAGES/django.po

@@ -4,8 +4,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: wcs 0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-02-29 10:53+0100\n"
-"PO-Revision-Date: 2024-02-29 10:53+0100\n"
+"POT-Creation-Date: 2024-03-01 17:03+0100\n"
+"PO-Revision-Date: 2024-03-01 17:03+0100\n"
 "Last-Translator: Thomas Noël <tnoel@entrouvert.com>\n"
 "Language-Team: french\n"
 "Language: fr\n"
@@ -1760,8 +1760,8 @@ msgid "Stack trace (most recent call first)"
 msgstr "Trace (appels les plus récents en premier)"
 
 #: admin/logged_errors.py api_export_import.py backoffice/management.py
-#: formdata.py formdef.py statistics/views.py wf/create_formdata.py wf/form.py
-#: wf/resubmit.py
+#: backoffice/submission.py formdata.py formdef.py statistics/views.py
+#: wf/create_formdata.py wf/form.py wf/resubmit.py
 msgid "Form"
 msgstr "Formulaire"
 
@@ -2148,12 +2148,12 @@ msgid "Configure geolocation and geocoding"
 msgstr "Configurer la géolocalisation et le géocodage"
 
 #: admin/settings.py
-msgid "Submission channels"
-msgstr "Canaux de saisie"
+msgid "Backoffice Submission"
+msgstr "Saisie backoffice"
 
 #: admin/settings.py
-msgid "Configure submission channels related options"
-msgstr "Configurer les options en rapport avec les canaux de saisie"
+msgid "Configure backoffice submission related options"
+msgstr "Configurer les options en rapport avec la saisie par les agents"
 
 #: admin/settings.py
 msgid "Configure users"
@@ -2456,10 +2456,36 @@ msgstr ""
 "Si complété, envoie tous les courriels à cette adresse au lieu des vrais "
 "destinataires"
 
+#: admin/settings.py
+msgid "Sidebar menu entry"
+msgstr "Icône dans le menu latéral gauche"
+
+#: admin/settings.py
+msgctxt "sidebar_menu_entry"
+msgid "Visible"
+msgstr "Visible"
+
+#: admin/settings.py
+msgctxt "sidebar_menu_entry"
+msgid "Hidden"
+msgstr "Cachée"
+
+#: admin/settings.py
+msgid "URL for backoffice submission"
+msgstr "URL pour la saisie backoffice"
+
+#: admin/settings.py
+msgid "Leave empty to use native screen."
+msgstr "Laisser vide pour utiliser l’écran par défaut."
+
 #: admin/settings.py
 msgid "Include submission channel column in global listing"
 msgstr "Inclure une colonne avec le canal de saisie dans la vue globale"
 
+#: admin/settings.py
+msgid "Backoffice submission settings"
+msgstr "Paramètres de saisie backoffice"
+
 #: admin/settings.py
 msgid "Back to settings"
 msgstr "Retourner au paramètres"
@@ -3974,6 +4000,14 @@ msgstr "Types de champ dépréciés"
 msgid "Obsolete action types"
 msgstr "Types d’actions dépréciés"
 
+#: backoffice/deprecations.py
+msgid "CSV connector"
+msgstr "Connecteur « Fichier tableur »"
+
+#: backoffice/deprecations.py
+msgid "JSON Data Store connector"
+msgstr "Connecteur « Stockage de données JSON »"
+
 #: backoffice/deprecations.py
 msgid "Use Django templates."
 msgstr "Utiliser des gabarits Django."
@@ -4252,7 +4286,7 @@ msgstr[1] "%(total)s éléments"
 msgid "Reference"
 msgstr "Référence"
 
-#: backoffice/management.py
+#: backoffice/management.py backoffice/submission.py
 msgid "Created"
 msgstr "Date de création"
 
@@ -4325,7 +4359,7 @@ msgstr "Fin"
 msgid "Current User Function"
 msgstr "Fonction de l’utilisateur connecté"
 
-#: backoffice/management.py
+#: backoffice/management.py backoffice/submission.py
 msgid "Submission Agent"
 msgstr "Agent à la saisie"
 
@@ -4837,6 +4871,7 @@ msgid "Per page: "
 msgstr "Par page : "
 
 #: backoffice/root.py backoffice/submission.py
+#: templates/wcs/backoffice/submission.html
 #: templates/wcs/backoffice/test_edit_sidebar.html
 msgid "Submission"
 msgstr "Saisie"
@@ -4993,26 +5028,9 @@ msgstr ""
 msgid "Discard this form"
 msgstr "Abandonner la saisie"
 
-#: backoffice/submission.py
-msgid "New submission"
-msgstr "Nouvelle demande"
-
-#: backoffice/submission.py
-msgid "Running submission"
-msgstr "Saisie entamée"
-
-#: backoffice/submission.py
-msgid "Submission to complete"
-msgstr "Prédemande"
-
-#: backoffice/submission.py
-#, python-format
-msgid "#%(id)s, %(time)s"
-msgstr "n°%(id)s, %(time)s"
-
-#: backoffice/submission.py
-msgid "unknown date"
-msgstr "date inconnue"
+#: backoffice/submission.py templates/wcs/backoffice/submission.html
+msgid "Pending submissions"
+msgstr "Saisies entamées"
 
 #: blocks.py
 msgid "Field block"
@@ -8629,7 +8647,7 @@ msgstr "liste"
 msgid "string"
 msgstr "texte"
 
-#: qommon/misc.py
+#: qommon/misc.py qommon/templatetags/qommon.py
 msgid "file"
 msgstr "fichier"
 
@@ -8990,6 +9008,24 @@ msgstr "|objects appelé sur une source invalide (%r)"
 msgid "|objects with invalid reference (%r)"
 msgstr "|objects utilisé avec une référence invalide (%r)"
 
+#: qommon/templatetags/qommon.py
+#, python-format
+msgid "|convert_image_format: unknown format (must be one of %s)"
+msgstr "|convert_image_format: format inconnu (doit être un de %s)"
+
+#: qommon/templatetags/qommon.py
+msgid "|convert_image_format: missing input"
+msgstr "|convert_image_format: données manquantes"
+
+#: qommon/templatetags/qommon.py
+msgid "|convert_image_format: not supported"
+msgstr "|convert_image_format: pas pris en charge"
+
+#: qommon/templatetags/qommon.py
+#, python-format
+msgid "|convert_image_format: conversion error (%s)"
+msgstr "|convert_image_format: erreur de conversion (%s)"
+
 #: qommon/templatetags/qommon.py
 #, python-format
 msgid "|check_no_duplicates not used on a list (%s)"
@@ -10496,6 +10532,10 @@ msgid ""
 msgstr ""
 "Erreur inattendue lors de l’appel webservice vers l’URL %(url)s : %(error)s."
 
+#: testdef.py
+msgid "method must be GET"
+msgstr "la méthode doit être GET"
+
 #: users.py
 #, python-format
 msgid "Session User Field: %s"
@@ -11941,6 +11981,10 @@ msgstr "Statut de la demande quand l’erreur s’est produite : %s"
 msgid "Simulate click on action button"
 msgstr "Clic sur un bouton d’action"
 
+#: workflow_tests.py
+msgid "Workflow has no action that displays a button."
+msgstr "Le workflow ne contient pas d’action qui affiche un bouton."
+
 #: workflow_tests.py
 #, python-format
 msgid "Click on \"%s\""
@@ -12059,6 +12103,18 @@ msgstr "Vérifier l’appel d’un webservice"
 msgid "Broken, missing webservice response"
 msgstr "Cassé, réponse webservice manquante"
 
+#: workflow_tests.py
+msgid ""
+"In order to assert a webservice is called, you must define corresponding "
+"webservice response."
+msgstr ""
+"Afin de vérifier l’appel d’un webservice, vous devez d’abord définir la "
+"réponse webservice correspondante."
+
+#: workflow_tests.py
+msgid "Add webservice response"
+msgstr "Ajouter une réponse webservice"
+
 #: workflow_tests.py
 #, python-format
 msgid ""

wcs/publisher.py

@@ -315,6 +315,7 @@ class WcsPublisher(QommonPublisher):
                             ('misc', ('default-position', 'default-zoom-level')),
                             ('sms', '*'),
                             ('submission-channels', '*'),
+                            ('backoffice-submission', '*'),
                             ('texts', '*'),
                             ('users', ('*_template',)),
                         ):

wcs/qommon/form.py

@@ -3636,6 +3636,7 @@ class MapWidget(CompositeWidget):
                         # lat;lon
                         self.map_attributes['data-def-lat'] = position.split(';')[0]
                         self.map_attributes['data-def-lng'] = position.split(';')[1]
+                        self.map_attributes['data-def-template'] = 'true'
                     else:
                         # address?
                         from wcs.wf.geolocate import GeolocateWorkflowStatusItem

wcs/qommon/http_response.py

@@ -28,6 +28,7 @@ class HTTPResponse(quixote.http_response.HTTPResponse):
     javascript_code_parts = None
     css_includes = None
     after_jobs = None
+    raw = False  # in case of html content, send result as is (True) or embedded in page template (False)
 
     def __init__(self, charset=None, **kwargs):
         quixote.http_response.HTTPResponse.__init__(self, charset=charset, **kwargs)

wcs/qommon/static/css/dc2/admin.scss

@@ -3096,13 +3096,10 @@ ul.objects-list.single-links li.loading-list-item {
 	}
 }
 
-ul.objects-list.single-links li.loading-list-item span {
-	padding: 0 0.5ex 0 2ex;
-}
-
+ul.objects-list.single-links li.loading-list-item p,
 ul.objects-list.single-links li.list-item-no-usage p {
+	margin: 0.5em 0 0 0;
 	padding: 0 0.5ex 0 2ex;
-	margin: 0;
 }
 
 .inline-hint-message {

wcs/qommon/template.py

@@ -78,7 +78,7 @@ def error_page(error_message, error_title=None, location_hint=None):
 def get_decorate_vars(body, response, generate_breadcrumb=True, **kwargs):
     from .publisher import get_cfg
 
-    if response.content_type != 'text/html':
+    if response.content_type != 'text/html' or response.raw:
         return {'body': body}
 
     if get_request().get_header('x-popup') == 'true':

wcs/qommon/templates/qommon/forms/widgets/mini-rich-text.html

@@ -1,9 +1,16 @@
 {% extends "qommon/forms/widget.html" %}
 {% block widget-control %}
-  <textarea style="width: 100%" id="form_{{widget.get_name_for_id}}" name="{{widget.name}}"
-            {% for attr in widget.attrs.items %}{{attr.0}}="{{attr.1}}" {% endfor %}
-            {% if widget.live_condition_source %}data-godo-instant-update="true"{% endif %}
-            data-godo-schema="{{widget.EDITION_MODE}}"
-            data-godo-update-event="wcs:live-update">{{widget.value|default:""}}</textarea>
+  <textarea hidden id="form_{{widget.get_name_for_id}}" name="{{widget.name}}">
+    {{widget.value|default:""}}
+  </textarea>
+  <godo-editor
+    style="width: 100%"
+    {% for attr in widget.attrs.items %}{{attr.0}}="{{attr.1}}" {% endfor %}
+    {% if widget.live_condition_source %}instant-update="true"{% endif %}
+    schema="{{widget.EDITION_MODE}}"
+    update-event="wcs:live-update"
+    linked-source="form_{{widget.get_name_for_id}}"
+  >
+  </godo-editor>
   <script type="module" src="/static/xstatic/js/godo.js?{{version_hash}}"></script>
 {% endblock %}

wcs/qommon/templatetags/qommon.py

@@ -24,6 +24,7 @@ import math
 import os
 import random
 import string
+import subprocess
 import urllib.parse
 from decimal import Decimal
 from decimal import DivisionByZero as DecimalDivisionByZero
@@ -1086,6 +1087,58 @@ def rename_file(value, new_name):
     return file_object
 
 
+@register.filter
+def convert_image_format(value, new_format):
+    from wcs.fields import FileField
+
+    formats = {
+        'jpeg': 'image/jpeg',
+        'pdf': 'application/pdf',
+        'png': 'image/png',
+    }
+    if new_format not in formats:
+        get_publisher().record_error(
+            _('|convert_image_format: unknown format (must be one of %s)') % ', '.join(formats.keys())
+        )
+        return None
+
+    try:
+        file_object = FileField.convert_value_from_anything(value)
+    except ValueError:
+        file_object = None
+    if not file_object:
+        get_publisher().record_error(_('|convert_image_format: missing input'))
+        return None
+
+    if file_object.base_filename:
+        current_name, current_format = os.path.splitext(file_object.base_filename)
+        if current_format == f'.{new_format}':
+            return file_object
+        new_name = f'{current_name}.{new_format}'
+    else:
+        new_name = '%s.%s' % (_('file'), new_format)
+
+    try:
+        proc = subprocess.run(
+            ['gm', 'convert', '-', f'{new_format}:-'],
+            input=file_object.get_content(),
+            capture_output=True,
+            check=True,
+        )
+    except FileNotFoundError:
+        get_publisher().record_error(_('|convert_image_format: not supported'))
+        return None
+    except subprocess.CalledProcessError as e:
+        get_publisher().record_error(_('|convert_image_format: conversion error (%s)' % e.stderr.decode()))
+        return None
+
+    new_file_object = FileField.convert_value_from_anything(
+        {'content': proc.stdout, 'filename': new_name, 'content_type': formats[new_format]}
+    )
+
+    return new_file_object
+
+
 @register.filter
 def first(value):
     try:

wcs/sql.py

@@ -1771,10 +1771,16 @@ $function$;"""
     ):
         # Second part: insert and update triggers for wcs_all_forms
         cur.execute(
-            'CREATE TRIGGER wcs_all_forms_fts_trg_ins AFTER INSERT ON wcs_all_forms FOR EACH ROW WHEN (NEW.fts IS NOT NULL) EXECUTE PROCEDURE wcs_search_tokens_trigger_fn();'
+            """CREATE TRIGGER wcs_all_forms_fts_trg_ins 
+            AFTER INSERT ON wcs_all_forms 
+            FOR EACH ROW WHEN (NEW.fts IS NOT NULL) 
+            EXECUTE PROCEDURE wcs_search_tokens_trigger_fn();"""
         )
         cur.execute(
-            'CREATE TRIGGER wcs_all_forms_fts_trg_upd AFTER UPDATE OF fts ON wcs_all_forms FOR EACH ROW WHEN (NEW.fts IS NOT NULL) EXECUTE PROCEDURE wcs_search_tokens_trigger_fn();'
+            """CREATE TRIGGER wcs_all_forms_fts_trg_upd
+            AFTER UPDATE OF fts ON wcs_all_forms
+            FOR EACH ROW WHEN (NEW.fts IS NOT NULL)
+            EXECUTE PROCEDURE wcs_search_tokens_trigger_fn();"""
         )
 
     if _table_exists(cur, 'searchable_formdefs') and not _trigger_exists(
@@ -1782,10 +1788,16 @@ $function$;"""
     ):
         # Third part: insert and update triggers for searchable_formdefs
         cur.execute(
-            'CREATE TRIGGER searchable_formdefs_fts_trg_ins AFTER INSERT ON searchable_formdefs FOR EACH ROW WHEN (NEW.fts IS NOT NULL) EXECUTE PROCEDURE wcs_search_tokens_trigger_fn();'
+            """CREATE TRIGGER searchable_formdefs_fts_trg_ins 
+            AFTER INSERT ON searchable_formdefs 
+            FOR EACH ROW WHEN (NEW.fts IS NOT NULL) 
+            EXECUTE PROCEDURE wcs_search_tokens_trigger_fn();"""
         )
         cur.execute(
-            'CREATE TRIGGER searchable_formdefs_fts_trg_upd AFTER UPDATE OF fts ON searchable_formdefs FOR EACH ROW WHEN (NEW.fts IS NOT NULL) EXECUTE PROCEDURE wcs_search_tokens_trigger_fn();'
+            """CREATE TRIGGER searchable_formdefs_fts_trg_upd 
+            AFTER UPDATE OF fts ON searchable_formdefs 
+            FOR EACH ROW WHEN (NEW.fts IS NOT NULL) 
+            EXECUTE PROCEDURE wcs_search_tokens_trigger_fn();"""
         )
 
 
@@ -1812,7 +1824,7 @@ def purge_obsolete_search_tokens(cur=None):
     own_cur = False
     if cur is None:
         own_cur = True
-        conn, cur = get_connection_and_cursor()
+        _, cur = get_connection_and_cursor()
 
     cur.execute(
         """DELETE FROM wcs_search_tokens 

wcs/templates/wcs/backoffice/submission.html

@@ -0,0 +1,29 @@
+{% extends "wcs/backoffice.html" %}
+{% load i18n %}
+
+{% block appbar-title %}{% trans "Submission" %}{% endblock %}
+
+{% block appbar-actions %}
+  <a href="pending">{% trans "Pending submissions" %}</a>
+{% endblock %}
+
+{% block content %}
+  {{ block.super }}
+
+  {% for category in categories %}
+    {% if category.formdefs %}
+      {% with section_folded_pref_name="folded-submission"|add:"-category-"|add:category.id %}
+        <div class="section foldable {% if user|get_preference:section_folded_pref_name %}folded{% endif %}"
+             data-section-folded-pref-name="{{ section_folded_pref_name }}">
+          <h2>{{ category.name }}</h2>
+          <ul class="objects-list single-links">
+            {% for formdef in category.formdefs %}
+              <li><a href="{{ formdef.url_name }}/">{{ formdef.name }}</a></li>
+            {% endfor %}
+          </ul>
+        </div>
+      {% endwith %}
+    {% endif %}
+  {% endfor %}
+
+{% endblock %}

wcs/templates/wcs/backoffice/wscall.html

@@ -50,7 +50,7 @@
   <div class="section">
     <h3>{% trans "Usage" %}</h3>
     <ul class="objects-list single-links" data-async-url="{{ publisher.get_request.get_path }}usage">
-      <li class="loading-list-item"><span>{% trans "Searching..." %}</span></li>
+      <li class="loading-list-item"><p>{% trans "Searching..." %}</p></li>
     </ul>
   </div>
 {% endblock %}

wcs/testdef.py

@@ -603,8 +603,8 @@ class MockWebserviceResponseAdapter(requests.adapters.HTTPAdapter):
     def send(self, request, *args, **kwargs):
         try:
             return self._send(request, *args, **kwargs)
-        except WebserviceResponseError:
-            raise requests.exceptions.RequestError
+        except WebserviceResponseError as e:
+            raise requests.RequestException(str(e))
         except Exception as e:
             # Webservice call can happen through templates which catch all exceptions.
             # Record error to ensure we have a trace nonetheless.
@@ -629,7 +629,7 @@ class MockWebserviceResponseAdapter(requests.adapters.HTTPAdapter):
         else:
             if request.method != 'GET':
                 request_info['forbidden_method'] = True
-                raise WebserviceResponseError
+                raise WebserviceResponseError(str(_('method must be GET')))
             return super().send(request, *args, **kwargs)
 
         request_info['webservice_response_id'] = response.id

wcs/workflow_tests.py

@@ -32,7 +32,12 @@ class WorkflowTestError(TestError):
 
 
 def get_test_action_options():
-    return [(x.key, x.label, x.key) for x in WorkflowTestAction.__subclasses__()]
+    actions = sorted(WorkflowTestAction.__subclasses__(), key=lambda x: x.label)
+
+    assertion_options = [(x.key, x.label, x.key) for x in actions if x.is_assertion]
+    other_options = [(x.key, x.label, x.key) for x in actions if not x.is_assertion]
+
+    return assertion_options + [('', '—', '')] + other_options
 
 
 def get_test_action_class_by_type(action_type):
@@ -167,6 +172,7 @@ class WorkflowTestAction(XmlStorableObject):
 
 class ButtonClick(WorkflowTestAction):
     label = _('Simulate click on action button')
+    empty_form_error = _('Workflow has no action that displays a button.')
 
     key = 'button-click'
     button_name = None
@@ -198,6 +204,9 @@ class ButtonClick(WorkflowTestAction):
             if isinstance(item, wf.choice.ChoiceWorkflowStatusItem) and item.status:
                 possible_button_names.add(item.label)
 
+        if not possible_button_names:
+            return
+
         possible_button_names = sorted(possible_button_names)
 
         value = self.button_name
@@ -473,6 +482,17 @@ class AssertWebserviceCall(WorkflowTestAction):
         else:
             return _('Broken, missing webservice response')
 
+    @property
+    def empty_form_error(self):
+        r = '<p>%s</p>' % _(
+            'In order to assert a webservice is called, you must define corresponding webservice response.'
+        )
+        r += '<p><a href="%swebservice-responses/">%s</a><p>' % (
+            self.parent.testdef.get_admin_url(),
+            _('Add webservice response'),
+        )
+        return r
+
     def perform(self, formdata, user):
         call_count = 0
         for response in formdata.used_webservice_responses.copy():
@@ -493,6 +513,9 @@ class AssertWebserviceCall(WorkflowTestAction):
             for response in self.parent.testdef.get_webservice_responses()
         ]
 
+        if not webservice_response_options:
+            return
+
         form.add(
             SingleSelectWidget,
             'webservice_response_id',