workflow_tests: apply global action timeout trigger on skip time (#88404)

This makes sure it works against postgresql installations who expects
Y-m-d dates.

setup.py

@@ -204,7 +204,7 @@ setup(
         'setproctitle',
         'phonenumbers',
         'emoji',
-        'pytest-freezegun',
+        'freezegun',
     ],
     package_dir={'wcs': 'wcs'},
     packages=find_packages(),

tests/admin_pages/test_card.py

@@ -1118,3 +1118,61 @@ def test_cards_last_test_result(pub):
 
     resp = resp.click('Last tests run')
     assert 'Result #%s' % test_result.id in resp.text
+
+
+def test_cards_management_options(pub):
+    create_superuser(pub)
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = [
+        fields.StringField(id='1', label='Test', varname='test'),
+    ]
+    carddef.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/cards/1/')
+
+    # Misc management
+    assert_option_display(resp, 'Management', 'Default')
+    resp = resp.click('Management', href='options/management')
+    assert resp.forms[0]['management_sidebar_items$elementgeneral'].checked is True
+    assert resp.forms[0]['management_sidebar_items$elementdownload-files'].checked is False
+    resp.forms[0]['management_sidebar_items$elementdownload-files'].checked = True
+    resp = resp.forms[0].submit().follow()
+    assert_option_display(resp, 'Management', 'Custom')
+    assert 'general' in CardDef.get(1).management_sidebar_items
+    assert 'download-files' in CardDef.get(1).management_sidebar_items
+
+    resp = resp.click('Management', href='options/management')
+    resp.forms[0]['management_sidebar_items$elementgeneral'].checked = False
+    resp = resp.forms[0].submit().follow()
+    assert 'general' not in CardDef.get(1).management_sidebar_items
+
+    resp = resp.click('Management', href='options/management')
+    resp.forms[0]['management_sidebar_items$elementgeneral'].checked = True
+    resp.forms[0]['management_sidebar_items$elementdownload-files'].checked = False
+    assert 'management_sidebar_items$elementuser' not in resp.forms[0].fields
+    resp = resp.forms[0].submit().follow()
+    assert CardDef.get(1).management_sidebar_items == {'__default__'}
+
+    carddef.user_support = 'optional'
+    carddef.store()
+
+    resp = resp.click('Management', href='options/management')
+    assert resp.forms[0]['management_sidebar_items$elementuser'].checked is True
+    resp = resp.forms[0].submit().follow()
+    assert CardDef.get(1).management_sidebar_items == {'__default__'}
+
+    assert_option_display(resp, 'Management', 'Default')
+    resp = resp.click('Management', href='options/management')
+    assert resp.form['history_pane_default_mode'].value == 'collapsed'
+    resp = resp.form.submit().follow()
+    assert_option_display(resp, 'Templates', 'Default')
+    resp = resp.click('Management', href='options/management')
+    resp.form['history_pane_default_mode'].value = 'expanded'
+    resp = resp.form.submit().follow()
+    assert_option_display(resp, 'Templates', 'Custom')
+    resp = resp.click('Management', href='options/management')
+    assert resp.form['history_pane_default_mode'].value == 'expanded'

tests/admin_pages/test_form.py

@@ -8,6 +8,7 @@ import xml.etree.ElementTree as ET
 
 import pytest
 import responses
+from django.utils.timezone import localtime
 from pyquery import PyQuery
 from webtest import Upload
 
@@ -263,6 +264,14 @@ def test_forms_edit_management(pub, formdef):
     resp = resp.forms[0].submit().follow()
     assert FormDef.get(1).management_sidebar_items == {'__default__'}
 
+    # unselect all
+    resp = resp.click('Management', href='options/management')
+    for field in resp.forms[0].fields:
+        if field.startswith('management_sidebar_items$'):
+            resp.forms[0][field].checked = False
+    resp = resp.forms[0].submit().follow()
+    assert FormDef.get(1).management_sidebar_items == set()
+
 
 def test_forms_edit_tracking_code(pub, formdef):
     create_superuser(pub)
@@ -1133,12 +1142,6 @@ def test_form_workflow_options(pub):
     resp = app.get('/backoffice/forms/1/')
     assert '"workflow-options"' not in resp.text
 
-    pub.site_options.set('options', 'enable-workflow-variable-parameter', 'true')
-    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
-        pub.site_options.write(fd)
-    resp = app.get('/backoffice/forms/1/')
-    assert '"workflow-options"' in resp.text
-
 
 def test_form_workflow_variables(pub):
     create_superuser(pub)
@@ -1979,7 +1982,7 @@ def test_form_preview_map_field(pub):
     app = login(get_app(pub))
     resp = app.get('/backoffice/forms/1/')
     assert 'qommon.map.js' in resp.text
-    assert resp.pyquery('#map-f1')
+    assert resp.pyquery('#form_f1.qommon-map')
 
 
 def test_form_preview_do_not_log_error(pub):
@@ -3691,6 +3694,35 @@ def test_form_edit_field_warnings(pub):
     assert not resp.pyquery('aside .errornotice')
     assert resp.pyquery('aside form[action=new]')
 
+    BlockDef.wipe()
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [
+        fields.StringField(id='123', required=True, label='Test'),
+        fields.StringField(id='234', required=True, label='Test2'),
+        fields.CommentField(id='345', label='comment'),
+    ]
+    block.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [
+        fields.StringField(id='1', label='Test'),
+        fields.BlockField(id='2', label='Block field', block_slug='foobar'),
+    ]
+    formdef.store()
+    resp = app.get('/backoffice/forms/%s/fields/' % formdef.id)
+    assert not resp.pyquery('.warningnotice')
+    formdef.fields[1].default_items_count = 1100
+    formdef.store()
+    resp = app.get('/backoffice/forms/%s/fields/' % formdef.id)
+    assert (
+        resp.pyquery('.warningnotice')
+        .text()
+        .startswith('There are at least 2201 data fields, including fields in blocks.')
+    )
+
     FormDef.wipe()
 
 
@@ -4816,6 +4848,141 @@ def test_admin_form_inspect_validation(pub):
     assert not resp.pyquery('[data-field-id="4"] .parameter-validation').length
 
 
+def test_admin_form_inspect_drafts(pub):
+    create_superuser(pub)
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.enable_tracking_codes = True
+    formdef.fields = [
+        fields.PageField(id='0', label='1st page'),
+        fields.StringField(id='1', label='string 1'),
+        fields.PageField(id='2', label='2nd page'),
+        fields.StringField(id='3', label='string 2'),
+        fields.PageField(id='4', label='3rd page'),
+        fields.StringField(id='5', label='string 3'),
+    ]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/%s/inspect' % formdef.id)
+    assert resp.pyquery('#inspect-drafts p').text() == 'There are currently no drafts for this form.'
+
+    data_class = formdef.data_class()
+    formdata = data_class()
+    formdata.status = 'draft'
+    formdata.page_id = '0'
+    formdata.receipt_time = localtime()
+    formdata.store()
+    formdata = data_class()
+    formdata.status = 'draft'
+    formdata.page_id = '2'
+    formdata.receipt_time = localtime()
+    formdata.store()
+    formdata = data_class()
+    formdata.status = 'draft'
+    formdata.page_id = '4'
+    formdata.receipt_time = localtime()
+    formdata.store()
+    formdata = data_class()
+    formdata.status = 'draft'
+    formdata.page_id = '_confirmation_page'
+    formdata.receipt_time = localtime()
+    formdata.store()
+    formdata = data_class()
+    formdata.status = 'draft'
+    formdata.page_id = 'xxxx'  # unkown page id
+    formdata.receipt_time = localtime()
+    formdata.store()
+
+    resp = app.get('/backoffice/forms/%s/inspect' % formdef.id)
+    assert resp.pyquery('#inspect-drafts p')[0].text == 'Statistics on drafts by page.'
+    assert (
+        resp.pyquery('#inspect-drafts p')[1].text
+        == 'Lifespan of drafts (in days): %s.' % formdef.get_drafts_lifespan()
+    )
+
+    assert resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="0"]').length == 1
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="0"] td.label').text()
+        == '1st page'
+    )
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="0"] td.percent').text()
+        == '20%'
+    )
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="0"] td.total').text()
+        == '(1/5)'
+    )
+
+    assert resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="2"]').length == 1
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="2"] td.label').text()
+        == '2nd page'
+    )
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="2"] td.percent').text()
+        == '20%'
+    )
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="2"] td.total').text()
+        == '(1/5)'
+    )
+
+    assert resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="4"]').length == 1
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="4"] td.label').text()
+        == '3rd page'
+    )
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="4"] td.percent').text()
+        == '20%'
+    )
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="4"] td.total').text()
+        == '(1/5)'
+    )
+
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="_confirmation_page"]').length
+        == 1
+    )
+    assert (
+        resp.pyquery(
+            'table[data-table-id="rate-among-drafts"] tr[data-page-id="_confirmation_page"] td.label'
+        ).text()
+        == 'Confirmation page'
+    )
+    assert (
+        resp.pyquery(
+            'table[data-table-id="rate-among-drafts"] tr[data-page-id="_confirmation_page"] td.percent'
+        ).text()
+        == '20%'
+    )
+    assert (
+        resp.pyquery(
+            'table[data-table-id="rate-among-drafts"] tr[data-page-id="_confirmation_page"] td.total'
+        ).text()
+        == '(1/5)'
+    )
+
+    assert resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="_unknown"]').length == 1
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="_unknown"] td.label').text()
+        == 'Unknown'
+    )
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="_unknown"] td.percent').text()
+        == '20%'
+    )
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="_unknown"] td.total').text()
+        == '(1/5)'
+    )
+
+
 def test_form_import_fields(pub):
     create_superuser(pub)
     create_role(pub)
@@ -4991,3 +5158,24 @@ def test_forms_last_test_result(pub, formdef):
     TestDef.remove_object(testdef.id)
     resp = app.get('/backoffice/forms/1/')
     assert 'Last tests run' not in resp.text
+
+
+def test_admin_form_sql_integrity_error(pub):
+    create_superuser(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [fields.BoolField(id='1', label='Bool')]
+    formdef.store()
+
+    formdef.fields = [fields.StringField(id='1', label='String')]
+    formdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get(formdef.get_admin_url())
+    assert (
+        resp.pyquery('.errornotice summary').text()
+        == 'There are integrity errors in the database column types.'
+    )
+    assert resp.pyquery('.errornotice li').text() == 'String, expected: character varying, got: boolean.'

tests/admin_pages/test_tests.py

@@ -1167,6 +1167,11 @@ def test_tests_duplicate(pub):
     response.name = 'Response xxx'
     response.store()
 
+    testdef.workflow_tests.actions.append(
+        workflow_tests.AssertWebserviceCall(id='3', webservice_response_uuid=response.uuid),
+    )
+    testdef.store()
+
     app = login(get_app(pub))
 
     assert TestDef.count() == 1
@@ -1196,6 +1201,8 @@ def test_tests_duplicate(pub):
     assert testdef2.workflow_tests.actions[0].button_name == 'Go to end status'
     assert testdef1.get_webservice_responses()[0].name == 'Changed'
     assert testdef2.get_webservice_responses()[0].name == 'Response xxx'
+    assert testdef1.workflow_tests.actions[2].details_label == 'Changed'
+    assert testdef2.workflow_tests.actions[2].details_label == 'Response xxx'
 
     resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
     resp = resp.click('Duplicate')

tests/admin_pages/test_workflow.py

@@ -2103,8 +2103,7 @@ def test_workflows_variables_edit(pub):
     assert resp.location == 'http://example.net/backoffice/workflows/1/variables/fields/'
     resp = resp.follow()
     resp = resp.click('Edit', href='%s/' % workflow.variables_formdef.fields[0].id)
-    assert resp.forms[0]['varname$name'].value == 'foobar'
-    assert 'varname$select' not in resp.forms[0].fields
+    assert resp.forms[0]['varname'].value == 'foobar'
 
     baz_status = workflow.add_status(name='baz')
     baz_status.add_action('displaymsg')
@@ -2112,24 +2111,7 @@ def test_workflows_variables_edit(pub):
 
     resp = app.get('/backoffice/workflows/1/variables/fields/')
     resp = resp.click('Edit', href='%s/' % workflow.variables_formdef.fields[0].id)
-    assert 'varname$select' not in resp.forms[0].fields
-
-    pub.site_options.set('options', 'enable-workflow-variable-parameter', 'true')
-    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
-        pub.site_options.write(fd)
-    resp = app.get('/backoffice/workflows/1/variables/fields/')
-    resp = resp.click('Edit', href='%s/' % workflow.variables_formdef.fields[0].id)
-    assert 'varname$select' in resp.forms[0].fields
-    resp.forms[0]['varname$select'].value = '1*1*message'
-    assert (
-        resp.pyquery('[data-widget-name="default_value"]')[0].attrib['data-dynamic-display-child-of']
-        == 'varname$select'
-    )
-    resp = resp.forms[0].submit('submit')
-
-    workflow = Workflow.get(1)
-    assert workflow.variables_formdef.fields[0].key == 'string'
-    assert workflow.variables_formdef.fields[0].varname == '1*1*message'
+    assert 'varname' in resp.forms[0].fields
 
 
 def test_workflows_variables_default_value(pub):
@@ -2183,20 +2165,6 @@ def test_workflows_variables_edit_with_all_action_types(pub):
     assert resp.location == 'http://example.net/backoffice/workflows/1/variables/fields/'
     resp = resp.follow()
 
-    pub.site_options.set('options', 'enable-workflow-variable-parameter', 'true')
-    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
-        pub.site_options.write(fd)
-    workflow = Workflow.get(1)
-    resp = app.get('/backoffice/workflows/1/variables/fields/')
-    resp = resp.click('Edit', href='%s/' % workflow.variables_formdef.fields[0].id)
-    assert 'varname$select' in resp.forms[0].fields
-    resp.forms[0]['varname$name'].value = 'xxx'
-    resp = resp.forms[0].submit('submit')
-
-    workflow = Workflow.get(1)
-    assert workflow.variables_formdef.fields[0].key == 'string'
-    assert workflow.variables_formdef.fields[0].varname == 'xxx'
-
 
 def test_workflows_variables_delete(pub):
     create_superuser(pub)
@@ -2234,55 +2202,6 @@ def test_workflows_variables_with_export_to_model_action(pub):
     resp = resp.click('Edit', href='%s/' % workflow.variables_formdef.fields[0].id)
 
 
-def test_workflows_variables_replacement(pub):
-    create_superuser(pub)
-    pub.site_options.set('options', 'enable-workflow-variable-parameter', 'true')
-    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
-        pub.site_options.write(fd)
-
-    Workflow.wipe()
-    workflow = Workflow(name='foo')
-    baz_status = workflow.add_status(name='baz')
-    baz_status.add_action('displaymsg', id='1')
-    workflow.store()
-
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/workflows/%s/variables/fields/' % workflow.id)
-
-    # add a field
-    resp.forms[0]['label'] = 'foobar'
-    resp.forms[0]['type'] = 'string'
-    resp = resp.forms[0].submit().follow()
-    workflow = Workflow.get(1)
-
-    # edit
-    resp = resp.click('Edit', href='%s/' % workflow.variables_formdef.fields[0].id)
-    resp.form['varname$select'].value = '1*1*message'
-    resp = resp.form.submit('submit').follow()
-
-    # make sure a wrong variable name is not displayed
-    assert 'form_option_1*1*message' not in resp.text
-    assert Workflow.get(workflow.id).variables_formdef.fields[0].varname == '1*1*message'
-
-    # and make sure it doesn't appear in formdata inspect page
-    formdef = FormDef()
-    formdef.name = 'Form title'
-    formdef.workflow = workflow
-    formdef.fields = []
-    formdef.store()
-
-    data_class = formdef.data_class()
-    data_class.wipe()
-
-    formdata = data_class()
-    formdata.data = {}
-    formdata.status = 'wf-new'
-    formdata.store()
-
-    resp = app.get(formdata.get_backoffice_url() + 'inspect')
-    assert 'form_option_1*1*message' not in resp.text
-
-
 def test_workflows_backoffice_fields(pub):
     create_superuser(pub)
 
@@ -2842,10 +2761,14 @@ def test_workflows_global_actions_timeout_triggers(pub):
     resp = resp.click(
         href='triggers/%s/' % Workflow.get(workflow.id).global_actions[0].triggers[0].id, index=0
     )
-    for invalid_value in ('foobar', '-'):
+    for invalid_value in ('foobar', '-', '0123'):
         resp.form['timeout'] = invalid_value
         resp = resp.form.submit('submit')
         assert 'wrong format' in resp.text
+    for invalid_value in ('833333335', '-833333335'):
+        resp.form['timeout'] = invalid_value
+        resp = resp.form.submit('submit')
+        assert 'invalid value, out of bounds' in resp.text
     resp.form['timeout'] = ''
     resp = resp.form.submit('submit')
     assert 'required field' in resp.text

tests/admin_pages/test_workflow_tests.py

@@ -664,13 +664,13 @@ def test_workflow_tests_action_assert_webservice_call(pub):
     response3.store()
 
     resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
-    assert resp.form['webservice_response_id'].options == [
-        (str(response.id), False, 'Fake response'),
-        (str(response2.id), False, 'Fake response 2'),
+    assert resp.form['webservice_response_uuid'].options == [
+        (str(response.uuid), False, 'Fake response'),
+        (str(response2.uuid), False, 'Fake response 2'),
     ]
     assert resp.form['call_count'].value == '1'
 
-    resp.form['webservice_response_id'] = 1
+    resp.form['webservice_response_uuid'] = response.uuid
     resp.form['call_count'] = 2
     resp = resp.form.submit().follow()
 
@@ -678,7 +678,7 @@ def test_workflow_tests_action_assert_webservice_call(pub):
     assert 'Broken' not in resp.text
 
     assert_webservice_call = TestDef.get(testdef.id).workflow_tests.actions[0]
-    assert assert_webservice_call.webservice_response_id == '1'
+    assert assert_webservice_call.webservice_response_uuid == response.uuid
     assert assert_webservice_call.call_count == 2
 
     response.remove_self()

tests/api/test_formdata.py

@@ -10,6 +10,7 @@ import zipfile
 from contextlib import contextmanager
 
 import pytest
+import responses
 from django.utils.encoding import force_bytes
 from django.utils.timezone import localtime, make_aware
 from quixote import get_publisher
@@ -49,6 +50,12 @@ def pub(emails):
             '''\
 [api-secrets]
 coucou = 1234
+
+[variables]
+idp_api_url = https://authentic.example.invalid/api/'
+
+[wscall-secrets]
+authentic.example.invalid = 4460cf12e156d841c116fbebd52d7ebe41282c63ac2605740068ba5fd89b7316
 '''
         )
 
@@ -2985,9 +2992,12 @@ def test_api_distance_filter(pub, local_user):
     get_app(pub).get(sign_uri('/api/forms/test/list?filter-distance=150000', user=local_user), status=400)
 
 
-@pytest.mark.parametrize('user', ['query-email', 'api-access'])
+@pytest.mark.parametrize('user', ['query-email', 'api-access', 'idp-api-client'])
 @pytest.mark.parametrize('auth', ['signature', 'http-basic'])
+@responses.activate
 def test_api_ods_formdata(pub, local_user, user, auth):
+    ApiAccess.wipe()
+
     pub.role_class.wipe()
     role = pub.role_class(name='test')
     role.store()
@@ -3007,7 +3017,6 @@ def test_api_ods_formdata(pub, local_user, user, auth):
     data_class.wipe()
 
     if user == 'api-access':
-        ApiAccess.wipe()
         access = ApiAccess()
         access.name = 'test'
         access.access_identifier = 'test'
@@ -3025,6 +3034,29 @@ def test_api_ods_formdata(pub, local_user, user, auth):
             def get_url(url, **kwargs):
                 return app.get(sign_uri(url, orig=access.access_identifier, key=access.access_key), **kwargs)
 
+    elif user == 'idp-api-client':
+        if auth == 'signature':
+            pytest.skip('signature authentication requires local user')
+
+        def get_url(url, **kwargs):
+            app.set_authorization(('Basic', ('test', '12345')))
+            return app.get(url, **kwargs)
+
+        responses.post(
+            'https://authentic.example.invalid/api/check-api-client/',
+            json={
+                'err': 0,
+                'data': {
+                    'is_active': True,
+                    'is_anonymous': False,
+                    'is_authenticated': True,
+                    'is_superuser': False,
+                    'restrict_to_anonymised_data': False,
+                    'roles': [],
+                },
+            },
+        )
+
     else:
         if auth == 'http-basic':
             pytest.skip('http basic authentication requires ApiAccess')
@@ -3053,6 +3085,21 @@ def test_api_ods_formdata(pub, local_user, user, auth):
     if user == 'api-access':
         access.roles = [role]
         access.store()
+    elif user == 'idp-api-client':
+        responses.post(
+            'https://authentic.example.invalid/api/check-api-client/',
+            json={
+                'err': 0,
+                'data': {
+                    'is_active': True,
+                    'is_anonymous': False,
+                    'is_authenticated': True,
+                    'is_superuser': False,
+                    'restrict_to_anonymised_data': False,
+                    'roles': [role.id],
+                },
+            },
+        )
     else:
         local_user.roles = [role.id]
         local_user.store()
@@ -3081,6 +3128,14 @@ def test_api_ods_formdata(pub, local_user, user, auth):
     formdef.store()
     get_url('/api/forms/test/ods', status=200)
 
+    if user == 'idp-api-client':
+        # check a single api access object has been created
+        assert ApiAccess.count() == 1
+        api_access = ApiAccess.select()[0]
+        assert api_access.idp_api_client
+        assert api_access.access_identifier == '_idp_test'
+        assert api_access.access_key is None
+
 
 def test_api_global_geojson(pub, local_user):
     pub.role_class.wipe()

tests/backoffice_pages/test_all.py

@@ -2184,6 +2184,7 @@ def test_backoffice_download_as_zip(pub):
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
     assert 'Download all files as .zip' not in resp
+    formdef.management_sidebar_items = formdef.get_default_management_sidebar_items()
     formdef.management_sidebar_items.add('download-files')
     formdef.store()
     resp = app.get('/backoffice/management/form-title/%s/' % number31.id)

tests/backoffice_pages/test_carddata.py

@@ -1867,12 +1867,14 @@ def test_carddata_add_edit_related(pub):
     childdata = child.data_class().select()[0]
     assert len(childdata.get_workflow_traces()) == 1
 
+    AfterJob.wipe()
     resp = app.get('/backoffice/data/child/%s/wfedit-_editable?_popup=1' % childdata.id)
     assert resp.form['f1'].value == 'foo'
     assert resp.form['f2'].value == 'bar'
     resp.form['f1'] = 'foo2'
     resp.form['f2'] = 'bar2'
     resp = resp.form.submit('submit')
+    assert AfterJob.count() == 1  # check a single job has been created to update relations
     childdata.refresh_from_storage()
     assert len(childdata.get_workflow_traces()) == 2
 
@@ -2128,3 +2130,28 @@ def test_carddata_edit_items_display(pub):
     assert resp.status_int == 302
     resp = resp.follow()
     assert not resp.pyquery('#sect-dataview').text()
+
+
+def test_carddata_history_pane_default_mode(pub):
+    CardDef.wipe()
+    user = create_user(pub)
+
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = []
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.store()
+    carddef.data_class().wipe()
+
+    carddata = carddef.data_class()()
+    carddata.just_created()
+    carddata.store()
+
+    app = login(get_app(pub))
+    resp = app.get(carddata.get_backoffice_url())
+    assert resp.pyquery('#evolution-log.folded')
+
+    carddef.history_pane_default_mode = 'expanded'
+    carddef.store()
+    resp = app.get(carddata.get_backoffice_url())
+    assert resp.pyquery('#evolution-log:not(.folded)')

tests/form_pages/test_block.py

@@ -77,6 +77,45 @@ def test_block_simple(pub):
     assert '>bar<' in resp
 
 
+def test_block_a11y(pub):
+    FormDef.wipe()
+    BlockDef.wipe()
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [
+        fields.StringField(id='123', label='Test'),
+        fields.StringField(id='234', label='Test2'),
+    ]
+    block.store()
+
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [
+        fields.BlockField(id='1', label='test', block_slug='foobar'),
+    ]
+    formdef.store()
+
+    app = get_app(pub)
+    resp = app.get(formdef.get_url())
+    assert resp.pyquery('.BlockWidget')[0].attrib.get('role') == 'group'
+    assert resp.pyquery('.BlockWidget')[0].attrib.get('aria-labelledby')
+    assert resp.pyquery('#' + resp.pyquery('.BlockWidget')[0].attrib.get('aria-labelledby'))
+
+    formdef.fields[0].label_display = 'subtitle'
+    formdef.store()
+    resp = app.get(formdef.get_url())
+    assert resp.pyquery('.BlockWidget')[0].attrib.get('role')
+    assert resp.pyquery('.BlockWidget')[0].attrib.get('aria-labelledby')
+    assert resp.pyquery('#' + resp.pyquery('.BlockWidget')[0].attrib.get('aria-labelledby'))
+
+    formdef.fields[0].label_display = 'hidden'
+    formdef.store()
+    resp = app.get(formdef.get_url())
+    assert not resp.pyquery('.BlockWidget')[0].attrib.get('role')
+    assert not resp.pyquery('.BlockWidget')[0].attrib.get('aria-labelledby')
+
+
 def test_block_required(pub):
     FormDef.wipe()
     BlockDef.wipe()

tests/form_pages/test_draft.py

@@ -376,6 +376,42 @@ def test_form_recall_draft(pub):
     assert 'href="%s/"' % draft2.id in resp.text
 
 
+def test_form_recall_draft_digests(pub):
+    user = create_user(pub)
+
+    formdef = create_formdef()
+    formdef.fields = [fields.StringField(id='0', label='string', varname='name')]
+    formdef.digest_templates = {'default': 'digest{{form_var_name}}digest'}
+    formdef.store()
+    formdef.data_class().wipe()
+
+    draft = formdef.data_class()()
+    draft.user_id = user.id
+    draft.status = 'draft'
+    draft.data = {'0': 'DIGEST'}
+    draft.store()
+
+    app = login(get_app(pub), username='foo', password='foo')
+    resp = app.get('/test/')
+    # single draft, digest is not displayed
+    assert 'digestDIGESTdigest' not in resp.pyquery(f'[href="{draft.id}/"]').text()
+
+    draft2 = formdef.data_class()()
+    draft2.user_id = user.id
+    draft2.status = 'draft'
+    draft2.data = {}
+    draft2.store()
+
+    resp = app.get('/test/')
+    # two drafts, the first one has its digest displayed
+    assert 'digestDIGESTdigest' in resp.pyquery(f'[href="{draft.id}/"]').text()
+    # the second doesn't have it as it contains "None"
+    assert (
+        resp.pyquery(f'[href="{draft2.id}/"]').text()
+        and draft2.default_digest not in resp.pyquery(f'[href="{draft2.id}/"]').text()
+    )
+
+
 def test_form_max_drafts(pub):
     user = create_user(pub)
 

tests/form_pages/test_file_field.py

@@ -11,6 +11,7 @@ from wcs.blocks import BlockDef
 from wcs.categories import Category
 from wcs.formdef import FormDef
 from wcs.qommon.errors import ConnectionError
+from wcs.wscalls import NamedWsCall
 
 from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
 from .test_all import create_user
@@ -463,9 +464,9 @@ def test_form_file_field_with_wrong_value(pub):
     assert pub.loggederror_class.count() == 1
     logged_error = pub.loggederror_class.select()[0]
     assert logged_error.formdef_id == formdef.id
-    assert logged_error.summary == 'Failed to set value on field "file"'
-    assert logged_error.exception_class == 'AttributeError'
-    assert logged_error.exception_message == "'str' object has no attribute 'time'"
+    assert logged_error.summary == 'Failed to convert value for field "file"'
+    assert logged_error.exception_class == 'ValueError'
+    assert logged_error.exception_message == "invalid data for file type ('foo bar wrong value')"
 
 
 def test_form_file_field_prefill(pub):
@@ -491,6 +492,72 @@ def test_form_file_field_prefill(pub):
     assert formdata.data['0'].get_content().startswith(b'\x89PNG')
 
 
+@responses.activate
+def test_form_file_field_dict_prefill(pub):
+    NamedWsCall.wipe()
+    wscall = NamedWsCall()
+    wscall.name = 'Hello'
+    wscall.request = {'url': 'http://example.net'}
+    wscall.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.FileField(
+            id='0',
+            label='file',
+            prefill={'type': 'string', 'value': '{{ webservice.hello }}'},
+        )
+    ]
+    formdef.store()
+
+    responses.get(
+        'http://example.net',
+        json={'b64_content': 'aGVsbG8K', 'filename': 'hello.txt', 'content_type': 'text/plain'},
+    )
+    resp = get_app(pub).get('/test/')
+    assert resp.form['f0$token']
+    assert resp.click('hello.txt').content_type == 'text/plain'
+    resp = resp.form.submit('submit')  # -> validation
+    resp = resp.form.submit('submit')  # -> submit
+    formdata = formdef.data_class().select()[0]
+    assert formdata.data['0'].base_filename == 'hello.txt'
+    assert formdata.data['0'].get_content() == b'hello\n'
+
+
+@responses.activate
+def test_form_file_field_url_prefill(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.FileField(
+            id='0',
+            label='file',
+            prefill={'type': 'string', 'value': 'http://example.net/hello.txt'},
+        )
+    ]
+    formdef.store()
+
+    responses.get('http://example.net/hello.txt', body=b'Hello\n', content_type='text/plain')
+    resp = get_app(pub).get('/test/')
+    assert resp.form['f0$token'].value
+    assert resp.click('hello.txt').content_type == 'text/plain'
+    resp = resp.form.submit('submit')  # -> validation
+    resp = resp.form.submit('submit')  # -> submit
+    formdata = formdef.data_class().select()[0]
+    assert formdata.data['0'].base_filename == 'hello.txt'
+    assert formdata.data['0'].get_content() == b'Hello\n'
+
+    pub.loggederror_class.wipe()
+    responses.get('http://example.net/hello.txt', status=404)
+    resp = get_app(pub).get('/test/')
+    assert not resp.form['f0$token'].value
+    assert 'hello.txt' not in resp.text
+    assert [x.summary for x in pub.loggederror_class.select()] == ['Failed to convert value for field "file"']
+
+
 SVG_CONTENT = b'''<?xml version="1.0" encoding="utf-8"?>
 <svg version="1.1" id="Calque_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
 viewBox="0 0 63.72 64.25" style="enable-background:new 0 0 63.72 64.25;" xml:space="preserve"> <g> </g> </svg>'''
@@ -592,3 +659,23 @@ def test_file_download_url_on_wrong_field(pub):
     resp = resp.form.submit('submit').follow()  # -> submit
     formdata = formdef.data_class().select()[0]
     app.get(formdata.get_url() + 'files/1/', status=404)
+
+
+def test_file_auto_convert_heic(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [fields.FileField(id='0', label='field label')]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    with open(os.path.join(os.path.dirname(__file__), '..', 'image.heic'), 'rb') as fd:
+        upload = Upload('image.heic', fd.read(), 'image/heic')
+
+    resp = get_app(pub).get('/test/')
+    resp.forms[0]['f0$file'] = upload
+    resp = resp.forms[0].submit('submit')  # -> validation
+    resp = resp.forms[0].submit('submit')  # -> submit
+    resp = resp.follow()
+    assert resp.click('image.jpeg').follow().content_type == 'image/jpeg'
+    assert b'JFIF' in resp.click('image.jpeg').follow().body

tests/test_formdata.py

@@ -2241,10 +2241,15 @@ def test_lazy_global_forms(pub):
     )
     assert tmpl.render(context) == '7,8,9,10,'
 
+    pub.loggederror_class.wipe()
     tmpl = Template('{{forms|objects:"foobarlazy"|with_custom_view:"private-form-view"|count}}')
     assert tmpl.render(context) == '0'
+    assert [x.summary for x in pub.loggederror_class.select()] == ['Unknown custom view "private-form-view"']
+
+    pub.loggederror_class.wipe()
     tmpl = Template('{{forms|objects:"foobarlazy"|with_custom_view:"unknown"|count}}')
     assert tmpl.render(context) == '0'
+    assert [x.summary for x in pub.loggederror_class.select()] == ['Unknown custom view "unknown"']
 
     custom_view4 = pub.custom_view_class()
     custom_view4.title = 'unknown filter'
@@ -2253,6 +2258,8 @@ def test_lazy_global_forms(pub):
     custom_view4.filters = {'filter-42': 'on', 'filter-42-value': 'foo', 'filter-foobar': 'baz'}
     custom_view4.visibility = 'any'
     custom_view4.store()
+
+    pub.loggederror_class.wipe()
     tmpl = Template('{{forms|objects:"foobarlazy"|with_custom_view:"unknown-filter"|count}}')
     assert tmpl.render(context) == '0'
     assert pub.loggederror_class.count() == 2
@@ -4705,6 +4712,7 @@ def test_formdata_filtering_on_block_fields(pub):
         fields.DateField(id='4', label='Date', varname='date'),
         fields.EmailField(id='5', label='Email', varname='email'),
         fields.TextField(id='6', label='Text', varname='text'),
+        fields.FileField(id='7', label='File', varname='file'),
     ]
     block.store()
 
@@ -4719,6 +4727,10 @@ def test_formdata_filtering_on_block_fields(pub):
     data_class = formdef.data_class()
     data_class.wipe()
 
+    upload = PicklableUpload('test.jpeg', 'image/jpeg')
+    with open(os.path.join(os.path.dirname(__file__), 'image-with-gps-data.jpeg'), 'rb') as fd:
+        upload.receive([fd.read()])
+
     for i in range(14):
         formdata = data_class()
         formdata.data = {
@@ -5048,6 +5060,10 @@ def test_formdata_filtering_on_block_fields(pub):
             tmpl = Template('{{forms|objects:"test"|filter_by:"blockdata_text"|%s|count}}' % operator)
         assert tmpl.render(context) == result
 
+    # file
+    tmpl = Template('{{forms|objects:"test"|filter_by:"blockdata_file"|absent|count}}')
+    assert tmpl.render(context) == '0'
+
 
 def test_items_field_getlist(pub):
     NamedDataSource.wipe()

tests/test_formdef.py

@@ -647,7 +647,7 @@ def test_wipe_on_object(pub):
         formdef.wipe()
 
 
-def test_update_storage_all_formdefs(pub):
+def test_update_storage_all_formdefs(pub, capfd):
     CardDef.wipe()
     FormDef.wipe()
 
@@ -664,6 +664,18 @@ def test_update_storage_all_formdefs(pub):
         update_storage_all_formdefs(pub)
         assert update_storage.call_count == 10
 
+    assert not capfd.readouterr().out
+
+    formdef = FormDef()
+    formdef.name = 'broken formdef'
+    formdef.fields = [StringField(id='1', label='Test')]
+    formdef.store()
+    formdef.fields = [DateField(id='1', label='Test')]
+    formdef.store()
+
+    update_storage_all_formdefs(pub)
+    assert capfd.readouterr().out == '! Integrity errors in %s\n' % formdef.get_admin_url()
+
 
 def test_lazy_formdef(pub):
     FormDef.wipe()

tests/test_misc.py

@@ -20,7 +20,7 @@ from wcs.fields import StringField
 from wcs.qommon import evalutils, force_str
 from wcs.qommon.form import FileSizeWidget
 from wcs.qommon.http_request import HTTPRequest
-from wcs.qommon.humantime import humanduration2seconds, seconds2humanduration
+from wcs.qommon.humantime import humanduration2seconds, seconds2humanduration, timewords
 from wcs.qommon.misc import (
     _http_request,
     date_format,
@@ -108,6 +108,10 @@ def test_humantime_short(seconds, expected):
     assert seconds2humanduration(seconds, short=True) == expected
 
 
+def test_humantime_timewords():
+    assert timewords() == ['day(s)', 'hour(s)', 'minute(s)', 'second(s)', 'month(s)', 'year(s)']
+
+
 def test_parse_mimetypes():
     assert FileTypesDirectory.parse_mimetypes('application/pdf') == ['application/pdf']
     assert FileTypesDirectory.parse_mimetypes('.pdf') == ['application/pdf']

tests/test_publisher.py

@@ -26,7 +26,7 @@ from wcs.qommon.afterjobs import AfterJob
 from wcs.qommon.cron import CronJob
 from wcs.qommon.form import UploadedFile
 from wcs.qommon.http_request import HTTPRequest
-from wcs.qommon.publisher import Tenant
+from wcs.qommon.publisher import MaxSizeDict, Tenant
 from wcs.workflows import Workflow
 
 from .utilities import clean_temporary_pub, create_temporary_pub
@@ -526,6 +526,32 @@ def test_cron_command_rewind_jobs(settings, freezer):
         assert sorted(jobs) == ['job1', 'job2', 'job3']
 
 
+def test_cron_command_job_exception(settings):
+    create_temporary_pub()
+
+    def job1(pub, job=None):
+        raise Exception('Error')
+
+    @classmethod
+    def register_test_cronjobs(cls):
+        cls.register_cronjob(CronJob(job1, name='job1', days=[10]))
+
+    get_publisher().set_tenant_by_hostname('example.net')
+    sql.mark_cron_status('done')
+
+    with mock.patch('wcs.publisher.WcsPublisher.register_cronjobs', register_test_cronjobs):
+        get_publisher_class().cronjobs = []
+        clear_log_files()
+        call_command('cron', job_name='job1', domain='example.net')
+        assert get_logs('example.net') == [
+            'start',
+            "running jobs: ['job1']",
+            'exception running job job1: Error',
+        ]
+
+    clean_temporary_pub()
+
+
 def test_clean_afterjobs():
     pub = create_temporary_pub()
 
@@ -736,3 +762,17 @@ def test_get_site_language():
 
     req.environ['HTTP_ACCEPT_LANGUAGE'] = 'xy,fr,en;q=0.7,es;q=0.3'
     assert pub.get_site_language() == 'fr'
+
+
+def test_maxsize_dict():
+    d = MaxSizeDict()
+    with pytest.raises(KeyError):
+        d['a']  # noqa pylint: disable=pointless-statement
+    for i in range(256):
+        d[str(i)] = f'i : {i}'
+        try:
+            assert d['10']  # keep accessing low value
+        except KeyError:
+            pass
+    # kept keys are the recently added one + '10' that we kept accessing
+    assert set(d.keys()) == set(['10'] + [str(x) for x in range(129, 256)])

tests/test_sql.py

@@ -17,6 +17,7 @@ import wcs.sql_criterias as st
 from wcs import fields, sql
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
+from wcs.categories import Category
 from wcs.data_sources import NamedDataSource
 from wcs.formdata import Evolution
 from wcs.formdef import FormDef
@@ -1576,6 +1577,51 @@ def test_all_forms_user_name_change(pub, formdef):
     conn.commit()
 
 
+def test_all_forms_category_change(pub, formdef):
+    Category.wipe()
+    FormDef.wipe()
+
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.store()
+
+    conn, cur = sql.get_connection_and_cursor()
+    cur.execute('SELECT category_id FROM wcs_all_forms WHERE formdef_id = %s', (formdef.id,))
+    row = cur.fetchone()
+    assert row[0] is None
+
+    category = Category()
+    category.name = 'Test'
+    category.store()
+
+    formdef.category_id = category.id
+    formdef.store()
+    cur.execute('SELECT category_id FROM wcs_all_forms WHERE formdef_id = %s', (formdef.id,))
+    row = cur.fetchone()
+    assert row[0] == int(category.id)
+
+    category2 = Category()
+    category2.name = 'Test2'
+    category2.store()
+    formdef.category_id = category2.id
+    formdef.store()
+    cur.execute('SELECT category_id FROM wcs_all_forms WHERE formdef_id = %s', (formdef.id,))
+    row = cur.fetchone()
+    assert row[0] == int(category2.id)
+
+    formdef.category_id = None
+    formdef.store()
+    cur.execute('SELECT category_id FROM wcs_all_forms WHERE formdef_id = %s', (formdef.id,))
+    row = cur.fetchone()
+    assert row[0] is None
+
+    cur.close()
+    conn.commit()
+
+
 def test_views_fts(pub):
     drop_formdef_tables()
     _, cur = sql.get_connection_and_cursor()
@@ -2375,7 +2421,7 @@ def test_migration_59_all_forms_table(pub):
     formdata.store()
 
     conn, cur = sql.get_connection_and_cursor()
-    cur.execute('DROP TABLE wcs_all_forms')
+    cur.execute('DROP TABLE wcs_all_forms CASCADE')
     cur.execute(
         'DROP TRIGGER %s ON %s' % (sql.get_formdef_trigger_name(formdef), sql.get_formdef_table_name(formdef))
     )
@@ -2937,3 +2983,20 @@ def test_sql_data_views(pub_with_views, formdef_class):
     assert column_exists_in_table(cur, f'{prefix}_test', 'geoloc_base_x')
     conn.commit()
     cur.close()
+
+
+def test_sql_integrity_errors(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.StringField(id='1', label='string'),
+    ]
+    formdef.store()
+    assert not formdef.sql_integrity_errors
+
+    formdef.fields = [
+        fields.FileField(id='1', label='string'),
+    ]
+    formdef.store()
+    assert formdef.sql_integrity_errors == {'1': {'got': 'character varying', 'expected': 'bytea'}}

tests/test_workflow_tests.py

@@ -1092,7 +1092,7 @@ def test_workflow_tests_webservice(pub):
 
     testdef.workflow_tests.actions = [
         workflow_tests.AssertStatus(status_name='End status'),
-        workflow_tests.AssertWebserviceCall(webservice_response_id=response.id, call_count=1),
+        workflow_tests.AssertWebserviceCall(webservice_response_uuid=response.uuid, call_count=1),
     ]
 
     with pytest.raises(WorkflowTestError) as excinfo:
@@ -1107,7 +1107,7 @@ def test_workflow_tests_webservice(pub):
     assert str(excinfo.value) == 'Webservice response Fake response was used 2 times (instead of 1).'
 
     testdef.workflow_tests.actions = [
-        workflow_tests.AssertWebserviceCall(webservice_response_id=response.id, call_count=2),
+        workflow_tests.AssertWebserviceCall(webservice_response_uuid=response.uuid, call_count=2),
     ]
 
     testdef.run(formdef)
@@ -1124,8 +1124,8 @@ def test_workflow_tests_webservice(pub):
     response2.store()
 
     testdef.workflow_tests.actions = [
-        workflow_tests.AssertWebserviceCall(webservice_response_id=response.id, call_count=1),
-        workflow_tests.AssertWebserviceCall(webservice_response_id=response2.id, call_count=1),
+        workflow_tests.AssertWebserviceCall(webservice_response_uuid=response.uuid, call_count=1),
+        workflow_tests.AssertWebserviceCall(webservice_response_uuid=response2.uuid, call_count=1),
     ]
 
     testdef.run(formdef)
@@ -1135,8 +1135,8 @@ def test_workflow_tests_webservice(pub):
     testdef.run(formdef)
 
     testdef.workflow_tests.actions = [
-        workflow_tests.AssertWebserviceCall(webservice_response_id=response.id, call_count=1),
-        workflow_tests.AssertWebserviceCall(webservice_response_id=response.id, call_count=1),
+        workflow_tests.AssertWebserviceCall(webservice_response_uuid=response.uuid, call_count=1),
+        workflow_tests.AssertWebserviceCall(webservice_response_uuid=response.uuid, call_count=1),
     ]
 
     with pytest.raises(WorkflowTestError) as excinfo:
@@ -1144,7 +1144,7 @@ def test_workflow_tests_webservice(pub):
     assert str(excinfo.value) == 'Webservice response Fake response was used 0 times (instead of 1).'
 
     testdef.workflow_tests.actions = [
-        workflow_tests.AssertWebserviceCall(webservice_response_id=response.id, call_count=0),
+        workflow_tests.AssertWebserviceCall(webservice_response_uuid=response.uuid, call_count=0),
     ]
 
     with pytest.raises(WorkflowTestError) as excinfo:
@@ -1152,7 +1152,7 @@ def test_workflow_tests_webservice(pub):
     assert str(excinfo.value) == 'Webservice response Fake response was used 1 times (instead of 0).'
 
     testdef.workflow_tests.actions = [
-        workflow_tests.AssertWebserviceCall(webservice_response_id='xxx', call_count=1),
+        workflow_tests.AssertWebserviceCall(webservice_response_uuid='xxx', call_count=1),
     ]
 
     with pytest.raises(WorkflowTestError) as excinfo:

tests/workflow/test_register_comment.py

@@ -607,3 +607,22 @@ def test_register_comment_to_with_attachment(pub):
     assert 'to-role.txt' in display_parts()[2]
     assert 'to-submitter.txt' in display_parts()[4]
     assert 'to-role-or-submitter.txt' in display_parts()[6]
+
+
+def test_register_comment_fts(pub):
+    pub.substitutions.feed(MockSubstitutionVariables())
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    item = RegisterCommenterWorkflowStatusItem()
+    item.comment = 'Hello\x00\nworld'
+    item.perform(formdata)
+    assert formdata.evolution[-1].parts[-1].content == '<p>Hello\x00\nworld</p>'  # kept
+    assert formdata.evolution[-1].parts[-1].render_for_fts() == 'Hello  world'  # not kept

tox.ini

@@ -23,6 +23,7 @@ deps =
     pytest-cov
     pytest-django
     pytest-xdist
+    pytest-freezegun
     WebTest
     mechanize
     pyquery
@@ -52,6 +53,7 @@ deps =
     pytest-mock
     pytest-cov
     pytest-django
+    pytest-freezegun
     WebTest
     mechanize
     pyquery

wcs/admin/api_access.py

@@ -201,7 +201,7 @@ class ApiAccessDirectory(Directory):
             templates=['wcs/backoffice/api_accesses.html'],
             context={
                 'view': self,
-                'api_accesses': ApiAccess.select(order_by='name'),
+                'api_accesses': [x for x in ApiAccess.select(order_by='name') if not x.idp_api_client],
                 'api_manage_url': api_manage_url,
             },
         )

wcs/admin/fields.py

@@ -387,6 +387,18 @@ class FieldsDirectory(Directory):
                 r += htmltext(' ')
                 r += htmltext(_('It is close to the system limits and no new fields should be added.'))
                 r += htmltext('</div>')
+            elif (
+                hasattr(self.objectdef, 'get_total_count_data_fields')
+                and self.objectdef.get_total_count_data_fields() > 2000
+            ):
+                # warn before DATA_UPLOAD_MAX_NUMBER_FIELDS
+                r += htmltext('<div class="warningnotice">')
+                r += htmltext('<p>%s %s</p>') % (
+                    _('There are at least %d data fields, including fields in blocks.')
+                    % self.objectdef.get_total_count_data_fields(),
+                    _('It is close to the system limits and no new fields should be added.'),
+                )
+                r += htmltext('</div>')
 
             if [x for x in self.objectdef.fields if x.key == 'page']:
                 if self.objectdef.fields[0].key != 'page':

wcs/admin/forms.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+import datetime
 import difflib
 import io
 import xml.etree.ElementTree as ET
@@ -28,6 +29,7 @@ from wcs.backoffice.deprecations import DeprecationsDirectory
 from wcs.backoffice.snapshots import SnapshotsDirectory
 from wcs.carddef import CardDef
 from wcs.categories import Category
+from wcs.fields import PageField
 from wcs.formdef import (
     DRAFTS_DEFAULT_LIFESPAN,
     DRAFTS_DEFAULT_MAX_PER_USER,
@@ -60,7 +62,7 @@ from wcs.qommon.form import (
 )
 from wcs.qommon.misc import localstrftime
 from wcs.roles import get_user_roles, logged_users_role
-from wcs.sql_criterias import Equal, Null, StrictNotEqual
+from wcs.sql_criterias import Equal, GreaterOrEqual, Null, StrictNotEqual
 from wcs.workflows import Workflow
 
 from . import utils
@@ -516,6 +518,7 @@ class OptionsDirectory(Directory):
                 'drafts_max_per_user',
                 'user_support',
                 'management_sidebar_items',
+                'history_pane_default_mode',
             ]
             for attr in attrs:
                 widget = form.get_widget(attr)
@@ -526,8 +529,8 @@ class OptionsDirectory(Directory):
                             continue
                     new_value = widget.parse()
                     if attr == 'management_sidebar_items':
-                        new_value = set(new_value)
-                        if new_value == self.formdef.__class__.management_sidebar_items:
+                        new_value = set(new_value or [])
+                        if new_value == self.formdef.get_default_management_sidebar_items():
                             new_value = {'__default__'}
                     if attr == 'digest_template':
                         if self.formdef.default_digest_template != new_value:
@@ -630,7 +633,6 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
         'enable',
         'workflow',
         'role',
-        ('workflow-options', 'workflow_options'),
         ('workflow-variables', 'workflow_variables'),
         ('workflow-status-remapping', 'workflow_status_remapping'),
         'roles',
@@ -805,7 +807,8 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
                 _('Custom')
                 if (
                     self.formdef.skip_from_360_view
-                    or self.formdef.management_sidebar_items != {'__default__'}
+                    or self.formdef.management_sidebar_items
+                    not in ({'__default__'}, self.formdef.get_default_management_sidebar_items())
                 )
                 else _('Default'),
             ),
@@ -855,17 +858,8 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
             )
 
         options['workflow_options'] = ''
-        if self.formdef.workflow_id:
-            pristine_workflow = Workflow.get(self.formdef.workflow_id, ignore_errors=True)
-            if pristine_workflow and pristine_workflow.variables_formdef:
-                options['workflow_options'] = self.add_option_line('workflow-variables', _('Options'), '')
-            elif self.formdef.workflow_options and get_publisher().has_site_option(
-                'enable-workflow-variable-parameter'
-            ):
-                # there are no variables defined but there are some values
-                # in workflow_options, this is probably the legacy stuff.
-                if any(x for x in self.formdef.workflow_options if '*' in x):
-                    options['workflow_options'] = self.add_option_line('workflow-options', _('Options'), '')
+        if self.formdef.workflow and self.formdef.workflow.variables_formdef:
+            options['workflow_options'] = self.add_option_line('workflow-variables', _('Options'), '')
 
         options['workflow_roles_list'] = []
         if self.formdef.workflow.roles:
@@ -1701,55 +1695,6 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
         r += form.render()
         return r.getvalue()
 
-    def workflow_options(self):
-        request = get_request()
-        if request.get_method() == 'GET' and request.form.get('file'):
-            value = self.formdef.workflow_options.get(request.form.get('file'))
-            if value:
-                return value.build_response()
-
-        get_response().set_title(title=_('Workflow Options'))
-        form = Form(enctype='multipart/form-data')
-        pristine_workflow = Workflow.get(self.formdef.workflow_id)
-        for status in self.formdef.workflow.possible_status:
-            had_options = False
-            for item in status.items:
-                prefix = '%s*%s*' % (status.id, item.id)
-                pristine_item = pristine_workflow.get_status(status.id).get_item(item.id)
-                parameters = [x for x in item.get_parameters() if not getattr(pristine_item, x)]
-                if not parameters:
-                    continue
-                if not had_options:
-                    form.widgets.append(HtmlWidget('<h3>%s</h3>' % status.name))
-                    had_options = True
-                label = getattr(item, 'label', None) or _(item.description)
-                form.widgets.append(HtmlWidget('<h4>%s</h4>' % label))
-                item.add_parameters_widgets(form, parameters, prefix=prefix, formdef=self.formdef)
-        form.add_submit('submit', _('Submit'))
-        form.add_submit('cancel', _('Cancel'))
-
-        if form.get_widget('cancel').parse():
-            return redirect('.')
-
-        if form.is_submitted() and not form.has_errors():
-            self.workflow_options_submit(form)
-            return redirect('.')
-
-        r = TemplateIO(html=True)
-        r += htmltext('<h2>%s</h2>') % _('Workflow Options')
-        r += form.render()
-        return r.getvalue()
-
-    def workflow_options_submit(self, form):
-        self.formdef.workflow_options = {}
-        for widget in form.get_all_widgets():
-            if widget in form.get_submit_widgets():
-                continue
-            if widget.name.startswith('_'):
-                continue
-            self.formdef.workflow_options[widget.name] = widget.parse()
-        self.formdef.store(comment=_('Change in workflow options'))
-
     def inspect(self):
         get_response().set_title(self.formdef.name)
         get_response().breadcrumb.append(('inspect', _('Inspector')))
@@ -1812,6 +1757,62 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
                 f'{self.formdef.xml_root_node}:{self.formdef.id}'
             )
             context['deprecation_titles'] = deprecations.titles
+
+        temp_drafts = defaultdict(int)
+        for formdata in self.formdef.data_class().select_iterator(
+            clause=[Equal('status', 'draft')], itersize=200
+        ):
+            page_id = formdata.page_id if formdata.page_id is not None else '_unknown'
+            temp_drafts[page_id] += 1
+
+        total_drafts = sum(temp_drafts.values()) if temp_drafts else 0
+        total_formdata = 0
+        drafts = {}
+        special_page_index_mapping = {
+            '_first_page': -1000,  # first
+            '_unknown': 1000,  # last
+            '_confirmation_page': 999,  # second to last
+        }
+        if total_drafts:
+            for page_id, page_index in special_page_index_mapping.items():
+                try:
+                    page_total = temp_drafts.pop(page_id)
+                except KeyError:
+                    page_total = 0
+                drafts[page_id] = {'total': page_total, 'field': None, 'page_index': page_index}
+            for page_id, page_total in temp_drafts.items():
+                for index, field in enumerate(self.formdef.iter_fields(with_backoffice_fields=False)):
+                    if page_id == field.id and isinstance(field, PageField):
+                        drafts[page_id] = {
+                            'total': page_total,
+                            'field': field,
+                            'page_index': index,
+                        }
+                        break
+                else:
+                    drafts['_unknown']['total'] += page_total
+
+            total_formdata = self.formdef.data_class().count(
+                [
+                    GreaterOrEqual(
+                        'receipt_time',
+                        datetime.datetime.now() - datetime.timedelta(days=self.formdef.get_drafts_lifespan()),
+                    )
+                ]
+            )
+            for draft_data in drafts.values():
+                draft_percent = 100 * draft_data['total'] / total_drafts
+                draft_data['percent'] = draft_percent
+                draft_data['percent_str'] = '%.1f' % draft_percent
+                to_formdata_percent = 100 * draft_data['total'] / total_formdata
+                draft_data['to_formdata_percent'] = to_formdata_percent
+                draft_data['to_formdata_percent_str'] = '%.1f' % to_formdata_percent
+
+        context['drafts'] = sorted(drafts.items(), key=lambda x: x[1]['page_index'])
+        context['total_formdata'] = total_formdata
+        context['total_drafts'] = total_drafts
+        context['is_carddef'] = isinstance(self.formdef, CardDef)
+
         return template.QommonTemplateResponse(
             templates=[self.inspect_template_name],
             context=context,

wcs/admin/logged_errors.py

@@ -317,7 +317,12 @@ class LoggedErrorsDirectory(AccessControlled, Directory):
                 if 'others' in form.get_widget('types').parse():
                     criterias.append(Null('formdef_class'))
                 criterias = [Or(criterias)]
-            criterias.append(Less('latest_occurence_timestamp', form.get_widget('latest_occurence').parse()))
+            criterias.append(
+                Less(
+                    'latest_occurence_timestamp',
+                    misc.get_as_datetime(form.get_widget('latest_occurence').parse()),
+                )
+            )
             get_publisher().loggederror_class.wipe(clause=criterias)
             return redirect('.')
 

wcs/admin/workflows.py

@@ -40,7 +40,6 @@ from wcs.qommon.afterjobs import AfterJob
 from wcs.qommon.form import (
     CheckboxWidget,
     ColourWidget,
-    CompositeWidget,
     ComputedExpressionWidget,
     FileWidget,
     Form,
@@ -51,7 +50,6 @@ from wcs.qommon.form import (
     SlugWidget,
     StringWidget,
     UrlWidget,
-    VarnameWidget,
 )
 from wcs.sql_criterias import Equal
 from wcs.workflows import (
@@ -1083,80 +1081,15 @@ class WorkflowStatusDirectory(Directory):
         return r.getvalue()
 
 
-class WorkflowVariableWidget(CompositeWidget):
-    def __init__(self, name, value=None, workflow=None, **kwargs):
-        CompositeWidget.__init__(self, name, **kwargs)
-        if value and '*' in value:
-            varname = None
-        else:
-            varname = value
-        self.add(VarnameWidget, 'name', render_br=False, value=varname)
-        if not get_publisher().has_site_option('enable-workflow-variable-parameter'):
-            return
-        options = []
-        if workflow:
-            excluded_parameters = ['backoffice_info_text']
-            for status in workflow.possible_status:
-                for item in status.items:
-                    prefix = '%s*%s*' % (status.id, item.id)
-                    parameters = [
-                        x
-                        for x in item.get_parameters()
-                        if not getattr(item, x) and x not in excluded_parameters
-                    ]
-                    label = getattr(item, 'label', None) or item.description
-                    for parameter in parameters:
-                        key = prefix + parameter
-                        fake_form = Form()
-                        item.add_parameters_widgets(fake_form, [parameter], orig='variable_widget')
-                        if not fake_form.widgets:
-                            continue
-                        parameter_label = fake_form.widgets[0].title
-                        option_value = '%s / %s / %s' % (status.name, label, parameter_label)
-                        options.append((key, option_value, key))
-        if not options:
-            return
-        options = [('', '---', '')] + options
-        self.widgets.append(
-            HtmlWidget(_('or you can use this field to directly replace a workflow parameter:'))
-        )
-        self.add(
-            SingleSelectWidget,
-            'select',
-            options=options,
-            value=value,
-            hint=_('This takes priority over a variable name'),
-            attrs={'data-dynamic-display-parent': 'true'},
-            render_br=False,
-        )
-
-    def _parse(self, request):
-        super()._parse(request)
-        if self.get('select'):
-            self.value = self.get('select')
-        elif self.get('name'):
-            self.value = self.get('name')
-
-
 class WorkflowVariablesFieldDefPage(FieldDefPage):
     section = 'workflows'
     blacklisted_attributes = ['condition', 'prefill', 'display_locations', 'anonymise']
 
     def form(self):
         form = super().form()
-        form.remove('varname')
-        form.add(
-            WorkflowVariableWidget,
-            'varname',
-            title=_('Variable'),
-            value=self.field.varname,
-            advanced=False,
-            required=True,
-            workflow=self.objectdef.workflow,
-        )
         # add default value widget
         if self.field.key in ('string', 'email', 'text', 'date'):
-            widget = form.add(
+            form.add(
                 self.field.widget_class,
                 'default_value',
                 title=_('Default Value'),
@@ -1167,11 +1100,6 @@ class WorkflowVariablesFieldDefPage(FieldDefPage):
                 ),
                 value=getattr(self.field, 'default_value', None),
             )
-            if get_publisher().has_site_option('enable-workflow-variable-parameter'):
-                widget.attrs = {
-                    'data-dynamic-display-child-of': 'varname$select',
-                    'data-dynamic-display-value': '',
-                }
         return form
 
     def submit(self, form):

wcs/api_access.py

@@ -33,6 +33,7 @@ class ApiAccess(XmlStorableObject):
     access_key = None
     description = None
     restrict_to_anonymised_data = False
+    idp_api_client = False
     _roles = None
     _role_ids = Ellipsis
 
@@ -44,6 +45,7 @@ class ApiAccess(XmlStorableObject):
         ('access_key', 'str'),
         ('restrict_to_anonymised_data', 'bool'),
         ('roles', 'roles'),
+        ('idp_api_client', 'bool'),
     ]
 
     @classmethod
@@ -98,7 +100,7 @@ class ApiAccess(XmlStorableObject):
     @classmethod
     def get_with_credentials(cls, username, password):
         api_access = cls.get_by_identifier(username)
-        if not api_access or api_access.access_key != password:
+        if not api_access or api_access.access_key != password or api_access.idp_api_client:
             api_access = cls.get_from_idp(username, password)
             if not api_access:
                 raise KeyError
@@ -143,11 +145,18 @@ class ApiAccess(XmlStorableObject):
         if data.get('err', 1) != 0:
             return None
 
-        api_access = cls.volatile()
+        # cache api client locally, it is necessary for serialization for afterjobs
+        # in uwsgi spooler.
+        access_identifier = f'_idp_{username}'
+        api_access = cls.get_by_identifier(access_identifier) or cls()
+        api_access.idp_api_client = True
+        api_access.access_identifier = access_identifier
+        role_class = get_publisher().role_class
         try:
             api_access.restrict_to_anonymised_data = data['data']['restrict_to_anonymised_data']
-            api_access._role_ids = data['data']['roles']
+            api_access.roles = [role_class.get(x, ignore_errors=True) for x in data['data']['roles']]
+            api_access.roles = [x for x in api_access.roles if x is not None]
         except KeyError:
             return None
-
+        api_access.store()
         return api_access

wcs/backoffice/cards.py

@@ -31,7 +31,7 @@ from wcs.categories import CardDefCategory
 from wcs.sql_criterias import Null, StrictNotEqual
 
 from ..qommon import _, pgettext_lazy
-from ..qommon.form import ComputedExpressionWidget, StringWidget
+from ..qommon.form import CheckboxesWidget, ComputedExpressionWidget, Form, RadiobuttonsWidget, StringWidget
 
 
 class CardDefUI(FormDefUI):
@@ -71,6 +71,26 @@ class CardDefOptionsDirectory(OptionsDirectory):
         )
         return form
 
+    def management(self):
+        form = Form(enctype='multipart/form-data')
+        form.add(
+            CheckboxesWidget,
+            'management_sidebar_items',
+            title=_('Sidebar elements'),
+            options=[(x[0], x[1], x[0]) for x in self.formdef.get_management_sidebar_available_items()],
+            value=self.formdef.get_management_sidebar_items(),
+            inline=False,
+        )
+        form.add(
+            RadiobuttonsWidget,
+            'history_pane_default_mode',
+            title=_('History pane default mode'),
+            options=[('collapsed', _('Collapsed'), 'collapsed'), ('expanded', _('Expanded'), 'expanded')],
+            value=self.formdef.history_pane_default_mode,
+            extra_css_class='widget-inline-radio',
+        )
+        return self.handle(form, pgettext_lazy('cards', 'Management'))
+
 
 class CardFieldDefPage(FormFieldDefPage):
     section = 'cards'
@@ -140,6 +160,15 @@ class CardDefPage(FormDefPage):
         options['user_support'] = self.add_option_line(
             'options/user_support', _('User support'), user_support_status
         )
+        options['management'] = self.add_option_line(
+            'options/management',
+            pgettext_lazy('cards', 'Management'),
+            _('Custom')
+            if self.formdef.history_pane_default_mode != 'collapsed'
+            or self.formdef.management_sidebar_items
+            not in ({'__default__'}, self.formdef.get_default_management_sidebar_items())
+            else _('Default'),
+        )
         return options
 
     def get_sorted_usage_in_formdefs(self):

wcs/backoffice/data_management.py

@@ -441,9 +441,6 @@ class CardBackOfficeStatusPage(FormBackOfficeStatusPage):
     def should_fold_summary(self, mine, request_user):
         return False
 
-    def should_fold_history(self):
-        return True
-
 
 class ImportFromCsvAfterJob(AfterJob):
     def __init__(self, carddef, data_lines, update_existing_cards, submission_agent_id):

wcs/backoffice/management.py

@@ -4513,11 +4513,10 @@ class MassActionAfterJob(AfterJob):
             # action not found
             return
 
-        if item_ids:
-            oldest_lazy_form = formdef.data_class().get(item_ids[0]).get_as_lazy()
         self.total_count = len(item_ids)
         self.store()
 
+        oldest_lazy_form = None
         publisher = get_publisher()
         for i, formdata_id in enumerate(item_ids):
             # do not load all formdatas at once as they can be modified during the loop
@@ -4525,6 +4524,8 @@ class MassActionAfterJob(AfterJob):
             formdata = formdef.data_class().get(formdata_id, ignore_errors=True)
             if not formdata:
                 continue
+            if oldest_lazy_form is None:
+                oldest_lazy_form = formdata.get_as_lazy()
             publisher.reset_formdata_state()
             publisher.substitutions.feed(user)
             publisher.substitutions.feed(formdef)

wcs/backoffice/submission.py

@@ -504,7 +504,6 @@ class SubmissionDirectory(Directory):
             if redirect_url:
                 return redirect(redirect_url)
 
-        get_response().breadcrumb.append(('submission/', _('Submission')))
         get_response().set_title(_('Submission'))
 
         list_forms = self.get_submittable_formdefs(prefetch=False)
@@ -587,7 +586,7 @@ class SubmissionDirectory(Directory):
 
         if get_request().form.get('ajax') == 'true':
             get_request().ignore_session = True
-            get_response().filter = {'raw': True}
+            get_response().raw = True
             return r.getvalue()
 
         rt = TemplateIO(html=True)

wcs/blocks.py

@@ -498,6 +498,17 @@ class BlockWidget(WidgetList):
             **kwargs,
         )
 
+    @property
+    def a11y_labelledby(self):
+        return bool(self.a11y_role)
+
+    @property
+    def a11y_role(self):
+        # don't mark block as a group if it has no label
+        if self.label_display != 'hidden':
+            return 'group'
+        return None
+
     def set_value(self, value):
         from .fields.block import BlockRowValue
 
@@ -566,7 +577,9 @@ class BlockWidget(WidgetList):
     def render_title(self, title):
         attrs = {'id': 'form_label_%s' % self.get_name_for_id()}
         if not title or self.label_display == 'hidden':
-            return htmltag('span', **attrs) + htmltext('</span>')
+            # add a tag even if there's no label to display as it's used as an anchor point
+            # for links to errors.
+            return htmltag('div', **attrs) + htmltext('</div>')
 
         if self.label_display == 'normal':
             return super().render_title(title)

wcs/carddata.py

@@ -122,11 +122,18 @@ class CardData(FormData):
         return '/api/card-file-by-token/%s' % token.id
 
     def update_related(self):
+        if self.is_draft():
+            return
         if self.formdef.reverse_relations:
             job = UpdateRelationsAfterJob(carddata=self)
             if get_response():
-                job.store()
-                get_response().add_after_job(job)
+                job._update_key = (self._formdef.id, self.id)
+                # do not register/run job if an identical job is already planned
+                if job._update_key not in (
+                    getattr(x, '_update_key', None) for x in get_response().after_jobs or []
+                ):
+                    job.store()
+                    get_response().add_after_job(job)
             else:
                 job.execute()
         self._has_changed_digest = False

wcs/carddef.py

@@ -49,6 +49,7 @@ class CardDef(FormDef):
     item_name_plural = pgettext_lazy('item', 'cards')
 
     confirmation = False
+    history_pane_default_mode = 'collapsed'
 
     # users are not allowed to access carddata where they're submitter.
     user_allowed_to_access_own_data = False
@@ -143,6 +144,10 @@ class CardDef(FormDef):
         self.roles = self.backoffice_submission_roles
         return super().store(comment=comment, *args, **kwargs)
 
+    def update_category_reference(self):
+        # only relevant for formdefs
+        pass
+
     @classmethod
     def get_carddefs_as_data_source(cls):
         carddefs_by_id = {}
@@ -311,6 +316,24 @@ class CardDef(FormDef):
                 return True
         return False
 
+    def get_default_management_sidebar_items(self):
+        management_sidebar_items = {
+            'general',
+            'submission-context',
+            'user',
+            'geolocation',
+            'custom-template',
+        }
+        if not self.user_support:
+            management_sidebar_items.remove('user')
+        return management_sidebar_items
+
+    def get_management_sidebar_available_items(self):
+        excluded_parts = ['pending-forms']
+        if not self.user_support:
+            excluded_parts.append('user')
+        return [x for x in super().get_management_sidebar_available_items() if x[0] not in excluded_parts]
+
 
 def get_cards_graph(category=None, show_orphans=False):
     out = io.StringIO()

wcs/ctl/management/commands/hobo_deploy.py

@@ -427,7 +427,7 @@ class Command(TenantCommand):
 
     def configure_site_options(self, current_service, pub, ignore_timestamp=False):
         # configure site-options.cfg
-        config = configparser.RawConfigParser()
+        config = configparser.ConfigParser(interpolation=None)
         site_options_filepath = os.path.join(pub.app_dir, 'site-options.cfg')
         if os.path.exists(site_options_filepath):
             config.read(site_options_filepath)

wcs/fields/file.py

@@ -16,6 +16,7 @@
 
 import base64
 import os
+import urllib.parse
 import xml.etree.ElementTree as ET
 
 from django.utils.encoding import force_bytes, force_str
@@ -139,6 +140,20 @@ class FileField(WidgetField):
             upload = PicklableUpload(value.filename, value.content_type)
             upload.receive([value.content])
             return upload
+
+        value = misc.unlazy(value)
+        if isinstance(value, str) and urllib.parse.urlparse(value).scheme in ('http', 'https'):
+            try:
+                response, dummy, data, dummy = misc.http_get_page(value, raise_on_http_errors=True)
+            except misc.ConnectionError:
+                pass
+            else:
+                value = {
+                    'filename': os.path.basename(urllib.parse.urlparse(value).path) or _('file.bin'),
+                    'content': data,
+                    'content_type': response.headers.get('content-type'),
+                }
+
         if isinstance(value, dict):
             # if value is a dictionary we expect it to have a content or
             # b64_content key and a filename keys and an optional

wcs/formdef.py

@@ -176,14 +176,7 @@ class FormDef(StorableObject):
     expiration_date = None
     has_captcha = False
     skip_from_360_view = False
-    management_sidebar_items = {
-        'general',
-        'submission-context',
-        'user',
-        'geolocation',
-        'custom-template',
-        'pending-forms',
-    }
+    management_sidebar_items = {'__default__'}
     include_download_all_button = False
     appearance_keywords = None
     digest_templates = None
@@ -195,6 +188,8 @@ class FormDef(StorableObject):
     user_support = None
 
     geolocations = None
+    history_pane_default_mode = 'expanded'
+    sql_integrity_errors = None
 
     # store reverse relations
     reverse_relations = None
@@ -242,7 +237,6 @@ class FormDef(StorableObject):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.fields = []
-        self.management_sidebar_items = {'__default__'}
 
     def __eq__(self, other):
         return bool(
@@ -280,7 +274,7 @@ class FormDef(StorableObject):
                     break
 
         if self.include_download_all_button:  # 2023-12-30
-            self.management_sidebar_items = self.__class__.management_sidebar_items.copy()
+            self.management_sidebar_items = self.get_default_management_sidebar_items()
             self.management_sidebar_items.add('download-files')
             self.include_download_all_button = False
             changed = True
@@ -305,6 +299,16 @@ class FormDef(StorableObject):
                 sql.clean_global_views(conn, cur)
             cur.close()
 
+    def get_default_management_sidebar_items(self):
+        return {
+            'general',
+            'submission-context',
+            'user',
+            'geolocation',
+            'custom-template',
+            'pending-forms',
+        }
+
     def get_management_sidebar_available_items(self):
         return [
             ('general', _('General Information')),
@@ -325,7 +329,7 @@ class FormDef(StorableObject):
 
     def get_management_sidebar_items(self):
         if self.management_sidebar_items == {'__default__'}:
-            return self.__class__.management_sidebar_items
+            return self.get_default_management_sidebar_items()
         return self.management_sidebar_items or []
 
     @property
@@ -474,6 +478,14 @@ class FormDef(StorableObject):
             self.update_storage()
             self.store_related_custom_views()
             self.update_searchable_formdefs_table()
+            self.update_category_reference()
+
+    def update_category_reference(self):
+        if getattr(self, '_onload_category_id', None) != self.category_id:
+            from . import sql
+
+            sql.update_global_view_formdef_category(self)
+            self._onload_category_id = self.category_id
 
     def has_captcha_enabled(self):
         return self.has_captcha and get_publisher().has_site_option('formdef-captcha-option')
@@ -576,6 +588,23 @@ class FormDef(StorableObject):
     def get_all_fields(self):
         return (self.fields or []) + self.workflow.get_backoffice_fields()
 
+    def get_all_fields_dict(self):
+        return {x.id: x for x in self.get_all_fields()}
+
+    def get_total_count_data_fields(self):
+        count = len([x for x in self.fields or [] if not x.is_no_data_field and not x.key == 'block'])
+        for field in self.fields or []:
+            if not field.key == 'block':
+                continue
+            try:
+                count += (
+                    len([x for x in field.block.fields or [] if not x.is_no_data_field])
+                    * field.default_items_count
+                )
+            except KeyError:
+                continue
+        return count
+
     def iter_fields(self, include_block_fields=False, with_backoffice_fields=True, with_no_data_fields=True):
         def _iter_fields(fields, block_field=None):
             for field in fields:
@@ -651,10 +680,9 @@ class FormDef(StorableObject):
 
         if self.workflow_id:
             try:
-                workflow = Workflow.get(self.workflow_id)
+                self._workflow = Workflow.get(self.workflow_id)
             except KeyError:
                 return Workflow.get_unknown_workflow()
-            self._workflow = self.get_workflow_with_options(workflow)
             return self._workflow
         else:
             self._workflow = self.get_default_workflow()
@@ -666,20 +694,6 @@ class FormDef(StorableObject):
 
         return Workflow.get_default_workflow()
 
-    def get_workflow_with_options(self, workflow):
-        # this needs to be kept in sync with admin/forms.ptl,
-        # FormDefPage::workflow
-        if not self.workflow_options:
-            return workflow
-        for status in workflow.possible_status:
-            for item in status.items:
-                prefix = '%s*%s*' % (status.id, item.id)
-                for parameter in item.get_parameters():
-                    value = self.workflow_options.get(prefix + parameter)
-                    if value:
-                        setattr(item, parameter, value)
-        return workflow
-
     def set_workflow(self, workflow):
         if workflow and workflow.id not in ['_carddef_default', '_default']:
             self.workflow_id = workflow.id
@@ -2015,6 +2029,8 @@ class FormDef(StorableObject):
             del odict['_custom_views']
         if '_import_orig_slug' in odict:
             del odict['_import_orig_slug']
+        if '_onload_category_id' in odict:
+            del odict['_onload_category_id']
         return odict
 
     def __setstate__(self, dict):
@@ -2029,6 +2045,7 @@ class FormDef(StorableObject):
     @classmethod
     def storage_load(cls, fd, **kwargs):
         o = super().storage_load(fd)
+        o._onload_category_id = o.category_id  # keep track of category, to update wcs_all_forms if changed
         if kwargs.get('lightweight'):
             o.fields = Ellipsis
             return o
@@ -2306,6 +2323,10 @@ def update_storage_all_formdefs(publisher, **kwargs):
 
     for formdef in itertools.chain(FormDef.select(), CardDef.select()):
         formdef.update_storage()
+        if formdef.sql_integrity_errors:
+            # print errors, this will get them in the cron output, that hopefully
+            # a sysadmin will read.
+            print(f'! Integrity errors in {formdef.get_admin_url()}')
 
 
 def get_formdefs_of_all_kinds(**kwargs):
@@ -2360,7 +2381,7 @@ class UpdateDigestAfterJob(AfterJob):
     def execute(self):
         for formdef_class, formdef_id in self.kwargs['formdefs']:
             formdef = formdef_class.get(formdef_id)
-            for formdata in formdef.data_class().select(order_by='id'):
+            for formdata in formdef.data_class().select_iterator(order_by='id', itersize=200):
                 formdata.store()
 
 

wcs/forms/common.py

@@ -541,7 +541,7 @@ class FormStatusPage(Directory, FormTemplateMixin):
         return False
 
     def should_fold_history(self):
-        return False
+        return bool(self.formdef.history_pane_default_mode == 'collapsed')
 
     def receipt(self, always_include_user=False, form_url='', mine=True):
         request_user = user = get_request().user

wcs/locale/fr/LC_MESSAGES/django.po

@@ -4,8 +4,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: wcs 0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-03-21 19:00+0100\n"
-"PO-Revision-Date: 2024-03-21 19:00+0100\n"
+"POT-Creation-Date: 2024-04-01 18:14+0200\n"
+"PO-Revision-Date: 2024-04-01 18:14+0200\n"
 "Last-Translator: Thomas Noël <tnoel@entrouvert.com>\n"
 "Language-Team: french\n"
 "Language: fr\n"
@@ -232,9 +232,9 @@ msgstr "Dupliquer"
 msgid "Save snapshot"
 msgstr "Enregistrer une sauvegarde"
 
-#: admin/blocks.py admin/forms.py
-msgid "Overwrite"
-msgstr "Écraser"
+#: admin/blocks.py templates/wcs/backoffice/formdef.html
+msgid "Overwrite with new import"
+msgstr "Écraser avec un nouvel import"
 
 #: admin/blocks.py templates/wcs/backoffice/blocks.html
 #: templates/wcs/backoffice/cards.html templates/wcs/backoffice/category.html
@@ -305,6 +305,10 @@ msgstr ""
 msgid "File"
 msgstr "Fichier"
 
+#: admin/blocks.py admin/forms.py
+msgid "Overwrite"
+msgstr "Écraser"
+
 #: admin/blocks.py admin/forms.py
 msgid "Overwritten"
 msgstr "Écrasement"
@@ -1014,6 +1018,12 @@ msgstr ""
 "Il approche les limites du système et de nouveaux champs ne devraient pas "
 "être ajoutés."
 
+#: admin/fields.py
+#, python-format
+msgid "There are at least %d data fields, including fields in blocks."
+msgstr ""
+"Il y a au moins %d champs de données, en comptant les champs dans les blocs."
+
 #: admin/fields.py
 msgid "In a multipage form, the first field should be of type \"page\"."
 msgstr ""
@@ -1220,7 +1230,7 @@ msgstr "Commencer par un CAPTCHA pour les utilisateurs anonymes"
 msgid "CAPTCHA"
 msgstr "CAPTCHA"
 
-#: admin/forms.py
+#: admin/forms.py backoffice/cards.py
 msgid "Sidebar elements"
 msgstr "Contenu de la barre latérale"
 
@@ -1427,11 +1437,11 @@ msgctxt "confirmation page"
 msgid "Disabled"
 msgstr "Désactivée"
 
-#: admin/forms.py
+#: admin/forms.py backoffice/cards.py
 msgid "Custom"
 msgstr "Personnalisé"
 
-#: admin/forms.py workflows.py
+#: admin/forms.py backoffice/cards.py workflows.py
 msgid "Default"
 msgstr "Par défaut"
 
@@ -1810,6 +1820,7 @@ msgid "Text"
 msgstr "Texte"
 
 #: admin/logged_errors.py backoffice/management.py formdata.py
+#: templates/wcs/backoffice/includes/inspect-draft-by-page.html
 #: wf/create_formdata.py workflows.py
 msgid "Unknown"
 msgstr "Inconnu"
@@ -3799,6 +3810,23 @@ msgstr "L’identifiant ne peut pas être modifié car il existe des fiches."
 msgid "Unique identifier template"
 msgstr "Gabarit pour un identifiant unique"
 
+#: backoffice/cards.py
+msgid "History pane default mode"
+msgstr "Affichage par défaut du volet « historique »"
+
+#: backoffice/cards.py
+msgid "Collapsed"
+msgstr "Plié"
+
+#: backoffice/cards.py
+msgid "Expanded"
+msgstr "Déplié"
+
+#: backoffice/cards.py
+msgctxt "cards"
+msgid "Management"
+msgstr "Gestion"
+
 #: backoffice/cards.py
 msgid ""
 "Warning: this field data will be permanently deleted from existing cards."
@@ -5700,6 +5728,10 @@ msgstr ""
 msgid "File storage system"
 msgstr "Système de stockage de fichier"
 
+#: fields/file.py
+msgid "file.bin"
+msgstr "fichier.bin"
+
 #: fields/item.py
 #, python-format
 msgid "unknown card value (%r)"
@@ -7167,6 +7199,11 @@ msgstr "Unités de temps utilisables : %s."
 msgid "too many characters (limit is %d)"
 msgstr "trop de caractères (la limite est à %d)"
 
+#: qommon/form.py
+#, python-format
+msgid "Failed to convert value for field \"%s\""
+msgstr "Erreur à la conversion de la valeur pour le champ « %s «"
+
 #: qommon/form.py
 #, python-format
 msgid "Failed to set value on field \"%s\""
@@ -7180,11 +7217,6 @@ msgstr "erreur système à l’enregistrement du fichier"
 msgid "unknown storage system (system error)"
 msgstr "système de stockage inconnu (erreur système)"
 
-#: qommon/form.py
-#, python-format
-msgid "over file size limit (%s)"
-msgstr "dépasse la taille limite (%s)"
-
 #: qommon/form.py
 msgid "invalid file type"
 msgstr "type de fichier invalide"
@@ -7193,6 +7225,11 @@ msgstr "type de fichier invalide"
 msgid "forbidden file type"
 msgstr "type de fichier interdit"
 
+#: qommon/form.py
+#, python-format
+msgid "over file size limit (%s)"
+msgstr "dépasse la taille limite (%s)"
+
 #: qommon/form.py
 msgid "You should enter a valid email address, for example name@example.com."
 msgstr "Veuillez saisir une adresse électronique, par exemple nom@example.com."
@@ -7573,6 +7610,10 @@ msgstr "jour"
 msgid "days"
 msgstr "jours"
 
+#: qommon/humantime.py
+msgid "day(s)"
+msgstr "jour(s)"
+
 #: qommon/humantime.py
 msgid "hour"
 msgstr "heure"
@@ -7581,6 +7622,34 @@ msgstr "heure"
 msgid "hours"
 msgstr "heures"
 
+#: qommon/humantime.py
+msgid "hour(s)"
+msgstr "heure(s)"
+
+#: qommon/humantime.py
+msgid "minute"
+msgstr "minute"
+
+#: qommon/humantime.py
+msgid "minutes"
+msgstr "minutes"
+
+#: qommon/humantime.py
+msgid "minute(s)"
+msgstr "minute(s)"
+
+#: qommon/humantime.py
+msgid "second"
+msgstr "seconde"
+
+#: qommon/humantime.py
+msgid "seconds"
+msgstr "secondes"
+
+#: qommon/humantime.py
+msgid "second(s)"
+msgstr "seconde(s)"
+
 #: qommon/humantime.py
 msgid "month"
 msgstr "mois"
@@ -7589,6 +7658,10 @@ msgstr "mois"
 msgid "months"
 msgstr "mois"
 
+#: qommon/humantime.py
+msgid "month(s)"
+msgstr "mois"
+
 #: qommon/humantime.py
 msgid "year"
 msgstr "année"
@@ -7598,20 +7671,8 @@ msgid "years"
 msgstr "années"
 
 #: qommon/humantime.py
-msgid "minute"
-msgstr "minute"
-
-#: qommon/humantime.py
-msgid "minutes"
-msgstr "minutes"
-
-#: qommon/humantime.py
-msgid "second"
-msgstr "seconde"
-
-#: qommon/humantime.py
-msgid "seconds"
-msgstr "secondes"
+msgid "year(s)"
+msgstr "année(s)"
 
 #: qommon/humantime.py
 #, python-format
@@ -8931,13 +8992,11 @@ msgstr "Désolé"
 
 #: qommon/publisher.py
 msgid ""
-"Map data &copy; <a href='https://openstreetmap.org'>OpenStreetMap</a> "
-"contributors, <a href='http://creativecommons.org/licenses/by-sa/2.0/'>CC-BY-"
-"SA</a>"
+"Map data &copy; <a href=\"https://www.openstreetmap.org/"
+"copyright\">OpenStreetMap</a>"
 msgstr ""
-"Données &copy; contributeurs <a href='https://openstreetmap."
-"org'>OpenStreetMap</a>, <a href='http://creativecommons.org/licenses/by-"
-"sa/2.0/deed.fr'>CC-BY-SA</a>"
+"Données cartographiques &copy; <a href=\"https://www.openstreetmap.org/"
+"copyright\">OpenStreetMap</a>"
 
 #: qommon/publisher.py
 msgid "Belgian eID"
@@ -9422,6 +9481,10 @@ msgstr "Identifiant d’accès :"
 msgid "Access key:"
 msgstr "Clé d’accès :"
 
+#: templates/wcs/backoffice/api_access.html
+msgid "API client from identity provider, identifier:"
+msgstr "Client d’API du fournisseur d’identité, identifiant :"
+
 #: templates/wcs/backoffice/api_access.html
 msgid "Restricted to anonymised data"
 msgstr "Limité aux données anonymisées"
@@ -9732,6 +9795,10 @@ msgstr ""
 "tel qu’il existe actuellement, pas nécessairement tel qu’il était au moment "
 "de l’exécution."
 
+#: templates/wcs/backoffice/formdef-inspect.html
+msgid "Drafts"
+msgstr "Brouillons"
+
 #: templates/wcs/backoffice/formdef-inspect.html
 msgid "Custom views"
 msgstr "Vues personnalisées"
@@ -9779,6 +9846,22 @@ msgid_plural "%(page_count)s pages"
 msgstr[0] "%(page_count)s page"
 msgstr[1] "%(page_count)s pages"
 
+#: templates/wcs/backoffice/formdef-inspect.html
+msgid "Statistics on drafts by page."
+msgstr "Statistiques de brouillons par page."
+
+#: templates/wcs/backoffice/formdef-inspect.html
+msgid "Rate among drafts"
+msgstr "Taux sur les brouillons"
+
+#: templates/wcs/backoffice/formdef-inspect.html
+msgid "Rate among total forms"
+msgstr "Taux sur l’ensemble des demandes de la période"
+
+#: templates/wcs/backoffice/formdef-inspect.html
+msgid "There are currently no drafts for this form."
+msgstr "Il n’y a actuellement pas de brouillons pour cette démarche."
+
 #: templates/wcs/backoffice/formdef-inspect.html
 msgid "Default view"
 msgstr "Vue par défaut"
@@ -9799,10 +9882,6 @@ msgstr "Afficher le code QR"
 msgid "change title"
 msgstr "changer le titre"
 
-#: templates/wcs/backoffice/formdef.html
-msgid "Overwrite with new import"
-msgstr "Écraser avec un nouvel import"
-
 #: templates/wcs/backoffice/formdef.html
 msgid "Preview Online"
 msgstr "Aperçu en ligne"
@@ -9899,6 +9978,19 @@ msgstr "Publié à partir du %(date1)s"
 msgid "Published until %(date2)s"
 msgstr "Publié jusqu’au %(date2)s"
 
+#: templates/wcs/backoffice/includes/inspect-draft-by-page.html
+msgid "Only page"
+msgstr "Page unique"
+
+#: templates/wcs/backoffice/includes/inspect-draft-by-page.html
+msgid "Confirmation page"
+msgstr "Page de confirmation"
+
+#: templates/wcs/backoffice/includes/inspect-draft-by-page.html
+#, python-format
+msgid "%%"
+msgstr " %%"
+
 #: templates/wcs/backoffice/includes/mail-templates.html
 msgid "There are no mail templates defined."
 msgstr "Il n’y a pas de modèle de courriel défini."
@@ -10783,6 +10875,11 @@ msgstr "Valeur invalide (%r) pour le filtre « order_by »"
 msgid "Invalid operator \"%(operator)s\" for filter \"%(filter)s\""
 msgstr "Opérateur « %(operator)s » invalide pour le filtre « %(filter)s »"
 
+#: variables.py
+#, python-format
+msgid "Unknown custom view \"%(slug)s\""
+msgstr "Vue personnalisée « %(slug)s »"
+
 #: variables.py
 #, python-format
 msgid "invalid value for distance (%r)"
@@ -10892,6 +10989,10 @@ msgstr "Géolocalisation : position non disponible"
 msgid "Geolocation: timeout"
 msgstr "Géolocalisation : délai expiré"
 
+#: views.py
+msgid "Marker of selected position"
+msgstr "Marqueur pointant la position sélectionnée"
+
 #: views.py
 msgid "An error occured while fetching results"
 msgstr "Erreur à la récupération des résultats"
@@ -10912,6 +11013,10 @@ msgstr "Dézoomer"
 msgid "Display my position"
 msgstr "Afficher ma position"
 
+#: views.py
+msgid "Leaflet, a JavaScript library for interactive maps"
+msgstr "Leaflet, une bibliothèque JavaScript pour des cartes interactives"
+
 #: views.py
 msgid "The results could not be loaded"
 msgstr "Les résultats ne peuvent pas être chargés"
@@ -12880,6 +12985,10 @@ msgstr ""
 msgid "Current Status"
 msgstr "Statut actuel"
 
+#: workflows.py
+msgid "invalid value, out of bounds"
+msgstr "valeur choisie invalide, hors des bornes"
+
 #: workflows.py
 msgid "Delay (in days)"
 msgstr "Délai (en jours)"

wcs/qommon/cron.py

@@ -16,6 +16,7 @@
 
 import datetime
 import os
+import sys
 import time
 from contextlib import contextmanager
 
@@ -128,4 +129,5 @@ def cron_worker(publisher, since, job_name=None):
             with job.log_long_job():
                 job.function(publisher, job=job)
         except Exception as e:
-            publisher.record_error(exception=e, context='[CRON]', notify=True)
+            job.log(f'exception running job {job.name}: {e}')
+            publisher.capture_exception(sys.exc_info())

wcs/qommon/form.py

@@ -29,6 +29,7 @@ import mimetypes
 import os
 import random
 import re
+import subprocess
 import sys
 import tempfile
 import time
@@ -924,6 +925,20 @@ class FileWithPreviewWidget(CompositeWidget):
         return False
 
     def set_value(self, value):
+        if isinstance(value, (str, dict)):
+            from wcs.fields.file import FileField
+
+            try:
+                value = FileField.convert_value_from_anything(value)
+            except ValueError as e:
+                value = None
+                if getattr(self, 'field', None):
+                    get_publisher().record_error(
+                        _('Failed to convert value for field "%s"') % self.field.label,
+                        formdef=getattr(self, 'formdef', None),
+                        exception=e,
+                    )
+
         try:
             self.value = value
             if self.value and self.get_value_from_token:
@@ -1047,12 +1062,6 @@ class FileWithPreviewWidget(CompositeWidget):
 
         self.value.content_type = filetype
 
-        if self.max_file_size and hasattr(self.value, 'file_size'):
-            # validate file size
-            if self.value.file_size > self.max_file_size_bytes:
-                self.set_error(_('over file size limit (%s)') % self.max_file_size)
-                return
-
         if self.file_type:
             # validate file type
             accepted_file_types = []
@@ -1098,6 +1107,40 @@ class FileWithPreviewWidget(CompositeWidget):
         ) or filetype in blacklisted_file_types:
             self.set_error(_('forbidden file type'))
 
+        if self.value.content_type in ('image/heic', 'image/heif') and not get_publisher().has_site_option(
+            'do-no-transform-heic-files'
+        ):
+            # convert HEIC files to JPEG
+            try:
+                with open(self.value.fp.name, 'rb') as fd:
+                    # libheic will automatically switch image orientation so we need to remove
+                    # EXIF profile to avoid it being applied a second time.
+                    # (graphicsmagick >= 1.3.41 have heif:ignore-transformations=false to avoid
+                    # that).
+                    rc = subprocess.run(
+                        ['gm', 'convert', '+profile', '"*"', 'HEIC:-', 'JPEG:-'],
+                        input=fd.read(),
+                        capture_output=True,
+                        check=True,
+                    )
+                    from wcs.fields.file import FileField
+
+                    self.value = FileField.convert_value_from_anything(
+                        {
+                            'content': rc.stdout,
+                            'filename': os.path.splitext(self.value.base_filename)[0] + '.jpeg',
+                            'content_type': 'image/jpeg',
+                        }
+                    )
+            except subprocess.CalledProcessError:
+                pass
+
+        if self.max_file_size and hasattr(self.value, 'file_size'):
+            # validate file size
+            if self.value.file_size > self.max_file_size_bytes:
+                self.set_error(_('over file size limit (%s)') % self.max_file_size)
+                return
+
 
 class EmailWidget(StringWidget):
     HTML_TYPE = 'email'

wcs/qommon/humantime.py

@@ -34,21 +34,20 @@ def list2human(stringlist):
 
 
 _humandurations = (
-    ((_('day'), _('days')), _day),
-    ((_('hour'), _('hours')), _hour),
-    ((_('month'), _('months')), _month),
-    ((_('year'), _('years')), _year),
-    ((_('minute'), _('minutes')), _minute),
-    ((_('second'), _('seconds')), 1),
+    ((_('day'), _('days'), _('day(s)')), _day),
+    ((_('hour'), _('hours'), _('hour(s)')), _hour),
+    ((_('minute'), _('minutes'), _('minute(s)')), _minute),
+    ((_('second'), _('seconds'), _('second(s)')), 1),
+    ((_('month'), _('months'), _('month(s)')), _month),
+    ((_('year'), _('years'), _('year(s)')), _year),
 )
 
 
 def timewords():
     '''List of words one can use to specify durations'''
     result = []
-    for words, dummy in _humandurations:
-        for word in words:
-            result.append(str(word))  # str() to force translation
+    for (dummy, dummy, word), dummy in _humandurations:
+        result.append(str(word))  # str() to force translation
     return result
 
 
@@ -56,12 +55,11 @@ def humanduration2seconds(humanduration):
     if not humanduration:
         raise ValueError()
     seconds = 0
-    for words, quantity in _humandurations:
-        for word in words:
-            m = re.search(r'(\d+)\s*\b%s\b' % word, humanduration)
-            if m:
-                seconds = seconds + int(m.group(1)) * quantity
-                break
+    for (word1, word2, dummy), quantity in _humandurations:
+        # look for number then singular or plural forms of unit
+        m = re.search(r'(\d+)\s*\b(%s|%s)\b' % (word1, word2), humanduration)
+        if m:
+            seconds = seconds + int(m.group(1)) * quantity
     return seconds
 
 

wcs/qommon/publisher.py

@@ -58,6 +58,25 @@ except ImportError:
     sentry_sdk = None
 
 
+class MaxSizeDict(collections.OrderedDict):
+    # dictionary that will store at most 128 items, least recently used items are removed first.
+
+    def __setitem__(self, key, value):
+        super().__setitem__(key, value)
+        self.move_to_end(key, last=False)
+        if len(self) > 128:
+            self.popitem(last=True)
+
+    def __getitem__(self, key):
+        if key in self:
+            self.move_to_end(key, last=False)
+        return super().__getitem__(key)
+
+    def get(self, key, default=None):
+        # native get() doesn't use __getitem__
+        return self[key] if key in self else default
+
+
 class ImmediateRedirectException(Exception):
     def __init__(self, location):
         self.location = location
@@ -421,7 +440,7 @@ class QommonPublisher(Publisher):
         return string
 
     def load_site_options(self):
-        self.site_options = configparser.ConfigParser()
+        self.site_options = configparser.ConfigParser(interpolation=None)
         site_options_filename = os.path.join(self.app_dir, 'site-options.cfg')
         if not os.path.exists(site_options_filename):
             return
@@ -504,7 +523,7 @@ class QommonPublisher(Publisher):
 
     def reset_caches(self):
         self._cached_user_fields_formdef = None
-        self._cached_objects = collections.defaultdict(dict)
+        self._cached_objects = collections.defaultdict(MaxSizeDict)
 
     def set_app_dir(self, request):
         """
@@ -814,9 +833,7 @@ class QommonPublisher(Publisher):
                 'map-bounds-bottom-right'
             ).split(';')
         attrs['data-map-attribution'] = self.get_site_option('map-attribution') or _(
-            'Map data © '
-            "<a href='https://openstreetmap.org'>OpenStreetMap</a> contributors, "
-            "<a href='http://creativecommons.org/licenses/by-sa/2.0/'>CC-BY-SA</a>"
+            'Map data &copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a>'
         )
         attrs['data-tile-urltemplate'] = (
             self.get_site_option('map-tile-urltemplate') or 'https://tiles.entrouvert.org/hdm/{z}/{x}/{y}.png'

wcs/qommon/static/css/dc2/admin.scss

@@ -1162,6 +1162,7 @@ div.PrefillSelectionWidget div.content input[type=submit] {
 ul#field-filter,
 ul.columns-filter {
 	list-style: none;
+	padding-bottom: 1px;
 	padding-left: 0;
 	margin-left: 0;
 	max-height: calc(100vh - 14em);
@@ -2832,6 +2833,9 @@ div.file-upload-widget {
 		}
 		div.widget-message {
 			padding-top: 20px;
+			p {
+				margin: 0;
+			}
 			&::before {
 				pointer-events: none;
 				content: "\f016";  // file-o

wcs/qommon/static/js/qommon.forms.js

@@ -851,6 +851,8 @@ $(function() {
         data: form_data,
         headers: {'x-wcs-ajax-action': 'block-add-row'},
         success: function(result, text_status, jqXHR) {
+          var new_form_token = $(result).find('input[name="_form_id"]').val()
+          $('input[name="_form_id"]').val(new_form_token)
           const $new_block = $(result).find('[data-field-id="' + block_id + '"]');
           $block.replaceWith($new_block);
           const $new_blockrow = $new_block.find('.BlockSubWidget').last();

wcs/qommon/static/js/qommon.map.js

@@ -26,6 +26,8 @@ $(window).on('wcs:maps-init', function() {
      }
      map_options.gestureHandling = true;
      var map = L.map($(this).attr('id'), map_options);
+     map.attributionControl.setPrefix(
+       '<a href="https://leafletjs.com" title="' + WCS_I18N.map_leaflet_title_attribute + '">Leaflet</a>')
      var map_controls_position = $('body').data('map-controls-position') || 'topleft';
      if (! ($map_widget.parents('#sidebar').length || $map_widget.parents('td').length)) {
        new L.Control.Zoom({
@@ -204,7 +206,7 @@ $(window).on('wcs:maps-init', function() {
 
      $map_widget.on('set-geolocation', function(e, coords, options) {
        if (map.marker === null) {
-         map.marker = L.marker([0, 0]);
+         map.marker = L.marker([0, 0], {alt: WCS_I18N.map_position_marker_alt});
          map.marker.addTo(map);
        }
        map.marker.setLatLng(coords);

wcs/qommon/templates/qommon/forms/widgets/block.html

@@ -1,3 +1,5 @@
 {% extends "qommon/forms/widget.html" %}
 
 {% block widget-css-classes %}{{ block.super }} {% if widget.had_add_clicked %}wcs-block-add-clicked{% endif %} {% if widget.remove_button %}wcs-block-with-remove-button{% endif %}{% endblock %}
+
+{% block widget-attrs %}id="form_{{ widget.field.id }}" {{ block.super }}{% endblock %}

wcs/qommon/templates/qommon/forms/widgets/file.html

@@ -9,10 +9,10 @@
       {{ w.render|safe }}
     {% endfor %}
     <div class="widget-message click-to-upload">
-      {% trans "Drop a file or click to select one" %}
+      <p>{% trans "Drop a file or click to select one" %}</p>
     </div>
     <div class="widget-message upload-done">
-      {% trans "Upload done" %}
+      <p>{% trans "Upload done" %}</p>
     </div>
     <div class="fileprogress" style="display: none;">
       <div class="bar"

wcs/qommon/templates/qommon/forms/widgets/map.html

@@ -2,7 +2,7 @@
 
 {% block widget-control %}
   <input type="hidden" name="{{widget.name}}$latlng" {% if widget.value %}value="{{widget.value}}"{% endif %}>
-  <div id="map-{{widget.get_name_for_id}}" class="qommon-map"
+  <div id="form_{{widget.get_name_for_id}}" class="qommon-map"
        {% if widget.readonly %}data-readonly="true"{% endif %}
        {% if widget.sync_map_and_address_fields %}data-address-sync="true"{% endif %}
        {% for key, value in widget.map_attributes.items %}{{key}}="{{value}}" {% endfor %}

wcs/sql.py

@@ -544,6 +544,7 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
             cur.execute(f'ALTER TABLE {table_name} ALTER COLUMN last_update_time SET DATA TYPE timestamptz')
 
         # add new fields
+        field_integrity_errors = {}
         for field in formdef.get_all_fields():
             assert field.id is not None
             sql_type = SQL_TYPE_MAPPING.get(field.key, 'varchar')
@@ -554,6 +555,16 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
                 cur.execute(
                     '''ALTER TABLE %s ADD COLUMN %s %s''' % (table_name, get_field_id(field), sql_type)
                 )
+            else:
+                existing_type = existing_field_types.get(get_field_id(field))
+                # map to names returned in data_type column
+                expected_type = {
+                    'varchar': 'character varying',
+                    'text[]': 'ARRAY',
+                    'text[][]': 'ARRAY',
+                }.get(sql_type) or sql_type
+                if existing_type != expected_type:
+                    field_integrity_errors[str(field.id)] = {'got': existing_type, 'expected': expected_type}
             if field.store_display_value:
                 needed_fields.add('%s_display' % get_field_id(field))
                 if '%s_display' % get_field_id(field) not in existing_fields:
@@ -569,6 +580,10 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
                         % (table_name, '%s_structured' % get_field_id(field))
                     )
 
+        if (field_integrity_errors or None) != formdef.sql_integrity_errors:
+            formdef.sql_integrity_errors = field_integrity_errors
+            formdef.store(object_only=True)
+
         for field in (formdef.geolocations or {}).keys():
             column_name = 'geoloc_%s' % field
             needed_fields.add(column_name)
@@ -1508,6 +1523,15 @@ def drop_global_views(conn, cur):
         cur.execute('''DROP VIEW IF EXISTS %s''' % view_name)
 
 
+def update_global_view_formdef_category(formdef):
+    _, cur = get_connection_and_cursor()
+    with cur:
+        cur.execute(
+            '''UPDATE wcs_all_forms set category_id = %s WHERE formdef_id = %s''',
+            (formdef.category_id, formdef.id),
+        )
+
+
 def do_global_views(conn, cur):
     # recreate global views
     # XXX TODO: make me dynamic, please ?

wcs/templates/wcs/backoffice/api_access.html

@@ -3,10 +3,12 @@
 {% block body %}
   <div id="appbar">
     <h2>{% trans "API access" %} - {{ api_access.name }}</h2>
-    <span class="actions">
-      <a href="delete" rel="popup">{% trans "Delete" %}</a>
-      <a href="edit">{% trans "Edit" %}</a>
-    </span>
+    {% if not api_access.idp_api_client %}
+      <span class="actions">
+        <a href="delete" rel="popup">{% trans "Delete" %}</a>
+        <a href="edit">{% trans "Edit" %}</a>
+      </span>
+    {% endif %}
   </div>
 
   {% if api_access.description %}
@@ -16,8 +18,12 @@
   <div class="bo-block">
     <h3>{% trans "Parameters" %}</h3>
     <ul>
-      <li>{% trans "Access identifier:" %} {{ api_access.access_identifier }}</li>
-      <li>{% trans "Access key:" %} {{ api_access.access_key }}</li>
+      {% if not api_access.idp_api_client %}
+        <li>{% trans "Access identifier:" %} {{ api_access.access_identifier }}</li>
+        <li>{% trans "Access key:" %} {{ api_access.access_key }}</li>
+      {% else %}
+        <li>{% trans "API client from identity provider, identifier:" %} {{ api_access.access_identifier|removeprefix:"_idp_" }}</li>
+      {% endif %}
       {% if api_access.restrict_to_anonymised_data %}<li>{% trans "Restricted to anonymised data" %}</li>{% endif %}
       {% if api_access.get_roles %}
         <li>{% trans "Roles:" %}

wcs/templates/wcs/backoffice/carddef.html

@@ -14,7 +14,9 @@
       </p>
     {% endif %}
   </div>
+
   {{ publisher.get_request.session.display_message|safe }}
+  {% include "wcs/backoffice/includes/sql-fields-integrity.html" %}
 
   <div class="bo-block">
     <h3>{% trans "Information" %}</h3>
@@ -41,6 +43,7 @@
       <ul class="biglist optionslist">
         {{ options.templates|safe }}
         {{ options.user_support|safe }}
+        {{ options.management|safe }}
       </ul>
     </div>
   </div>

wcs/templates/wcs/backoffice/formdef-inspect.html

@@ -11,6 +11,9 @@
       <button role="tab" aria-selected="false" aria-controls="inspect-workflow" id="tab-workflow" tabindex="-1">{% trans "Workflow" %}</button>
       <button role="tab" aria-selected="false" aria-controls="inspect-options" id="tab-options" tabindex="-1">{% trans "Options" %}</button>
       <button role="tab" aria-selected="false" aria-controls="inspect-fields" id="tab-fields" tabindex="-1">{% trans "Fields" %}</button>
+      {% if not snapshots_diff and not is_carddef %}
+        <button role="tab" aria-selected="false" aria-controls="inspect-drafts" id="tab-drafts" tabindex="-1">{% trans "Drafts" %}</button>
+      {% endif %}
       {% if custom_views %}
         <button role="tab" aria-selected="false" aria-controls="inspect-customviews" id="tab-customviews" tabindex="-1">{% trans "Custom views" %}</button>
       {% endif %}
@@ -93,6 +96,35 @@
         {% endfor %}
       </div>
 
+      {% if not snapshots_diff and not is_carddef %}
+        <div id="inspect-drafts" role="tabpanel" tabindex="0" aria-labelledby="tab-drafts" hidden>
+          {% if drafts %}
+            <div class="infonotice">
+              <p>{% trans "Statistics on drafts by page." %}</p>
+              <p>{% trans "Lifespan of drafts (in days)" %}{% trans ":" %} {{ formdef.get_drafts_lifespan }}.</p>
+            </div>
+            <h3>{% trans "Rate among drafts" %}</h2>
+            <table class="stats" data-table-id="rate-among-drafts">
+              <tbody>
+                {% for page_drafts in drafts %}
+                  {% include "wcs/backoffice/includes/inspect-draft-by-page.html" with page_id=page_drafts.0 field=page_drafts.1.field percent=page_drafts.1.percent percent_str=page_drafts.1.percent_str num=page_drafts.1.total den=total_drafts %}
+                {% endfor %}
+              </tbody>
+            </table>
+            <h3>{% trans "Rate among total forms" %}</h2>
+            <table class="stats" data-table-id="rate-among-total-forms">
+              <tbody>
+                {% for page_drafts in drafts %}
+                  {% include "wcs/backoffice/includes/inspect-draft-by-page.html" with page_id=page_drafts.0 field=page_drafts.1.field percent=page_drafts.1.to_formdata_percent percent_str=page_drafts.1.to_formdata_percent_str num=page_drafts.1.total den=total_formdata %}
+                {% endfor %}
+              </tbody>
+            </table>
+          {% else %}
+            <p>{% trans "There are currently no drafts for this form." %}</p>
+          {% endif %}
+        </div>
+      {% endif %}
+
       <div id="inspect-customviews" role="tabpanel" tabindex="0" aria-labelledby="tab-customviews" hidden>
         <div>
           {% for custom_view in custom_views %}

wcs/templates/wcs/backoffice/formdef.html

@@ -35,6 +35,7 @@
   </div>
 
   {{ publisher.get_request.session.display_message|safe }}
+  {% include "wcs/backoffice/includes/sql-fields-integrity.html" %}
 
   <div class="bo-block">
     <h3>{% trans "Information" %}</h3>

wcs/templates/wcs/backoffice/includes/inspect-draft-by-page.html

@@ -0,0 +1,23 @@
+{% load i18n %}
+{% if num %}
+  <tr data-page-id="{{ page_id }}">
+    <td class="label">
+      {% if page_id == "_unknown" %}
+        {% trans "Unknown" %}
+      {% elif page_id == "_first_page" %}
+        {% trans "Only page" %}
+      {% elif page_id == "_confirmation_page" %}
+        {% trans "Confirmation page" %}
+      {% else %}
+        {{ field.ellipsized_label }}
+      {% endif %}
+    </td>
+    <td class="percent">{{ percent|floatformat }}{% trans "%" %}</td>
+    <td class="total">({{ num }}/{{ den }})</td>
+  </tr>
+  <tr>
+    <td class="bar" colspan="3">
+      <span style="width: {{ percent_str }}%"></span>
+    </td>
+  </tr>
+{% endif %}

wcs/templates/wcs/backoffice/includes/sql-fields-integrity.html

@@ -0,0 +1,25 @@
+{% load i18n %}
+
+{% if formdef.sql_integrity_errors %}
+  <div class="errornotice">
+    <details><summary>
+      {% blocktrans trimmed %}
+        There are integrity errors in the database column types.
+      {% endblocktrans %}
+    </summary>
+      <ul>
+        {% for error in formdef.sql_integrity_errors.items %}
+          <li>
+            {% with field=formdef.get_all_fields_dict|get:error.0 %}
+              <a href="{{ field.get_admin_url }}">{{ field.ellipsized_label }}</a>,
+              {% blocktrans trimmed with expected=error.1.expected got=error.1.got %}
+                expected: {{ expected }}, got: {{ got }}.
+              {% endblocktrans %}
+            {% endwith %}
+          </li>
+        {% endfor %}
+      </ul>
+    </p>
+  </details>
+  </div>
+{% endif %}

wcs/templates/wcs/includes/drafts-recall.html

@@ -14,7 +14,8 @@
         <ul>
           {% for draft in view.initial_drafts %}
             <li><a href="{{draft.internal_id}}/">{% trans "continue with draft from " %} {{draft.receipt_date}}
-              {{draft.receipt_time}}</a>, {% blocktrans with page_no=draft.page_no|add:1 %}on page {{page_no}}{% endblocktrans %}</li>
+              {{draft.receipt_time}}{% if draft.digest and "None" not in draft.digest %} ({{ draft.digest }}){% endif %}</a>,
+              {% blocktrans with page_no=draft.page_no|add:1 %}on page {{page_no}}{% endblocktrans %}</li>
           {% endfor %}
         </ul>
       {% endif %}

wcs/testdef.py

@@ -21,6 +21,7 @@ import io
 import json
 import socket
 import urllib.parse
+import uuid
 import xml.etree.ElementTree as ET
 from contextlib import contextmanager
 
@@ -75,8 +76,8 @@ class TestDefXmlProxy(XmlStorableObject):
         }
         excluded_fields = ['id', 'object_type', 'object_id']
         extra_fields = [
-            ('workflow_tests', 'workflow_tests'),
             ('_webservice_responses', 'webservice_responses'),
+            ('workflow_tests', 'workflow_tests'),
         ]
 
         return [
@@ -674,6 +675,7 @@ class WebserviceResponse(XmlStorableObject):
     _names = 'webservice-response'
     xml_root_node = 'webservice-response'
 
+    uuid = None
     testdef_id = None
     name = ''
     payload = None
@@ -684,6 +686,7 @@ class WebserviceResponse(XmlStorableObject):
     post_data = None
 
     XML_NODES = [
+        ('uuid', 'str'),
         ('testdef_id', 'int'),
         ('name', 'str'),
         ('payload', 'str'),
@@ -694,6 +697,10 @@ class WebserviceResponse(XmlStorableObject):
         ('post_data', 'kv_data'),
     ]
 
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.uuid = str(uuid.uuid4())
+
     def __str__(self):
         return self.name
 

wcs/variables.py

@@ -176,6 +176,7 @@ class LazyFormDefObjectsManager:
         try:
             custom_view = get_publisher().custom_view_class.select(lookup_criterias)[0]
         except IndexError:
+            self.report_error(_('Unknown custom view "%(slug)s"') % {'slug': custom_view_slug})
             return self.none()
         return self._clone(self._criterias + custom_view.get_criterias(), order_by=custom_view.order_by)
 
@@ -330,6 +331,8 @@ class LazyFormDefObjectsManager:
             return equality_operators + empty_operators
         if field.key == 'email':
             return equality_operators + in_operators + empty_operators + text_operators
+        if field.key == 'file':
+            return empty_operators
         return None
 
     def format_value(self, op, value, field):
@@ -499,7 +502,17 @@ class LazyFormDefObjectsManager:
 
         # check operator
         for field in fields:
-            if field.key not in ['date', 'item', 'items', 'string', 'text', 'bool', 'email', 'numeric']:
+            if field.key not in [
+                'date',
+                'item',
+                'items',
+                'string',
+                'text',
+                'bool',
+                'email',
+                'numeric',
+                'file',
+            ]:
                 continue
             operators = self.get_field_allowed_operators(field) or []
             if op not in [o[0] for o in operators]:
@@ -540,7 +553,17 @@ class LazyFormDefObjectsManager:
                 else:
                     criteria_class = NotNull if exclude else Null
                     criteria = criteria_class(field_id)
-            elif field.key not in ['date', 'item', 'items', 'string', 'text', 'bool', 'email', 'numeric']:
+            elif field.key not in [
+                'date',
+                'item',
+                'items',
+                'string',
+                'text',
+                'bool',
+                'email',
+                'numeric',
+                'file',
+            ]:
                 criteria_class = NotEqual if exclude else Equal
                 criteria = criteria_class(field_id, value, field=field)
             else:

wcs/views.py

@@ -74,12 +74,14 @@ def i18n_js(request):
         'geoloc_permission_denied': _('Geolocation: permission denied'),
         'geoloc_position_unavailable': _('Geolocation: position unavailable'),
         'geoloc_timeout': _('Geolocation: timeout'),
+        'map_position_marker_alt': _('Marker of selected position'),
         'map_search_error': _('An error occured while fetching results'),
         'map_search_hint': _('Search address'),
         'map_search_searching': _('Searching...'),
         'map_zoom_in': _('Zoom in'),
         'map_zoom_out': _('Zoom out'),
         'map_display_position': _('Display my position'),
+        'map_leaflet_title_attribute': _('Leaflet, a JavaScript library for interactive maps'),
         's2_errorloading': _('The results could not be loaded'),
         's2_nomatches': _('No matches found'),
         's2_tooshort': _('Please enter more characters'),

wcs/workflow_tests.py

@@ -35,7 +35,7 @@ from wcs.qommon.form import (
 )
 from wcs.qommon.humantime import humanduration2seconds, seconds2humanduration, timewords
 from wcs.qommon.xml_storage import XmlStorableObject
-from wcs.testdef import TestError, WebserviceResponse
+from wcs.testdef import TestError
 from wcs.wf.backoffice_fields import SetBackofficeFieldRowWidget, SetBackofficeFieldsTableWidget
 from wcs.wf.profile import FieldNode
 from wcs.workflows import WorkflowGlobalActionTimeoutTrigger
@@ -677,20 +677,22 @@ class AssertWebserviceCall(WorkflowTestAction):
     label = _('Assert webservice call')
 
     key = 'assert-webservice-call'
-    webservice_response_id = None
+    webservice_response_uuid = None
     call_count = 1
 
     optional_fields = ['call_count']
 
     XML_NODES = WorkflowTestAction.XML_NODES + [
-        ('webservice_response_id', 'str'),
+        ('webservice_response_uuid', 'str'),
         ('call_count', 'int'),
     ]
 
     @property
     def details_label(self):
         webservice_responses = [
-            x for x in self.parent.testdef.get_webservice_responses() if x.id == self.webservice_response_id
+            x
+            for x in self.parent.testdef.get_webservice_responses()
+            if x.uuid == self.webservice_response_uuid
         ]
         if webservice_responses:
             return webservice_responses[0].name
@@ -710,13 +712,17 @@ class AssertWebserviceCall(WorkflowTestAction):
 
     def perform(self, formdata):
         try:
-            response = WebserviceResponse.get(self.webservice_response_id)
-        except KeyError:
+            response = [
+                x
+                for x in self.parent.testdef.get_webservice_responses()
+                if x.uuid == self.webservice_response_uuid
+            ][0]
+        except IndexError:
             raise WorkflowTestError(_('Broken, missing webservice response'))
 
         call_count = 0
         for used_response in formdata.used_webservice_responses.copy():
-            if used_response.id == self.webservice_response_id:
+            if used_response.uuid == self.webservice_response_uuid:
                 formdata.used_webservice_responses.remove(used_response)
                 call_count += 1
 
@@ -728,7 +734,7 @@ class AssertWebserviceCall(WorkflowTestAction):
 
     def fill_admin_form(self, form, formdef):
         webservice_response_options = [
-            (response.id, response.name, response.id)
+            (response.uuid, response.name, response.uuid)
             for response in self.parent.testdef.get_webservice_responses()
         ]
 
@@ -737,11 +743,11 @@ class AssertWebserviceCall(WorkflowTestAction):
 
         form.add(
             SingleSelectWidget,
-            'webservice_response_id',
+            'webservice_response_uuid',
             title=_('Webservice response'),
             options=webservice_response_options,
             required=True,
-            value=self.webservice_response_id,
+            value=self.webservice_response_uuid,
         )
         form.add(IntWidget, 'call_count', title=_('Call count'), required=True, value=self.call_count)
 

wcs/workflows.py

@@ -360,7 +360,8 @@ class EvolutionPart:
         if not self.view or self.to:
             # don't include parts with no content or restricted visibility
             return ''
-        return misc.html2text(self.view() or '')
+        illegal_fts_chars = re.compile(r'[\x00-\x1F]')
+        return illegal_fts_chars.sub(' ', misc.html2text(self.view() or ''))
 
 
 class AttachmentEvolutionPart(EvolutionPart):
@@ -2023,9 +2024,11 @@ class WorkflowGlobalActionTimeoutTrigger(WorkflowGlobalActionTrigger):
         def validate_timeout(value):
             if Template.is_template_string(value):
                 return ComputedExpressionWidget.validate_template(value)
-            match = re.match(r'^-?\d+$', value or '')
+            match = re.match(r'^-?[1-9]\d*$', value or '')
             if not match or not match.group() == value:
                 raise ValueError(_('wrong format'))
+            if not (365 * -100 < float(value) < 365 * 100):  # ±100 years should be enough
+                raise ValueError(_('invalid value, out of bounds'))
 
         form.add(
             StringWidget,