sql: test purge of search tokens (#86527)

introduce a new mechanism to implement FTS with fuzzy-match.
This is made possible by adding and maintaining a table of the
FTS tokens, wcs_search_tokens, fed with searchable_formdefs
and wcs_all_forms.
When a query is issued, its tokens are matched against the
tokens with a fuzzy match when no direct match is found, and
the query is then rebuilt.

debian/control

@@ -23,10 +23,12 @@ Depends: graphviz,
          python3-django-ratelimit,
          python3-dnspython,
          python3-emoji,
+         python3-freezegun,
          python3-hobo,
          python3-lasso,
          python3-lxml,
          python3-pil,
+         python3-psutil,
          python3-psycopg2,
          python3-pyproj,
          python3-quixote,

setup.py

@@ -204,6 +204,8 @@ setup(
         'setproctitle',
         'phonenumbers',
         'emoji',
+        'psutil',
+        'freezegun',
     ],
     package_dir={'wcs': 'wcs'},
     packages=find_packages(),

tests/admin_pages/test_all.py

@@ -31,6 +31,7 @@ def create_superuser(pub):
 
     user1 = pub.user_class(name='admin')
     user1.is_admin = True
+    user1.email = 'admin@example.com'
     user1.store()
 
     account1 = PasswordAccount(id='admin')

tests/admin_pages/test_card.py

@@ -1118,3 +1118,61 @@ def test_cards_last_test_result(pub):
 
     resp = resp.click('Last tests run')
     assert 'Result #%s' % test_result.id in resp.text
+
+
+def test_cards_management_options(pub):
+    create_superuser(pub)
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = [
+        fields.StringField(id='1', label='Test', varname='test'),
+    ]
+    carddef.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/cards/1/')
+
+    # Misc management
+    assert_option_display(resp, 'Management', 'Default')
+    resp = resp.click('Management', href='options/management')
+    assert resp.forms[0]['management_sidebar_items$elementgeneral'].checked is True
+    assert resp.forms[0]['management_sidebar_items$elementdownload-files'].checked is False
+    resp.forms[0]['management_sidebar_items$elementdownload-files'].checked = True
+    resp = resp.forms[0].submit().follow()
+    assert_option_display(resp, 'Management', 'Custom')
+    assert 'general' in CardDef.get(1).management_sidebar_items
+    assert 'download-files' in CardDef.get(1).management_sidebar_items
+
+    resp = resp.click('Management', href='options/management')
+    resp.forms[0]['management_sidebar_items$elementgeneral'].checked = False
+    resp = resp.forms[0].submit().follow()
+    assert 'general' not in CardDef.get(1).management_sidebar_items
+
+    resp = resp.click('Management', href='options/management')
+    resp.forms[0]['management_sidebar_items$elementgeneral'].checked = True
+    resp.forms[0]['management_sidebar_items$elementdownload-files'].checked = False
+    assert 'management_sidebar_items$elementuser' not in resp.forms[0].fields
+    resp = resp.forms[0].submit().follow()
+    assert CardDef.get(1).management_sidebar_items == {'__default__'}
+
+    carddef.user_support = 'optional'
+    carddef.store()
+
+    resp = resp.click('Management', href='options/management')
+    assert resp.forms[0]['management_sidebar_items$elementuser'].checked is True
+    resp = resp.forms[0].submit().follow()
+    assert CardDef.get(1).management_sidebar_items == {'__default__'}
+
+    assert_option_display(resp, 'Management', 'Default')
+    resp = resp.click('Management', href='options/management')
+    assert resp.form['history_pane_default_mode'].value == 'collapsed'
+    resp = resp.form.submit().follow()
+    assert_option_display(resp, 'Templates', 'Default')
+    resp = resp.click('Management', href='options/management')
+    resp.form['history_pane_default_mode'].value = 'expanded'
+    resp = resp.form.submit().follow()
+    assert_option_display(resp, 'Templates', 'Custom')
+    resp = resp.click('Management', href='options/management')
+    assert resp.form['history_pane_default_mode'].value == 'expanded'

tests/admin_pages/test_deprecations.py

@@ -2,6 +2,7 @@ import io
 import json
 import os
 import zipfile
+from unittest import mock
 
 import pytest
 from quixote.http_request import Upload as QuixoteUpload
@@ -617,7 +618,7 @@ def test_deprecations_on_import(pub):
         job.check_deprecated_elements_in_object(formdef)
     assert str(excinfo.value) == 'Python expression detected'
     with pytest.raises(FormdefImportError) as excinfo:
-        FormDef.import_from_xml_tree(formdef_xml)
+        FormDef.import_from_xml_tree(formdef_xml, check_deprecated=True)
     assert str(excinfo.value) == 'Python expression detected'
 
     job = DeprecationsScan()
@@ -625,7 +626,7 @@ def test_deprecations_on_import(pub):
         job.check_deprecated_elements_in_object(blockdef)
     assert str(excinfo.value) == 'Python expression detected'
     with pytest.raises(BlockdefImportError) as excinfo:
-        BlockDef.import_from_xml_tree(blockdef_xml)
+        BlockDef.import_from_xml_tree(blockdef_xml, check_deprecated=True)
     assert str(excinfo.value) == 'Python expression detected'
 
     job = DeprecationsScan()
@@ -633,7 +634,7 @@ def test_deprecations_on_import(pub):
         job.check_deprecated_elements_in_object(workflow)
     assert str(excinfo.value) == 'Python expression detected'
     with pytest.raises(WorkflowImportError) as excinfo:
-        Workflow.import_from_xml_tree(workflow_xml)
+        Workflow.import_from_xml_tree(workflow_xml, check_deprecated=True)
     assert str(excinfo.value) == 'Python expression detected'
 
     job = DeprecationsScan()
@@ -641,7 +642,7 @@ def test_deprecations_on_import(pub):
         job.check_deprecated_elements_in_object(data_source)
     assert str(excinfo.value) == 'Python expression detected'
     with pytest.raises(NamedDataSourceImportError) as excinfo:
-        NamedDataSource.import_from_xml_tree(data_source_xml)
+        NamedDataSource.import_from_xml_tree(data_source_xml, check_deprecated=True)
     assert str(excinfo.value) == 'Python expression detected'
 
     job = DeprecationsScan()
@@ -649,10 +650,17 @@ def test_deprecations_on_import(pub):
         job.check_deprecated_elements_in_object(wscall)
     assert str(excinfo.value) == 'Python expression detected'
     with pytest.raises(NamedWsCallImportError) as excinfo:
-        NamedWsCall.import_from_xml_tree(wscall_xml)
+        NamedWsCall.import_from_xml_tree(wscall_xml, check_deprecated=True)
     assert str(excinfo.value) == 'Python expression detected'
 
     # no python expressions
     job = DeprecationsScan()
     job.check_deprecated_elements_in_object(mail_template)
     MailTemplate.import_from_xml_tree(mail_template_xml)
+
+    # check that DeprecationsScan is not run on object load
+    with mock.patch(
+        'wcs.backoffice.deprecations.DeprecationsScan.check_deprecated_elements_in_object'
+    ) as check:
+        NamedDataSource.get(data_source.id)
+        assert check.call_args_list == []

tests/admin_pages/test_form.py

@@ -8,6 +8,7 @@ import xml.etree.ElementTree as ET
 
 import pytest
 import responses
+from django.utils.timezone import localtime
 from pyquery import PyQuery
 from webtest import Upload
 
@@ -263,6 +264,14 @@ def test_forms_edit_management(pub, formdef):
     resp = resp.forms[0].submit().follow()
     assert FormDef.get(1).management_sidebar_items == {'__default__'}
 
+    # unselect all
+    resp = resp.click('Management', href='options/management')
+    for field in resp.forms[0].fields:
+        if field.startswith('management_sidebar_items$'):
+            resp.forms[0][field].checked = False
+    resp = resp.forms[0].submit().follow()
+    assert FormDef.get(1).management_sidebar_items == set()
+
 
 def test_forms_edit_tracking_code(pub, formdef):
     create_superuser(pub)
@@ -1133,12 +1142,6 @@ def test_form_workflow_options(pub):
     resp = app.get('/backoffice/forms/1/')
     assert '"workflow-options"' not in resp.text
 
-    pub.site_options.set('options', 'enable-workflow-variable-parameter', 'true')
-    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
-        pub.site_options.write(fd)
-    resp = app.get('/backoffice/forms/1/')
-    assert '"workflow-options"' in resp.text
-
 
 def test_form_workflow_variables(pub):
     create_superuser(pub)
@@ -1979,7 +1982,7 @@ def test_form_preview_map_field(pub):
     app = login(get_app(pub))
     resp = app.get('/backoffice/forms/1/')
     assert 'qommon.map.js' in resp.text
-    assert resp.pyquery('#map-f1')
+    assert resp.pyquery('#form_f1.qommon-map')
 
 
 def test_form_preview_do_not_log_error(pub):
@@ -3691,6 +3694,35 @@ def test_form_edit_field_warnings(pub):
     assert not resp.pyquery('aside .errornotice')
     assert resp.pyquery('aside form[action=new]')
 
+    BlockDef.wipe()
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [
+        fields.StringField(id='123', required=True, label='Test'),
+        fields.StringField(id='234', required=True, label='Test2'),
+        fields.CommentField(id='345', label='comment'),
+    ]
+    block.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [
+        fields.StringField(id='1', label='Test'),
+        fields.BlockField(id='2', label='Block field', block_slug='foobar'),
+    ]
+    formdef.store()
+    resp = app.get('/backoffice/forms/%s/fields/' % formdef.id)
+    assert not resp.pyquery('.warningnotice')
+    formdef.fields[1].default_items_count = 1100
+    formdef.store()
+    resp = app.get('/backoffice/forms/%s/fields/' % formdef.id)
+    assert (
+        resp.pyquery('.warningnotice')
+        .text()
+        .startswith('There are at least 2201 data fields, including fields in blocks.')
+    )
+
     FormDef.wipe()
 
 
@@ -4816,6 +4848,135 @@ def test_admin_form_inspect_validation(pub):
     assert not resp.pyquery('[data-field-id="4"] .parameter-validation').length
 
 
+def test_admin_form_inspect_drafts(pub):
+    create_superuser(pub)
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.enable_tracking_codes = True
+    formdef.fields = [
+        fields.PageField(id='0', label='1st page'),
+        fields.StringField(id='1', label='string 1'),
+        fields.PageField(id='2', label='2nd page'),
+        fields.StringField(id='3', label='string 2'),
+        fields.PageField(id='4', label='3rd page'),
+        fields.StringField(id='5', label='string 3'),
+    ]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/%s/inspect' % formdef.id)
+    assert resp.pyquery('#inspect-drafts p').text() == 'There are currently no drafts for this form.'
+
+    data_class = formdef.data_class()
+    for page_id in ('0', '2', '4', '_confirmation_page', 'xxxx'):
+        formdata = data_class()
+        formdata.status = 'draft'
+        formdata.page_id = page_id
+        formdata.receipt_time = localtime()
+        formdata.store()
+
+    # create a non-draft
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    # create a non-draft but before draft duration
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.receipt_time = localtime() - datetime.timedelta(days=200)
+    formdata.store()
+
+    resp = app.get('/backoffice/forms/%s/inspect' % formdef.id)
+    assert resp.pyquery('#inspect-drafts h2').text() == 'Key indicators on existing drafts'
+    assert resp.pyquery('#inspect-drafts .infonotice').text() == 'Covered period: last 100 days.'
+
+    assert resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="0"]').length == 1
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="0"] td.label').text()
+        == '1st page'
+    )
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="0"] td.percent').text()
+        == '20%'
+    )
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="0"] td.total').text()
+        == '(1/5)'
+    )
+
+    assert resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="2"]').length == 1
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="2"] td.label').text()
+        == '2nd page'
+    )
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="2"] td.percent').text()
+        == '20%'
+    )
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="2"] td.total').text()
+        == '(1/5)'
+    )
+
+    assert resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="4"]').length == 1
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="4"] td.label').text()
+        == '3rd page'
+    )
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="4"] td.percent').text()
+        == '20%'
+    )
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="4"] td.total').text()
+        == '(1/5)'
+    )
+
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="_confirmation_page"]').length
+        == 1
+    )
+    assert (
+        resp.pyquery(
+            'table[data-table-id="rate-among-drafts"] tr[data-page-id="_confirmation_page"] td.label'
+        ).text()
+        == 'Confirmation page'
+    )
+    assert (
+        resp.pyquery(
+            'table[data-table-id="rate-among-drafts"] tr[data-page-id="_confirmation_page"] td.percent'
+        ).text()
+        == '20%'
+    )
+    assert (
+        resp.pyquery(
+            'table[data-table-id="rate-among-drafts"] tr[data-page-id="_confirmation_page"] td.total'
+        ).text()
+        == '(1/5)'
+    )
+
+    assert resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="_unknown"]').length == 1
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="_unknown"] td.label').text()
+        == 'Unknown'
+    )
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="_unknown"] td.percent').text()
+        == '20%'
+    )
+    assert (
+        resp.pyquery('table[data-table-id="rate-among-drafts"] tr[data-page-id="_unknown"] td.total').text()
+        == '(1/5)'
+    )
+
+    # check completion rate
+    assert resp.pyquery('.completion-rate .percent').text() == '16.7%'
+    assert resp.pyquery('.completion-rate .total').text() == '(1/6)'
+    assert 'width: 16.6' in resp.pyquery('.completion-rate .bar span').attr.style
+
+
 def test_form_import_fields(pub):
     create_superuser(pub)
     create_role(pub)
@@ -4991,3 +5152,24 @@ def test_forms_last_test_result(pub, formdef):
     TestDef.remove_object(testdef.id)
     resp = app.get('/backoffice/forms/1/')
     assert 'Last tests run' not in resp.text
+
+
+def test_admin_form_sql_integrity_error(pub):
+    create_superuser(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [fields.BoolField(id='1', label='Bool')]
+    formdef.store()
+
+    formdef.fields = [fields.StringField(id='1', label='String')]
+    formdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get(formdef.get_admin_url())
+    assert (
+        resp.pyquery('.errornotice summary').text()
+        == 'There are integrity errors in the database column types.'
+    )
+    assert resp.pyquery('.errornotice li').text() == 'String, expected: character varying, got: boolean.'

tests/admin_pages/test_tests.py

@@ -7,11 +7,13 @@ from django.utils.timezone import make_aware
 from webtest import Upload
 
 from wcs import fields, workflow_tests
+from wcs.admin.settings import UserFieldsFormDef
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.formdef import FormDef
 from wcs.qommon.http_request import HTTPRequest
 from wcs.qommon.upload_storage import PicklableUpload
+from wcs.sql_criterias import NotNull
 from wcs.testdef import TestDef, TestResult, WebserviceResponse
 from wcs.workflow_tests import WorkflowTests
 from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef
@@ -30,6 +32,7 @@ def pub():
     pub.cfg['identification'] = {'methods': ['password']}
     pub.write_cfg()
 
+    pub.user_class.wipe()
     FormDef.wipe()
     TestDef.wipe()
     TestResult.wipe()
@@ -43,7 +46,7 @@ def teardown_module(module):
 
 
 def test_tests_page(pub):
-    user = create_superuser(pub)
+    create_superuser(pub)
 
     formdef = FormDef()
     formdef.name = 'test title'
@@ -61,8 +64,12 @@ def test_tests_page(pub):
     resp.form['name'] = 'First test'
     resp = resp.form.submit()
 
+    users = pub.user_class.select([NotNull('test_uuid')])
+    assert len(users) == 1
+    test_user = users[0]
+
     testdef = TestDef.select()[0]
-    assert testdef.agent_id == str(user.id)
+    assert testdef.agent_id == test_user.test_uuid
 
     resp = resp.follow()
     assert 'Edit test data' in resp.text
@@ -145,9 +152,13 @@ def test_tests_page_creation_from_formdata(pub):
     assert 'First test' in resp.text
     assert 'abcdefg' in resp.text
 
+    users = pub.user_class.select([NotNull('test_uuid')])
+    assert len(users) == 1
+    test_user = users[0]
+
     testdef = TestDef.select()[0]
-    assert testdef.data['user']['id'] == 1
-    assert testdef.agent_id == str(user.id)
+    assert testdef.user_uuid == test_user.test_uuid
+    assert testdef.agent_id == test_user.test_uuid
     assert not testdef.is_in_backoffice
 
     formdata = formdef.data_class()()
@@ -166,7 +177,7 @@ def test_tests_page_creation_from_formdata(pub):
     assert 'hijklmn' in resp.text
 
     testdef = TestDef.select()[1]
-    assert testdef.data['user'] is None
+    assert not testdef.user_uuid
     assert testdef.is_in_backoffice
 
 
@@ -370,6 +381,7 @@ def test_tests_edit(pub):
     user.store()
     new_user = pub.user_class(name='new user')
     new_user.email = 'new@example.com'
+    new_user.test_uuid = '42'
     new_user.store()
 
     formdef = FormDef()
@@ -395,7 +407,7 @@ def test_tests_edit(pub):
 
     resp = resp.click('Options')
     resp.form['name'] = 'Second test'
-    resp.form['user'] = new_user.id
+    resp.form['user'] = new_user.test_uuid
     resp = resp.form.submit('submit').follow()
     assert 'Second test' in resp.text
     assert 'new user' in resp.text
@@ -407,7 +419,7 @@ def test_tests_edit(pub):
     assert 'new user' not in resp.text
 
     resp = resp.click('Options')
-    resp.form['user'] = new_user.id
+    resp.form['user'] = new_user.test_uuid
     resp = resp.form.submit('submit').follow()
     assert 'Second test' in resp.text
     assert 'new user' in resp.text
@@ -702,6 +714,13 @@ def test_tests_edit_data_live_url(formdef_class, pub):
             required=True,
             condition={'type': 'django', 'value': 'form_var_foo == "ok"'},
         ),
+        fields.StringField(
+            id='3',
+            label='Condi 2',
+            varname='bar2',
+            required=True,
+            condition={'type': 'django', 'value': 'form_var_foo and is_in_backoffice'},
+        ),
     ]
     formdef.store()
     formdef.data_class().wipe()
@@ -717,10 +736,13 @@ def test_tests_edit_data_live_url(formdef_class, pub):
     live_url = resp.html.find('form').attrs['data-live-url']
     live_resp = app.post(live_url, params=resp.form.submit_fields())
     assert live_resp.json['result']['2']['visible'] is True
+    assert live_resp.json['result']['3']['visible'] is False
 
+    resp = resp.click('Switch to backoffice mode.').follow()
     resp.form['f1'] = 'nok'
     live_resp = app.post(live_url, params=resp.form.submit_fields())
     assert live_resp.json['result']['2']['visible'] is False
+    assert live_resp.json['result']['3']['visible'] is True
 
 
 def test_tests_manual_run(pub):
@@ -1006,12 +1028,16 @@ def test_tests_result_error_field(pub):
 
 
 def test_tests_result_inspect(pub):
-    user = create_superuser(pub)
+    create_superuser(pub)
 
     role = pub.role_class(name='test role')
     role.store()
-    user.roles = [role.id]
-    user.store()
+
+    test_user = pub.user_class(name='new user')
+    test_user.email = 'new@example.com'
+    test_user.test_uuid = '42'
+    test_user.roles = [role.id]
+    test_user.store()
 
     workflow = Workflow(name='Workflow One')
     workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
@@ -1049,7 +1075,7 @@ def test_tests_result_inspect(pub):
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
     testdef.name = 'First test'
-    testdef.agent_id = user.id
+    testdef.agent_id = test_user.test_uuid
     testdef.is_in_backoffice = True
     testdef.workflow_tests.actions = [
         workflow_tests.ButtonClick(id='1', button_name='Loop on status'),
@@ -1430,3 +1456,94 @@ def test_tests_webservice_response(pub):
     resp = resp.form.submit()
 
     assert 'must start with http://' in resp.text
+
+
+def test_tests_test_users_management(pub):
+    create_superuser(pub)
+
+    role = pub.role_class(name='test role')
+    role.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.store()
+
+    user_formdef = UserFieldsFormDef(pub)
+    user_formdef.fields = [
+        fields.StringField(id='1', label='first_name', varname='first_name'),
+        fields.StringField(id='2', label='last_name', varname='last_name'),
+        fields.StringField(id='3', label='email', varname='email'),
+    ]
+    user_formdef.store()
+    pub.cfg['users'][
+        'fullname_template'
+    ] = '{{ user_var_first_name|default:"" }} {{ user_var_last_name|default:"" }}'
+    pub.cfg['users']['field_email'] = '3'
+    pub.write_cfg()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/%s/tests/' % formdef.id)
+
+    resp = resp.click('Test users')
+    assert 'There are no test users yet.' in resp.text
+
+    resp = resp.click('New')
+    resp.form['name'] = 'User test'
+    resp = resp.form.submit().follow()
+
+    assert 'There are no test users yet.' not in resp.text
+
+    resp = resp.click('User test')
+    resp.form['roles$element0'] = role.id
+    resp.form['f1'] = 'Jon'
+    resp.form['f2'] = 'Doe'
+    resp.form['f3'] = 'jon@example.com'
+    resp = resp.form.submit('submit').follow()
+
+    user = pub.user_class.select([NotNull('test_uuid')])[0]
+    assert user.name == 'User test'
+    assert user.email == 'jon@example.com'
+    assert user.roles == [role.id]
+    assert user.form_data['1'] == 'Jon'
+    assert user.form_data['2'] == 'Doe'
+
+    real_user = pub.user_class(name='new user')
+    real_user.email = 'jane@example.com'
+    real_user.form_data = {
+        '1': 'Jane',
+        '2': 'Doe',
+    }
+    real_user.store()
+
+    resp = resp.click('New')
+    resp.form['name'] = 'User test 2'
+    resp.form['creation_mode'] = 'copy'
+    resp.form['user_id'].force_value(real_user.id)
+    resp = resp.form.submit().follow()
+
+    user = pub.user_class.select([NotNull('test_uuid')], order_by='id')[1]
+    assert user.name == 'User test 2'
+    assert user.email == 'jane@example.com'
+    assert user.form_data['1'] == 'Jane'
+    assert user.form_data['2'] == 'Doe'
+
+    resp = resp.click('New')
+    resp.form['name'] = 'User test 3'
+    resp.form['creation_mode'] = 'copy'
+    resp.form['user_id'].force_value(real_user.id)
+    resp = resp.form.submit()
+
+    assert 'A test user with this email already exists.' in resp.text
+
+    resp = app.get('/backoffice/forms/test-users/')
+    resp = resp.click('User test 2')
+    resp.form['f3'] = 'jon@example.com'
+    resp = resp.form.submit('submit')
+
+    assert 'A test user with this email already exists.' in resp.text
+
+    resp = app.get('/backoffice/forms/test-users/')
+    resp = resp.click('Remove', href=str(user.id))
+    resp = resp.form.submit().follow()
+
+    assert 'User test 2' not in resp.text

tests/admin_pages/test_workflow.py

@@ -8,7 +8,7 @@ import responses
 from pyquery import PyQuery
 from webtest import Upload
 
-from wcs import fields
+from wcs import fields, workflow_tests
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.categories import WorkflowCategory
@@ -17,6 +17,7 @@ from wcs.mail_templates import MailTemplate
 from wcs.qommon.afterjobs import AfterJob
 from wcs.qommon.errors import ConnectionError
 from wcs.qommon.http_request import HTTPRequest
+from wcs.testdef import TestDef, TestResult
 from wcs.wf.create_formdata import CreateFormdataWorkflowStatusItem, Mapping
 from wcs.wf.form import WorkflowFormFieldsFormDef
 from wcs.workflows import (
@@ -633,7 +634,10 @@ def test_workflows_delete_status_reassign(pub, name):
             resp = resp.follow()
             assert formdefs[-1].data_class().get(formdata2.id).status == 'wf-%s' % wf_bar.id
 
-        assert AfterJob.count() == 3  # status change + rebuild_security + tests
+        if name in ('forms', 'cards'):
+            assert AfterJob.count() == 3  # status change + rebuild_security + form or card tests
+        else:
+            assert AfterJob.count() == 4  # status change + rebuild_security + card tests + form tests
         resp = resp.click('Back')
         assert resp.request.path == f'/backoffice/workflows/{workflow.id}/'
 
@@ -2103,8 +2107,7 @@ def test_workflows_variables_edit(pub):
     assert resp.location == 'http://example.net/backoffice/workflows/1/variables/fields/'
     resp = resp.follow()
     resp = resp.click('Edit', href='%s/' % workflow.variables_formdef.fields[0].id)
-    assert resp.forms[0]['varname$name'].value == 'foobar'
-    assert 'varname$select' not in resp.forms[0].fields
+    assert resp.forms[0]['varname'].value == 'foobar'
 
     baz_status = workflow.add_status(name='baz')
     baz_status.add_action('displaymsg')
@@ -2112,24 +2115,7 @@ def test_workflows_variables_edit(pub):
 
     resp = app.get('/backoffice/workflows/1/variables/fields/')
     resp = resp.click('Edit', href='%s/' % workflow.variables_formdef.fields[0].id)
-    assert 'varname$select' not in resp.forms[0].fields
-
-    pub.site_options.set('options', 'enable-workflow-variable-parameter', 'true')
-    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
-        pub.site_options.write(fd)
-    resp = app.get('/backoffice/workflows/1/variables/fields/')
-    resp = resp.click('Edit', href='%s/' % workflow.variables_formdef.fields[0].id)
-    assert 'varname$select' in resp.forms[0].fields
-    resp.forms[0]['varname$select'].value = '1*1*message'
-    assert (
-        resp.pyquery('[data-widget-name="default_value"]')[0].attrib['data-dynamic-display-child-of']
-        == 'varname$select'
-    )
-    resp = resp.forms[0].submit('submit')
-
-    workflow = Workflow.get(1)
-    assert workflow.variables_formdef.fields[0].key == 'string'
-    assert workflow.variables_formdef.fields[0].varname == '1*1*message'
+    assert 'varname' in resp.forms[0].fields
 
 
 def test_workflows_variables_default_value(pub):
@@ -2183,20 +2169,6 @@ def test_workflows_variables_edit_with_all_action_types(pub):
     assert resp.location == 'http://example.net/backoffice/workflows/1/variables/fields/'
     resp = resp.follow()
 
-    pub.site_options.set('options', 'enable-workflow-variable-parameter', 'true')
-    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
-        pub.site_options.write(fd)
-    workflow = Workflow.get(1)
-    resp = app.get('/backoffice/workflows/1/variables/fields/')
-    resp = resp.click('Edit', href='%s/' % workflow.variables_formdef.fields[0].id)
-    assert 'varname$select' in resp.forms[0].fields
-    resp.forms[0]['varname$name'].value = 'xxx'
-    resp = resp.forms[0].submit('submit')
-
-    workflow = Workflow.get(1)
-    assert workflow.variables_formdef.fields[0].key == 'string'
-    assert workflow.variables_formdef.fields[0].varname == 'xxx'
-
 
 def test_workflows_variables_delete(pub):
     create_superuser(pub)
@@ -2234,55 +2206,6 @@ def test_workflows_variables_with_export_to_model_action(pub):
     resp = resp.click('Edit', href='%s/' % workflow.variables_formdef.fields[0].id)
 
 
-def test_workflows_variables_replacement(pub):
-    create_superuser(pub)
-    pub.site_options.set('options', 'enable-workflow-variable-parameter', 'true')
-    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
-        pub.site_options.write(fd)
-
-    Workflow.wipe()
-    workflow = Workflow(name='foo')
-    baz_status = workflow.add_status(name='baz')
-    baz_status.add_action('displaymsg', id='1')
-    workflow.store()
-
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/workflows/%s/variables/fields/' % workflow.id)
-
-    # add a field
-    resp.forms[0]['label'] = 'foobar'
-    resp.forms[0]['type'] = 'string'
-    resp = resp.forms[0].submit().follow()
-    workflow = Workflow.get(1)
-
-    # edit
-    resp = resp.click('Edit', href='%s/' % workflow.variables_formdef.fields[0].id)
-    resp.form['varname$select'].value = '1*1*message'
-    resp = resp.form.submit('submit').follow()
-
-    # make sure a wrong variable name is not displayed
-    assert 'form_option_1*1*message' not in resp.text
-    assert Workflow.get(workflow.id).variables_formdef.fields[0].varname == '1*1*message'
-
-    # and make sure it doesn't appear in formdata inspect page
-    formdef = FormDef()
-    formdef.name = 'Form title'
-    formdef.workflow = workflow
-    formdef.fields = []
-    formdef.store()
-
-    data_class = formdef.data_class()
-    data_class.wipe()
-
-    formdata = data_class()
-    formdata.data = {}
-    formdata.status = 'wf-new'
-    formdata.store()
-
-    resp = app.get(formdata.get_backoffice_url() + 'inspect')
-    assert 'form_option_1*1*message' not in resp.text
-
-
 def test_workflows_backoffice_fields(pub):
     create_superuser(pub)
 
@@ -2375,6 +2298,19 @@ def test_workflows_backoffice_fields(pub):
     )
     assert 'prefill$type' not in resp.form.fields.keys()
 
+    # check display_locations
+    resp.form['display_locations$element0'] = False
+    resp.form['display_locations$element1'] = False
+    resp = resp.form.submit('submit')
+    assert (
+        resp.location
+        == 'http://example.net/backoffice/workflows/1/backoffice-fields/fields/#fieldId_%s'
+        % workflow.backoffice_fields_formdef.fields[1].id
+    )
+    resp = resp.follow()
+    workflow = Workflow.get(workflow.id)
+    assert workflow.backoffice_fields_formdef.fields[1].display_locations is None
+
     # add a title field
     resp = app.get('/backoffice/workflows/1/backoffice-fields/fields/')
     resp.forms[0]['label'] = 'foobar3'
@@ -2842,10 +2778,14 @@ def test_workflows_global_actions_timeout_triggers(pub):
     resp = resp.click(
         href='triggers/%s/' % Workflow.get(workflow.id).global_actions[0].triggers[0].id, index=0
     )
-    for invalid_value in ('foobar', '-'):
+    for invalid_value in ('foobar', '-', '0123'):
         resp.form['timeout'] = invalid_value
         resp = resp.form.submit('submit')
         assert 'wrong format' in resp.text
+    for invalid_value in ('833333335', '-833333335'):
+        resp.form['timeout'] = invalid_value
+        resp = resp.form.submit('submit')
+        assert 'invalid value, out of bounds' in resp.text
     resp.form['timeout'] = ''
     resp = resp.form.submit('submit')
     assert 'required field' in resp.text
@@ -4434,3 +4374,58 @@ def test_workflows_function_and_role_with_same_name(pub):
         (str(role1.id), False, 'Foo'),
         (str(role2.id), False, 'Foobar [role]'),  # same name as function -> role suffix
     ]
+
+
+def test_workflow_test_results(pub):
+    create_superuser(pub)
+    TestDef.wipe()
+    TestResult.wipe()
+
+    Workflow.wipe()
+    workflow = Workflow(name='Workflow One')
+    workflow.add_status(name='New status')
+    workflow.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.workflow_id = workflow.id
+    formdef.name = 'test title'
+    formdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/workflows/1/edit')
+    resp.form['name'] = 'test'
+    resp = resp.form.submit('submit').follow()
+    assert TestResult.count() == 0
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.store()
+
+    resp = app.get('/backoffice/workflows/1/edit')
+    resp.form['name'] = 'test 2'
+    resp = resp.form.submit('submit').follow()
+    assert TestResult.count() == 0
+
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertStatus(id='1'),
+    ]
+    testdef.store()
+
+    resp = app.get('/backoffice/workflows/1/edit')
+    resp.form['name'] = 'test 3'
+    resp = resp.form.submit('submit').follow()
+
+    assert TestResult.count() == 1
+    result = TestResult.select()[0]
+    assert result.reason == 'Change in workflow'
+
+    resp = resp.click('add status')
+    resp.forms[0]['name'] = 'new status'
+    resp = resp.forms[0].submit()
+
+    assert TestResult.count() == 2
+    result = TestResult.select(order_by='id')[1]
+    assert result.reason == 'Workflow: New status "new status"'

tests/admin_pages/test_workflow_tests.py

@@ -11,7 +11,7 @@ from wcs.qommon.http_request import HTTPRequest
 from wcs.testdef import TestDef, WebserviceResponse
 from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef, WorkflowCriticalityLevel
 
-from ..utilities import create_temporary_pub, get_app, login
+from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
 from .test_all import create_superuser
 
 
@@ -30,12 +30,17 @@ def pub():
     with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
         pub.site_options.write(fd)
 
+    pub.user_class.wipe()
     FormDef.wipe()
     TestDef.wipe()
     WebserviceResponse.wipe()
     return pub
 
 
+def teardown_module(module):
+    clean_temporary_pub()
+
+
 def test_workflow_tests_link_feature_flag(pub):
     create_superuser(pub)
 
@@ -68,6 +73,7 @@ def test_workflow_tests_options(pub):
     create_superuser(pub)
     user = pub.user_class(name='test user')
     user.email = 'test@example.com'
+    user.test_uuid = '42'
     user.store()
 
     formdef = FormDef()
@@ -87,44 +93,15 @@ def test_workflow_tests_options(pub):
     resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
     resp = resp.click('Options')
 
-    resp.form['agent'] = user.id
+    resp.form['agent'] = user.test_uuid
     resp = resp.form.submit('submit').follow()
 
     testdef = TestDef.get(testdef.id)
-    assert testdef.agent_id == str(user.id)
-
-
-def test_workflow_tests_disabled_no_agent(pub):
-    user = create_superuser(pub)
-
-    formdef = FormDef()
-    formdef.name = 'test title'
-    formdef.store()
-
-    formdata = formdef.data_class()()
-    formdata.just_created()
-    formdata.store()
-
-    testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.name = 'First test'
-    testdef.store()
-
-    app = login(get_app(pub))
-
-    resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
-    assert 'Backoffice user is not defined, workflow tests will not be executed.' in resp.text
-
-    resp = resp.click('Open test options')
-    resp.form['agent'] = user.id
-    resp.form.submit().follow()
-
-    resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
-    assert 'Backoffice user is not defined' not in resp.text
-    assert 'Open test options' not in resp.text
+    assert testdef.agent_id == user.test_uuid
 
 
 def test_workflow_tests_edit_actions(pub):
-    user = create_superuser(pub)
+    create_superuser(pub)
 
     formdef = FormDef()
     formdef.name = 'test title'
@@ -136,7 +113,6 @@ def test_workflow_tests_edit_actions(pub):
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
     testdef.name = 'First test'
-    testdef.agent_id = user.id
     testdef.store()
 
     app = login(get_app(pub))
@@ -170,22 +146,38 @@ def test_workflow_tests_edit_actions(pub):
     resp = resp.form.submit().follow()
 
     assert 'not configured' not in resp.text
-    assert resp.text.count(escape('Click on "Accept"')) == 1
+    assert [x.text for x in resp.pyquery('ul li.workflow-test-action span.type')] == [
+        'Click on "Accept" by backoffice user',
+    ]
 
     resp = resp.click('Duplicate').follow()
-    assert resp.text.count(escape('Click on "Accept"')) == 2
+    assert [x.text for x in resp.pyquery('ul li.workflow-test-action span.type')] == [
+        'Click on "Accept" by backoffice user',
+        'Click on "Accept" by backoffice user',
+    ]
 
     resp = resp.click('Edit', index=0)
     resp.form['button_name'] = 'Reject'
     resp = resp.form.submit().follow()
 
-    assert resp.text.count(escape('Click on "Accept"')) == 1
-    assert resp.text.count(escape('Click on "Reject"')) == 1
+    assert [x.text for x in resp.pyquery('ul li.workflow-test-action span.type')] == [
+        'Click on "Reject" by backoffice user',
+        'Click on "Accept" by backoffice user',
+    ]
+
+    resp = resp.click('Duplicate', index=0).follow()
+    assert [x.text for x in resp.pyquery('ul li.workflow-test-action span.type')] == [
+        'Click on "Reject" by backoffice user',
+        'Click on "Reject" by backoffice user',
+        'Click on "Accept" by backoffice user',
+    ]
 
     resp = resp.click('Delete', index=0)
     resp = resp.form.submit().follow()
-    assert resp.text.count(escape('Click on "Accept"')) == 1
-    assert resp.text.count(escape('Click on "Reject"')) == 0
+    assert [x.text for x in resp.pyquery('ul li.workflow-test-action span.type')] == [
+        'Click on "Reject" by backoffice user',
+        'Click on "Accept" by backoffice user',
+    ]
 
     # simulate invalid action
     testdef = TestDef.get(testdef.id)
@@ -193,12 +185,15 @@ def test_workflow_tests_edit_actions(pub):
     testdef.store()
 
     resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
-    assert 'There are no workflow test actions yet.' in resp.text
+    assert [x.text for x in resp.pyquery('ul li.workflow-test-action span.type')] == [
+        'Click on "Accept" by backoffice user',
+    ]
 
 
 def test_workflow_tests_action_button_click(pub):
     create_superuser(pub)
     user = pub.user_class(name='test user')
+    user.test_uuid = '42'
     user.store()
 
     workflow = Workflow(name='Workflow One')
@@ -260,7 +255,7 @@ def test_workflow_tests_action_button_click(pub):
 
     resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
     resp.form['who'] = 'other'
-    resp.form['who_id'].force_value(user.id)
+    resp.form['who_id'] = user.test_uuid
     resp = resp.form.submit().follow()
 
     assert escape('Click on "Button 1" by test user') in resp.text
@@ -269,6 +264,21 @@ def test_workflow_tests_action_button_click(pub):
     resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
     assert escape('Click on "Button 1" by missing user') in resp.text
 
+    user.store()
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
+    resp.form['who'] = 'receiver'
+    resp = resp.form.submit().follow()
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
+    assert escape('Selected user is "Backoffice user" but it is not defined.') in resp.text
+
+    resp = resp.click('Open test options')
+    resp.form['agent'] = user.test_uuid
+    resp.form.submit().follow()
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/1/' % testdef.id)
+    assert escape('Selected user is "Backoffice user" but it is not defined.') not in resp.text
+
 
 def test_workflow_tests_action_assert_status(pub):
     create_superuser(pub)
@@ -379,6 +389,20 @@ def test_workflow_tests_action_assert_email(pub):
     resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
     assert escape('Email to "a@entrouvert.com" (+2)') in resp.text
 
+    assert_email.addresses = []
+    assert_email.subject_strings = ['Hello your form has been submitted']
+    assert_email.parent.store()
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
+    assert escape('Subject must contain "Hello your form has been su(…)"') in resp.text
+
+    assert_email.subject_strings = []
+    assert_email.body_strings = ['Hello your form has been submitted']
+    assert_email.parent.store()
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
+    assert escape('Body must contain "Hello your form has been su(…)"') in resp.text
+
 
 def test_workflow_tests_action_assert_sms(pub):
     create_superuser(pub)
@@ -408,14 +432,14 @@ def test_workflow_tests_action_assert_sms(pub):
 
     resp = resp.click('Edit')
     resp.form['phone_numbers$element0'] = '0123456789'
-    resp.form['body'] = 'Hello'
+    resp.form['body'] = 'Hello your form has been submitted'
     resp = resp.form.submit().follow()
 
     assert 'SMS to 0123456789' in resp.text
 
     assert_sms = TestDef.get(testdef.id).workflow_tests.actions[0]
     assert assert_sms.phone_numbers == ['0123456789']
-    assert assert_sms.body == 'Hello'
+    assert assert_sms.body == 'Hello your form has been submitted'
 
     assert_sms.phone_numbers = ['0123456789', '0123456781', '0123456782']
     assert_sms.parent.store()
@@ -423,6 +447,12 @@ def test_workflow_tests_action_assert_sms(pub):
     resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
     assert escape('SMS to 0123456789 (+2)') in resp.text
 
+    assert_sms.phone_numbers = []
+    assert_sms.parent.store()
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
+    assert 'Hello your form has been su(…)' in resp.text
+
 
 def test_workflow_tests_action_assert_anonymise(pub):
     create_superuser(pub)
@@ -497,10 +527,11 @@ def test_workflow_tests_action_assert_history_message(pub):
     assert 'not configured' in resp.text
 
     resp = resp.click('Edit')
-    resp.form['message'] = 'Hello'
+    resp.form['message'] = 'Hello your form has been submitted'
     resp = resp.form.submit().follow()
 
     assert 'not configured' not in resp.text
+    assert 'Hello your form has been su(…)' in resp.text
 
 
 def test_workflow_tests_action_assert_alert(pub):
@@ -525,10 +556,11 @@ def test_workflow_tests_action_assert_alert(pub):
     assert 'not configured' in resp.text
 
     resp = resp.click('Edit')
-    resp.form['message'] = 'Hello'
+    resp.form['message'] = 'Hello your form has been submitted'
     resp = resp.form.submit().follow()
 
     assert 'not configured' not in resp.text
+    assert 'Hello your form has been su(…)' in resp.text
 
 
 def test_workflow_tests_action_assert_criticality(pub):
@@ -739,12 +771,16 @@ def test_workflow_tests_actions_reorder(pub):
 
 
 def test_workflow_tests_run(pub):
-    user = create_superuser(pub)
+    create_superuser(pub)
 
     role = pub.role_class(name='test role')
     role.store()
-    user.roles = [role.id]
-    user.store()
+
+    test_user = pub.user_class(name='test user')
+    test_user.email = 'test@example.com'
+    test_user.test_uuid = '42'
+    test_user.roles = [role.id]
+    test_user.store()
 
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')
@@ -770,7 +806,7 @@ def test_workflow_tests_run(pub):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
+    testdef.agent_id = test_user.test_uuid
     testdef.workflow_tests.actions = [
         workflow_tests.ButtonClick(id='1', button_name='Loop on status'),
     ]

tests/api/test_export_import.py

@@ -9,6 +9,7 @@ import pytest
 
 from wcs.api_export_import import BundleDeclareJob, BundleImportJob, klass_to_slug
 from wcs.applications import Application, ApplicationElement
+from wcs.backoffice.deprecations import DeprecationsScan
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.categories import (
@@ -25,6 +26,7 @@ from wcs.data_sources import NamedDataSource
 from wcs.fields import BlockField, CommentField, ComputedField, PageField, StringField
 from wcs.formdef import FormDef
 from wcs.mail_templates import MailTemplate
+from wcs.qommon.afterjobs import AfterJob
 from wcs.sql import Equal
 from wcs.wf.form import WorkflowFormFieldsFormDef
 from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef, WorkflowVariablesFieldsFormDef
@@ -322,6 +324,11 @@ def test_export_import_dependencies(pub):
                 'value': '{{ forms|objects:"test-bis" }} {{ webservice.test_quinquies }}',
             },
         ),
+        BlockField(
+            id='1bis',
+            label='test_missing',
+            block_slug='test-missing',  # Unknown BlockDef
+        ),
         CommentField(
             id='2',
             label='X {{ webservice.test }} X {{ cards|objects:"test" }} X {{ forms|objects:"test-ter" }} X',
@@ -775,7 +782,9 @@ def test_export_import_bundle_import(pub):
     role = pub.role_class(name='test')
     role.store()
 
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundles[0])
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundles[0])]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'completed'
@@ -870,7 +879,9 @@ def test_export_import_bundle_import(pub):
     element2.store()
 
     # run new import to check it doesn't duplicate objects
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundles[1])
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundles[1])]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'completed'
@@ -931,7 +942,9 @@ def test_export_import_bundle_import(pub):
     formdef.disabled = True
     formdef.store()
 
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundles[1])
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundles[1])]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'completed'
@@ -941,7 +954,9 @@ def test_export_import_bundle_import(pub):
     assert formdef.workflow_roles == {'_receiver': extra_role.id}
 
     # bad file format
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), b'garbage')
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', b'garbage')]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'failed'
@@ -956,7 +971,10 @@ def test_export_import_bundle_import(pub):
         tarinfo = tarfile.TarInfo('foo.json')
         tarinfo.size = len(foo_fd.getvalue())
         tar.addfile(tarinfo, fileobj=foo_fd)
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), tar_io.getvalue())
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'),
+        upload_files=[('bundle', 'bundle.tar', tar_io.getvalue())],
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'failed'
@@ -976,7 +994,10 @@ def test_export_import_bundle_import(pub):
         tarinfo = tarfile.TarInfo('manifest.json')
         tarinfo.size = len(manifest_fd.getvalue())
         tar.addfile(tarinfo, fileobj=manifest_fd)
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), tar_io.getvalue())
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'),
+        upload_files=[('bundle', 'bundle.tar', tar_io.getvalue())],
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'failed'
@@ -1030,7 +1051,9 @@ def test_export_import_bundle_import_categories_ordering(pub, category_class):
     category.store()
 
     # import bundle
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundle)]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'completed'
@@ -1055,7 +1078,9 @@ def test_export_import_bundle_import_categories_ordering(pub, category_class):
     category.position = 3
     category.store()
 
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundle)]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'completed'
@@ -1080,7 +1105,9 @@ def test_export_import_bundle_import_categories_ordering(pub, category_class):
     category.position = 3
     category.store()
 
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundle)]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'completed'
@@ -1105,7 +1132,9 @@ def test_export_import_bundle_import_categories_ordering(pub, category_class):
     category.position = 3
     category.store()
 
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundle)]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'completed'
@@ -1133,7 +1162,9 @@ def test_export_import_bundle_import_categories_ordering(pub, category_class):
     category.position = 20
     category.store()
 
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundle)]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'completed'
@@ -1161,7 +1192,9 @@ def test_export_import_bundle_import_categories_ordering(pub, category_class):
     category.position = None  # no position
     category.store()
 
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundle)]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'completed'
@@ -1195,7 +1228,9 @@ def test_export_import_formdef_do_not_overwrite_table_name(pub):
 
     bundle = create_bundle([{'type': 'forms', 'slug': 'test', 'name': 'test'}], ('forms/test', formdef))
 
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundle)]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'completed'
@@ -1295,7 +1330,9 @@ def test_export_import_bundle_declare(pub):
         visible=False,
     )
 
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-declare/'), bundle)
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-declare/'), upload_files=[('bundle', 'bundle.tar', bundle)]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'completed'
@@ -1328,7 +1365,9 @@ def test_export_import_bundle_declare(pub):
     # and remove an object to have an unkown reference in manifest
     MailTemplate.wipe()
 
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-declare/'), bundle)
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-declare/'), upload_files=[('bundle', 'bundle.tar', bundle)]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'completed'
@@ -1366,7 +1405,9 @@ def test_export_import_bundle_declare(pub):
     )
 
     # bad file format
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-declare/'), b'garbage')
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-declare/'), upload_files=[('bundle', 'bundle.tar', b'garbage')]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'failed'
@@ -1381,7 +1422,10 @@ def test_export_import_bundle_declare(pub):
         tarinfo = tarfile.TarInfo('foo.json')
         tarinfo.size = len(foo_fd.getvalue())
         tar.addfile(tarinfo, fileobj=foo_fd)
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-declare/'), tar_io.getvalue())
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-declare/'),
+        upload_files=[('bundle', 'bundle.tar', tar_io.getvalue())],
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'failed'
@@ -1631,13 +1675,21 @@ def test_export_import_bundle_check(pub):
         incomplete_bundles.append(tar_io.getvalue())
 
     # incorrect bundles, missing information
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-check/'), incomplete_bundles[0])
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-check/'),
+        upload_files=[('bundle', 'bundle.tar', incomplete_bundles[0])],
+    )
     assert resp.json == {'data': {}}
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-check/'), incomplete_bundles[1])
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-check/'),
+        upload_files=[('bundle', 'bundle.tar', incomplete_bundles[1])],
+    )
     assert resp.json == {'data': {}}
 
     # not yet imported
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-check/'), bundles[0])
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-check/'), upload_files=[('bundle', 'bundle.tar', bundles[0])]
+    )
     assert resp.json == {
         'data': {
             'differences': [],
@@ -1664,7 +1716,9 @@ def test_export_import_bundle_check(pub):
     }
 
     # import bundle
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundles[0])
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundles[0])]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'completed'
@@ -1675,7 +1729,9 @@ def test_export_import_bundle_check(pub):
     # remove application links
     Application.wipe()
     ApplicationElement.wipe()
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-check/'), bundles[0])
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-check/'), upload_files=[('bundle', 'bundle.tar', bundles[0])]
+    )
     assert resp.json == {
         'data': {
             'differences': [],
@@ -1777,7 +1833,9 @@ def test_export_import_bundle_check(pub):
     }
 
     # import bundle again, recreate links
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundles[0])
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundles[0])]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'completed'
@@ -1786,7 +1844,9 @@ def test_export_import_bundle_check(pub):
     assert ApplicationElement.count() == 15
 
     # no changes since last import
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-check/'), bundles[0])
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-check/'), upload_files=[('bundle', 'bundle.tar', bundles[0])]
+    )
     assert resp.json == {
         'data': {
             'differences': [],
@@ -1814,7 +1874,9 @@ def test_export_import_bundle_check(pub):
             assert len(new_snapshots) > len(old_snapshots)
 
     # and check
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-check/'), bundles[0])
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-check/'), upload_files=[('bundle', 'bundle.tar', bundles[0])]
+    )
     assert resp.json == {
         'data': {
             'differences': [
@@ -1916,14 +1978,18 @@ def test_export_import_bundle_check(pub):
     }
 
     # update bundle
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundles[1])
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundles[1])]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'completed'
     assert resp.json['data']['completion_status'] == '34/34 (100%)'
 
     # and check
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-check/'), bundles[1])
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-check/'), upload_files=[('bundle', 'bundle.tar', bundles[1])]
+    )
     assert resp.json == {
         'data': {
             'differences': [],
@@ -1938,7 +2004,9 @@ def test_export_import_bundle_check(pub):
         snapshot.application_slug = None
         snapshot.application_version = None
         snapshot.store()
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-check/'), bundles[1])
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-check/'), upload_files=[('bundle', 'bundle.tar', bundles[1])]
+    )
     assert resp.json == {
         'data': {
             'differences': [],
@@ -1965,7 +2033,9 @@ def test_export_import_bundle_check(pub):
     }
 
     # bad file format
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-check/'), b'garbage')
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-check/'), upload_files=[('bundle', 'bundle.tar', b'garbage')]
+    )
     assert resp.json['err_desc'] == 'Invalid tar file'
 
     # missing manifest
@@ -1975,7 +2045,10 @@ def test_export_import_bundle_check(pub):
         tarinfo = tarfile.TarInfo('foo.json')
         tarinfo.size = len(foo_fd.getvalue())
         tar.addfile(tarinfo, fileobj=foo_fd)
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-check/'), tar_io.getvalue())
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-check/'),
+        upload_files=[('bundle', 'bundle.tar', tar_io.getvalue())],
+    )
     assert resp.json['err']
     assert resp.json['err_desc'] == 'Invalid tar file, missing manifest'
 
@@ -1992,7 +2065,10 @@ def test_export_import_bundle_check(pub):
         tarinfo = tarfile.TarInfo('manifest.json')
         tarinfo.size = len(manifest_fd.getvalue())
         tar.addfile(tarinfo, fileobj=manifest_fd)
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-check/'), tar_io.getvalue())
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-check/'),
+        upload_files=[('bundle', 'bundle.tar', tar_io.getvalue())],
+    )
     assert resp.json['err']
     assert resp.json['err_desc'] == 'Invalid tar file, missing component forms/foo'
 
@@ -2025,7 +2101,9 @@ def test_export_import_workflow_options(pub):
     FormDef.wipe()
     Workflow.wipe()
 
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundle)]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'completed'
@@ -2038,7 +2116,9 @@ def test_export_import_workflow_options(pub):
     # check workflow options are not reset on further installs
     formdef.workflow_options = {'foo': 'bar2'}
     formdef.store()
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundle)]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'completed'
@@ -2047,6 +2127,7 @@ def test_export_import_workflow_options(pub):
 
 
 def test_export_import_with_deprecated(pub):
+    AfterJob.wipe()
     pub.load_site_options()
     if not pub.site_options.has_section('options'):
         pub.site_options.add_section('options')
@@ -2066,10 +2147,15 @@ def test_export_import_with_deprecated(pub):
         ],
         ('forms/foo', formdef),
     )
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundle)]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'failed'
+    assert AfterJob.count() == 1
+    job = AfterJob.select()[0]
+    assert not isinstance(job, DeprecationsScan)
 
     blockdef = BlockDef()
     blockdef.name = 'foo'
@@ -2083,7 +2169,9 @@ def test_export_import_with_deprecated(pub):
         ],
         ('blocks/foo', blockdef),
     )
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundle)]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'failed'
@@ -2102,7 +2190,9 @@ def test_export_import_with_deprecated(pub):
         ],
         ('workflows/foo', workflow),
     )
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundle)]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'failed'
@@ -2116,7 +2206,9 @@ def test_export_import_with_deprecated(pub):
         ],
         ('data-sources/foo', data_source),
     )
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundle)]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'failed'
@@ -2131,7 +2223,9 @@ def test_export_import_with_deprecated(pub):
         ],
         ('wscalls/foo', wscall),
     )
-    resp = get_app(pub).put(sign_uri('/api/export-import/bundle-import/'), bundle)
+    resp = get_app(pub).post(
+        sign_uri('/api/export-import/bundle-import/'), upload_files=[('bundle', 'bundle.tar', bundle)]
+    )
     afterjob_url = resp.json['url']
     resp = get_app(pub).put(sign_uri(afterjob_url))
     assert resp.json['data']['status'] == 'failed'

tests/api/test_formdata.py

@@ -10,6 +10,7 @@ import zipfile
 from contextlib import contextmanager
 
 import pytest
+import responses
 from django.utils.encoding import force_bytes
 from django.utils.timezone import localtime, make_aware
 from quixote import get_publisher
@@ -49,6 +50,12 @@ def pub(emails):
             '''\
 [api-secrets]
 coucou = 1234
+
+[variables]
+idp_api_url = https://authentic.example.invalid/api/'
+
+[wscall-secrets]
+authentic.example.invalid = 4460cf12e156d841c116fbebd52d7ebe41282c63ac2605740068ba5fd89b7316
 '''
         )
 
@@ -2985,9 +2992,12 @@ def test_api_distance_filter(pub, local_user):
     get_app(pub).get(sign_uri('/api/forms/test/list?filter-distance=150000', user=local_user), status=400)
 
 
-@pytest.mark.parametrize('user', ['query-email', 'api-access'])
+@pytest.mark.parametrize('user', ['query-email', 'api-access', 'idp-api-client'])
 @pytest.mark.parametrize('auth', ['signature', 'http-basic'])
+@responses.activate
 def test_api_ods_formdata(pub, local_user, user, auth):
+    ApiAccess.wipe()
+
     pub.role_class.wipe()
     role = pub.role_class(name='test')
     role.store()
@@ -3007,7 +3017,6 @@ def test_api_ods_formdata(pub, local_user, user, auth):
     data_class.wipe()
 
     if user == 'api-access':
-        ApiAccess.wipe()
         access = ApiAccess()
         access.name = 'test'
         access.access_identifier = 'test'
@@ -3025,6 +3034,29 @@ def test_api_ods_formdata(pub, local_user, user, auth):
             def get_url(url, **kwargs):
                 return app.get(sign_uri(url, orig=access.access_identifier, key=access.access_key), **kwargs)
 
+    elif user == 'idp-api-client':
+        if auth == 'signature':
+            pytest.skip('signature authentication requires local user')
+
+        def get_url(url, **kwargs):
+            app.set_authorization(('Basic', ('test', '12345')))
+            return app.get(url, **kwargs)
+
+        responses.post(
+            'https://authentic.example.invalid/api/check-api-client/',
+            json={
+                'err': 0,
+                'data': {
+                    'is_active': True,
+                    'is_anonymous': False,
+                    'is_authenticated': True,
+                    'is_superuser': False,
+                    'restrict_to_anonymised_data': False,
+                    'roles': [],
+                },
+            },
+        )
+
     else:
         if auth == 'http-basic':
             pytest.skip('http basic authentication requires ApiAccess')
@@ -3053,6 +3085,21 @@ def test_api_ods_formdata(pub, local_user, user, auth):
     if user == 'api-access':
         access.roles = [role]
         access.store()
+    elif user == 'idp-api-client':
+        responses.post(
+            'https://authentic.example.invalid/api/check-api-client/',
+            json={
+                'err': 0,
+                'data': {
+                    'is_active': True,
+                    'is_anonymous': False,
+                    'is_authenticated': True,
+                    'is_superuser': False,
+                    'restrict_to_anonymised_data': False,
+                    'roles': [role.id],
+                },
+            },
+        )
     else:
         local_user.roles = [role.id]
         local_user.store()
@@ -3081,6 +3128,14 @@ def test_api_ods_formdata(pub, local_user, user, auth):
     formdef.store()
     get_url('/api/forms/test/ods', status=200)
 
+    if user == 'idp-api-client':
+        # check a single api access object has been created
+        assert ApiAccess.count() == 1
+        api_access = ApiAccess.select()[0]
+        assert api_access.idp_api_client
+        assert api_access.access_identifier == '_idp_test'
+        assert api_access.access_key is None
+
 
 def test_api_global_geojson(pub, local_user):
     pub.role_class.wipe()

tests/api/test_formdef.py

@@ -429,6 +429,9 @@ def test_backoffice_submission_formdef_list_search(pub, local_user, access, auth
     resp = get_url('/api/formdefs/?backoffice-submission=on&q=test')
     assert len(resp.json['data']) == 2
 
+    resp = get_url('/api/formdefs/?backoffice-submission=on&q=tes')
+    assert len(resp.json['data']) == 2
+
     resp = get_url('/api/formdefs/?backoffice-submission=on&q=xyz')
     assert len(resp.json['data']) == 0
 

tests/backoffice_pages/test_all.py

@@ -2184,6 +2184,7 @@ def test_backoffice_download_as_zip(pub):
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
     assert 'Download all files as .zip' not in resp
+    formdef.management_sidebar_items = formdef.get_default_management_sidebar_items()
     formdef.management_sidebar_items.add('download-files')
     formdef.store()
     resp = app.get('/backoffice/management/form-title/%s/' % number31.id)

tests/backoffice_pages/test_carddata.py

@@ -1867,12 +1867,14 @@ def test_carddata_add_edit_related(pub):
     childdata = child.data_class().select()[0]
     assert len(childdata.get_workflow_traces()) == 1
 
+    AfterJob.wipe()
     resp = app.get('/backoffice/data/child/%s/wfedit-_editable?_popup=1' % childdata.id)
     assert resp.form['f1'].value == 'foo'
     assert resp.form['f2'].value == 'bar'
     resp.form['f1'] = 'foo2'
     resp.form['f2'] = 'bar2'
     resp = resp.form.submit('submit')
+    assert AfterJob.count() == 1  # check a single job has been created to update relations
     childdata.refresh_from_storage()
     assert len(childdata.get_workflow_traces()) == 2
 
@@ -2128,3 +2130,28 @@ def test_carddata_edit_items_display(pub):
     assert resp.status_int == 302
     resp = resp.follow()
     assert not resp.pyquery('#sect-dataview').text()
+
+
+def test_carddata_history_pane_default_mode(pub):
+    CardDef.wipe()
+    user = create_user(pub)
+
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = []
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.store()
+    carddef.data_class().wipe()
+
+    carddata = carddef.data_class()()
+    carddata.just_created()
+    carddata.store()
+
+    app = login(get_app(pub))
+    resp = app.get(carddata.get_backoffice_url())
+    assert resp.pyquery('#evolution-log.folded')
+
+    carddef.history_pane_default_mode = 'expanded'
+    carddef.store()
+    resp = app.get(carddata.get_backoffice_url())
+    assert resp.pyquery('#evolution-log:not(.folded)')

tests/form_pages/test_all.py

@@ -3273,6 +3273,48 @@ def test_workflow_message_with_template_error(pub):
     assert logged_error.summary == "Error in template of workflow message ('int' object is not iterable)"
 
 
+def test_workflow_condition_on_message_age_in_hours(pub, freezer):
+    create_user(pub)
+
+    formdef = create_formdef()
+    formdef.fields = []
+    formdef.store()
+
+    workflow = Workflow(name='test')
+    st1 = workflow.add_status('Status1', 'st1')
+
+    display1 = st1.add_action('displaymsg')
+    display1.message = 'message-to-all'
+    display1.to = []
+
+    workflow.store()
+
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdef.data_class().wipe()
+
+    app = login(get_app(pub), username='foo', password='foo')
+    page = app.get('/test/')
+    page = page.forms[0].submit('submit')  # form page
+    page = page.forms[0].submit('submit')  # confirmation page
+    page = page.follow()
+    assert 'message-to-all' in page.text
+
+    formdata = formdef.data_class().select()[0]
+    page = app.get(formdata.get_url())
+    assert 'message-to-all' in page.text
+
+    display1.condition = {'type': 'django', 'value': 'form_receipt_datetime|age_in_hours >= 1'}
+    workflow.store()
+    page = app.get(formdata.get_url())
+    assert 'message-to-all' not in page.text
+
+    freezer.tick(60 * 60)
+    page = app.get(formdata.get_url())
+    assert 'message-to-all' in page.text
+
+
 def test_session_cookie_flags(pub):
     create_formdef()
     app = get_app(pub)
@@ -3949,6 +3991,22 @@ def test_email_actions(pub, emails):
     formdata.remove_self()
     app = get_app(pub)
     resp = app.get(action_url, status=404)
+    assert 'This action link is no longer valid as the attached form has been removed.' in resp.text
+
+    # check action link referencing a formdata with an invalid/unknown status
+    emails.empty()
+    formdef.data_class().wipe()
+    app = login(get_app(pub), username='foo', password='foo')
+    resp = app.get(formdef.get_url())
+    resp = resp.form.submit('submit')
+    resp = resp.form.submit('submit')
+    email_data = emails.get('New form2 (test email action)')
+    action_url = re.findall(r'http.* ', email_data['payload'])[0].strip()
+    formdata = formdef.data_class().select()[0]
+    formdata.status = 'wf-abc'
+    formdata.store()
+    app = get_app(pub)
+    resp = app.get(action_url, status=404)
     assert 'This action link is no longer valid' in resp.text
 
     # two buttons on the same line, two urls

tests/form_pages/test_block.py

@@ -77,6 +77,45 @@ def test_block_simple(pub):
     assert '>bar<' in resp
 
 
+def test_block_a11y(pub):
+    FormDef.wipe()
+    BlockDef.wipe()
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [
+        fields.StringField(id='123', label='Test'),
+        fields.StringField(id='234', label='Test2'),
+    ]
+    block.store()
+
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [
+        fields.BlockField(id='1', label='test', block_slug='foobar'),
+    ]
+    formdef.store()
+
+    app = get_app(pub)
+    resp = app.get(formdef.get_url())
+    assert resp.pyquery('.BlockWidget')[0].attrib.get('role') == 'group'
+    assert resp.pyquery('.BlockWidget')[0].attrib.get('aria-labelledby')
+    assert resp.pyquery('#' + resp.pyquery('.BlockWidget')[0].attrib.get('aria-labelledby'))
+
+    formdef.fields[0].label_display = 'subtitle'
+    formdef.store()
+    resp = app.get(formdef.get_url())
+    assert resp.pyquery('.BlockWidget')[0].attrib.get('role')
+    assert resp.pyquery('.BlockWidget')[0].attrib.get('aria-labelledby')
+    assert resp.pyquery('#' + resp.pyquery('.BlockWidget')[0].attrib.get('aria-labelledby'))
+
+    formdef.fields[0].label_display = 'hidden'
+    formdef.store()
+    resp = app.get(formdef.get_url())
+    assert not resp.pyquery('.BlockWidget')[0].attrib.get('role')
+    assert not resp.pyquery('.BlockWidget')[0].attrib.get('aria-labelledby')
+
+
 def test_block_required(pub):
     FormDef.wipe()
     BlockDef.wipe()

tests/form_pages/test_draft.py

@@ -376,6 +376,42 @@ def test_form_recall_draft(pub):
     assert 'href="%s/"' % draft2.id in resp.text
 
 
+def test_form_recall_draft_digests(pub):
+    user = create_user(pub)
+
+    formdef = create_formdef()
+    formdef.fields = [fields.StringField(id='0', label='string', varname='name')]
+    formdef.digest_templates = {'default': 'digest{{form_var_name}}digest'}
+    formdef.store()
+    formdef.data_class().wipe()
+
+    draft = formdef.data_class()()
+    draft.user_id = user.id
+    draft.status = 'draft'
+    draft.data = {'0': 'DIGEST'}
+    draft.store()
+
+    app = login(get_app(pub), username='foo', password='foo')
+    resp = app.get('/test/')
+    # single draft, digest is not displayed
+    assert 'digestDIGESTdigest' not in resp.pyquery(f'[href="{draft.id}/"]').text()
+
+    draft2 = formdef.data_class()()
+    draft2.user_id = user.id
+    draft2.status = 'draft'
+    draft2.data = {}
+    draft2.store()
+
+    resp = app.get('/test/')
+    # two drafts, the first one has its digest displayed
+    assert 'digestDIGESTdigest' in resp.pyquery(f'[href="{draft.id}/"]').text()
+    # the second doesn't have it as it contains "None"
+    assert (
+        resp.pyquery(f'[href="{draft2.id}/"]').text()
+        and draft2.default_digest not in resp.pyquery(f'[href="{draft2.id}/"]').text()
+    )
+
+
 def test_form_max_drafts(pub):
     user = create_user(pub)
 
@@ -821,3 +857,25 @@ def test_draft_store_page_id_when_no_page_and_no_confirmation(pub):
     assert formdef.data_class().count() == 1
     formdata = formdef.data_class().select()[0]
     assert formdata.status == 'wf-new'
+
+
+def test_draft_error_then_autosave(pub):
+    formdef = create_formdef()
+    formdef.enable_tracking_codes = True
+    formdef.fields = [
+        fields.PageField(id='0', label='1st page'),
+        fields.StringField(id='1', label='string 1'),
+        fields.PageField(id='2', label='2nd page'),
+    ]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    app = get_app(pub)
+    resp = app.get('/test/')
+    resp = resp.form.submit('submit')  # error
+    assert formdef.data_class().count() == 1  # server roundtrip -> draft
+
+    resp.form['f1'] = 'test'
+    app.post('/test/autosave', params=resp.form.submit_fields())
+    assert formdef.data_class().count() == 1  # make sure same draft got reused
+    assert formdef.data_class().select()[0].data['1'] == 'test'

tests/form_pages/test_file_field.py

@@ -11,6 +11,7 @@ from wcs.blocks import BlockDef
 from wcs.categories import Category
 from wcs.formdef import FormDef
 from wcs.qommon.errors import ConnectionError
+from wcs.wscalls import NamedWsCall
 
 from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
 from .test_all import create_user
@@ -463,9 +464,9 @@ def test_form_file_field_with_wrong_value(pub):
     assert pub.loggederror_class.count() == 1
     logged_error = pub.loggederror_class.select()[0]
     assert logged_error.formdef_id == formdef.id
-    assert logged_error.summary == 'Failed to set value on field "file"'
-    assert logged_error.exception_class == 'AttributeError'
-    assert logged_error.exception_message == "'str' object has no attribute 'time'"
+    assert logged_error.summary == 'Failed to convert value for field "file"'
+    assert logged_error.exception_class == 'ValueError'
+    assert logged_error.exception_message == "invalid data for file type ('foo bar wrong value')"
 
 
 def test_form_file_field_prefill(pub):
@@ -491,6 +492,72 @@ def test_form_file_field_prefill(pub):
     assert formdata.data['0'].get_content().startswith(b'\x89PNG')
 
 
+@responses.activate
+def test_form_file_field_dict_prefill(pub):
+    NamedWsCall.wipe()
+    wscall = NamedWsCall()
+    wscall.name = 'Hello'
+    wscall.request = {'url': 'http://example.net'}
+    wscall.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.FileField(
+            id='0',
+            label='file',
+            prefill={'type': 'string', 'value': '{{ webservice.hello }}'},
+        )
+    ]
+    formdef.store()
+
+    responses.get(
+        'http://example.net',
+        json={'b64_content': 'aGVsbG8K', 'filename': 'hello.txt', 'content_type': 'text/plain'},
+    )
+    resp = get_app(pub).get('/test/')
+    assert resp.form['f0$token']
+    assert resp.click('hello.txt').content_type == 'text/plain'
+    resp = resp.form.submit('submit')  # -> validation
+    resp = resp.form.submit('submit')  # -> submit
+    formdata = formdef.data_class().select()[0]
+    assert formdata.data['0'].base_filename == 'hello.txt'
+    assert formdata.data['0'].get_content() == b'hello\n'
+
+
+@responses.activate
+def test_form_file_field_url_prefill(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.FileField(
+            id='0',
+            label='file',
+            prefill={'type': 'string', 'value': 'http://example.net/hello.txt'},
+        )
+    ]
+    formdef.store()
+
+    responses.get('http://example.net/hello.txt', body=b'Hello\n', content_type='text/plain')
+    resp = get_app(pub).get('/test/')
+    assert resp.form['f0$token'].value
+    assert resp.click('hello.txt').content_type == 'text/plain'
+    resp = resp.form.submit('submit')  # -> validation
+    resp = resp.form.submit('submit')  # -> submit
+    formdata = formdef.data_class().select()[0]
+    assert formdata.data['0'].base_filename == 'hello.txt'
+    assert formdata.data['0'].get_content() == b'Hello\n'
+
+    pub.loggederror_class.wipe()
+    responses.get('http://example.net/hello.txt', status=404)
+    resp = get_app(pub).get('/test/')
+    assert not resp.form['f0$token'].value
+    assert 'hello.txt' not in resp.text
+    assert [x.summary for x in pub.loggederror_class.select()] == ['Failed to convert value for field "file"']
+
+
 SVG_CONTENT = b'''<?xml version="1.0" encoding="utf-8"?>
 <svg version="1.1" id="Calque_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
 viewBox="0 0 63.72 64.25" style="enable-background:new 0 0 63.72 64.25;" xml:space="preserve"> <g> </g> </svg>'''
@@ -592,3 +659,23 @@ def test_file_download_url_on_wrong_field(pub):
     resp = resp.form.submit('submit').follow()  # -> submit
     formdata = formdef.data_class().select()[0]
     app.get(formdata.get_url() + 'files/1/', status=404)
+
+
+def test_file_auto_convert_heic(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [fields.FileField(id='0', label='field label')]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    with open(os.path.join(os.path.dirname(__file__), '..', 'image.heic'), 'rb') as fd:
+        upload = Upload('image.heic', fd.read(), 'image/heic')
+
+    resp = get_app(pub).get('/test/')
+    resp.forms[0]['f0$file'] = upload
+    resp = resp.forms[0].submit('submit')  # -> validation
+    resp = resp.forms[0].submit('submit')  # -> submit
+    resp = resp.follow()
+    assert resp.click('image.jpeg').follow().content_type == 'image/jpeg'
+    assert b'JFIF' in resp.click('image.jpeg').follow().body

tests/test_carddef.py

@@ -624,6 +624,7 @@ def test_data_source_custom_view_digest(pub):
         'custom-view:view': '{{ form_var_foo }} Foo Bar',
     }
     carddef.store()
+    pub.reset_caches()
     # rebuild digests
     carddata.store()
     carddata2.store()
@@ -761,6 +762,7 @@ def test_get_data_source_custom_view_order_by(pub):
     ]
     carddef.digest_templates['custom-view:view'] = '{{ form_var_bar }}'
     carddef.store()
+    pub.reset_caches()
     for carddata in carddef.data_class().select():
         carddata.store()  # rebuild digests
     assert [i['text'] for i in CardDef.get_data_source_items('carddef:foo:view')] == [
@@ -1386,6 +1388,7 @@ def test_card_update_related(pub):
         ItemsField(id='1', label='Test', data_source={'type': 'carddef:foo'}),
     ]
     formdef.store()
+    pub.reset_caches()
 
     formdata = formdef.data_class()()
     formdata.data = {'1': ['1', '2']}
@@ -1419,6 +1422,7 @@ def test_card_update_related(pub):
         BlockField(id='2', label='Test2', block_slug=blockdef.slug),  # left empty
     ]
     formdef.store()
+    pub.reset_caches()
 
     formdata = formdef.data_class()()
     formdata.data = {

tests/test_content_snapshots.py

@@ -596,6 +596,26 @@ def test_backoffice_show_history(pub, user, formdef_class):
     }
     evo.add_part(part4)
     formdata.store()
+    part5 = ContentSnapshotPart(formdata=formdata, old_data=copy.deepcopy(part4.new_data))
+    part5.new_data = {
+        '1': 'reset',
+        '2': 'foo bar blah',
+        '3': 'foo@bar.com',
+        '4': True,
+        '6': time.strptime('2022-11-06', '%Y-%m-%d'),
+        '7': 'b',
+        '7_display': 'b',
+        '8': ['a', 'b'],
+        '8_display': 'a, b',
+        '9': '1.5;2.26',
+        '10': {'cleartext': 'fooo'},
+        '11': 'computed',
+        # bad format, 12 is a block field
+        '12': 'foobar',
+        'bo1': 'foobar',
+    }
+    evo.add_part(part5)
+    formdata.store()
 
     app = login(get_app(pub))
     resp = app.get(formdata.get_backoffice_url())
@@ -741,6 +761,25 @@ def test_backoffice_show_history(pub, user, formdef_class):
     assert len(resp.pyquery('%s tr[data-field-id="12"]' % table4)) == 0
     assert len(resp.pyquery('%s tr[data-field-id="bo1"]' % table4)) == 0
 
+    assert resp.pyquery(
+        '#evolutions fieldset[data-datetime="%s"] legend' % part5.datetime.isoformat()
+    ).text() == 'changed at %s' % localtime(part5.datetime).strftime('%Y-%m-%d %H:%M')
+    table4 = '#evolutions table[data-datetime="%s"]' % part5.datetime.isoformat()
+    assert len(resp.pyquery('%s tr[data-field-id="1"]' % table4)) == 0
+    assert len(resp.pyquery('%s tr[data-field-id="2"]' % table4)) == 0
+    assert len(resp.pyquery('%s tr[data-field-id="3"]' % table4)) == 0
+    assert len(resp.pyquery('%s tr[data-field-id="4"]' % table4)) == 0
+    assert len(resp.pyquery('%s tr[data-field-id="5"]' % table4)) == 0
+    assert len(resp.pyquery('%s tr[data-field-id="6"]' % table4)) == 0
+    assert len(resp.pyquery('%s tr[data-field-id="7"]' % table4)) == 0
+    assert len(resp.pyquery('%s tr[data-field-id="8"]' % table4)) == 0
+    assert len(resp.pyquery('%s tr[data-field-id="9"]' % table4)) == 0
+    assert len(resp.pyquery('%s tr[data-field-id="10"]' % table4)) == 0
+    assert len(resp.pyquery('%s tr[data-field-id="11"]' % table4)) == 0
+    assert resp.pyquery('%s tr[data-field-id="12"] td' % table3).text() == 'Block'
+    assert len(resp.pyquery('%s tr[data-block-id="12"]' % table3)) == 2
+    assert len(resp.pyquery('%s tr[data-field-id="bo1"]' % table4)) == 0
+
     # check user display
     part5 = ContentSnapshotPart(formdata=formdata, old_data=copy.deepcopy(part4.new_data))
     part5.new_data = copy.deepcopy(part4.new_data)

tests/test_formdata.py

@@ -1347,6 +1347,14 @@ def test_objects_filter(pub):
     tmpl = Template('{{forms|objects:"form"|count}}')
     assert tmpl.render(context) == '1'
 
+    pub.loggederror_class.wipe()
+    context = pub.substitutions.get_context_variables(mode='lazy')
+    tmpl = Template('{{forms|objects:"form"|first|count}}')
+    assert tmpl.render(context) == '0'
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.summary == '|count used on uncountable value'
+
     # called on invalid object
     pub.loggederror_class.wipe()
     tmpl = Template('{{xxx|objects:"form"|count}}')
@@ -1942,14 +1950,14 @@ def test_lazy_formdata_queryset_filter(pub, variable_test_data):
     assert tmpl.render(context) == 'None'
     assert pub.loggederror_class.count() == 1
     logged_error = pub.loggederror_class.select()[0]
-    assert logged_error.summary == '|pending used on invalid queryset (\'\')'
+    assert logged_error.summary == '|pending used on something else than a queryset (\'\')'
 
     pub.loggederror_class.wipe()
     tmpl = Template('{{""|filter_value:"foo"}}')
     assert tmpl.render(context) == 'None'
     assert pub.loggederror_class.count() == 1
     logged_error = pub.loggederror_class.select()[0]
-    assert logged_error.summary == '|filter_value used on invalid queryset (\'\')'
+    assert logged_error.summary == '|filter_value used on something else than a queryset (\'\')'
 
 
 def test_lazy_formdata_queryset_filter_non_unique_varname(pub, variable_test_data):
@@ -2241,10 +2249,15 @@ def test_lazy_global_forms(pub):
     )
     assert tmpl.render(context) == '7,8,9,10,'
 
+    pub.loggederror_class.wipe()
     tmpl = Template('{{forms|objects:"foobarlazy"|with_custom_view:"private-form-view"|count}}')
     assert tmpl.render(context) == '0'
+    assert [x.summary for x in pub.loggederror_class.select()] == ['Unknown custom view "private-form-view"']
+
+    pub.loggederror_class.wipe()
     tmpl = Template('{{forms|objects:"foobarlazy"|with_custom_view:"unknown"|count}}')
     assert tmpl.render(context) == '0'
+    assert [x.summary for x in pub.loggederror_class.select()] == ['Unknown custom view "unknown"']
 
     custom_view4 = pub.custom_view_class()
     custom_view4.title = 'unknown filter'
@@ -2253,6 +2266,8 @@ def test_lazy_global_forms(pub):
     custom_view4.filters = {'filter-42': 'on', 'filter-42-value': 'foo', 'filter-foobar': 'baz'}
     custom_view4.visibility = 'any'
     custom_view4.store()
+
+    pub.loggederror_class.wipe()
     tmpl = Template('{{forms|objects:"foobarlazy"|with_custom_view:"unknown-filter"|count}}')
     assert tmpl.render(context) == '0'
     assert pub.loggederror_class.count() == 2
@@ -4705,6 +4720,7 @@ def test_formdata_filtering_on_block_fields(pub):
         fields.DateField(id='4', label='Date', varname='date'),
         fields.EmailField(id='5', label='Email', varname='email'),
         fields.TextField(id='6', label='Text', varname='text'),
+        fields.FileField(id='7', label='File', varname='file'),
     ]
     block.store()
 
@@ -4719,6 +4735,10 @@ def test_formdata_filtering_on_block_fields(pub):
     data_class = formdef.data_class()
     data_class.wipe()
 
+    upload = PicklableUpload('test.jpeg', 'image/jpeg')
+    with open(os.path.join(os.path.dirname(__file__), 'image-with-gps-data.jpeg'), 'rb') as fd:
+        upload.receive([fd.read()])
+
     for i in range(14):
         formdata = data_class()
         formdata.data = {
@@ -5048,6 +5068,10 @@ def test_formdata_filtering_on_block_fields(pub):
             tmpl = Template('{{forms|objects:"test"|filter_by:"blockdata_text"|%s|count}}' % operator)
         assert tmpl.render(context) == result
 
+    # file
+    tmpl = Template('{{forms|objects:"test"|filter_by:"blockdata_file"|absent|count}}')
+    assert tmpl.render(context) == '0'
+
 
 def test_items_field_getlist(pub):
     NamedDataSource.wipe()
@@ -5760,6 +5784,7 @@ def test_reverse_links(pub):
 
     # test reverse relation
     carddef1.store()  # build & store reverse_relations
+    pub.reset_caches()
     pub.substitutions.reset()
     pub.substitutions.feed(pub)
     pub.substitutions.feed(carddef1)
@@ -5774,6 +5799,7 @@ def test_reverse_links(pub):
     # test with natural id
     carddef1.id_template = 'X{{ form_var_name1 }}Y'
     carddef1.store()
+    pub.reset_caches()
     carddata1.store()
     assert carddata1.id_display == 'Xfoo1Y'
     carddata2.data['1'] = carddata1.get_natural_key()

tests/test_formdef.py

@@ -647,7 +647,7 @@ def test_wipe_on_object(pub):
         formdef.wipe()
 
 
-def test_update_storage_all_formdefs(pub):
+def test_update_storage_all_formdefs(pub, capfd):
     CardDef.wipe()
     FormDef.wipe()
 
@@ -664,6 +664,18 @@ def test_update_storage_all_formdefs(pub):
         update_storage_all_formdefs(pub)
         assert update_storage.call_count == 10
 
+    assert not capfd.readouterr().out
+
+    formdef = FormDef()
+    formdef.name = 'broken formdef'
+    formdef.fields = [StringField(id='1', label='Test')]
+    formdef.store()
+    formdef.fields = [DateField(id='1', label='Test')]
+    formdef.store()
+
+    update_storage_all_formdefs(pub)
+    assert capfd.readouterr().out == '! Integrity errors in %s\n' % formdef.get_admin_url()
+
 
 def test_lazy_formdef(pub):
     FormDef.wipe()

tests/test_formdef_import.py

@@ -87,7 +87,7 @@ def test_empty_display_locations_tag(pub):
     formdef = FormDef()
     formdef.name = 'Foo'
     formdef.fields = [
-        fields.TitleField(label='title', display_locations=[]),
+        fields.TitleField(label='title', display_locations=None),
         fields.SubtitleField(label='subtitle', display_locations=[]),
         fields.TextField(label='string', display_locations=[]),
     ]

tests/test_http_request.py

@@ -0,0 +1,49 @@
+import pytest
+
+from wcs.qommon.http_request import HTTPRequest
+from wcs.variables import LazyRequest
+
+from .utilities import clean_temporary_pub, create_temporary_pub
+
+
+@pytest.fixture
+def pub():
+    return create_temporary_pub()
+
+
+def teardown_module(module):
+    clean_temporary_pub()
+
+
+def test_is_in_backoffice(pub):
+    req = HTTPRequest(None, {'SCRIPT_NAME': '/', 'SERVER_NAME': 'example.net'})
+    assert not req.is_in_backoffice()
+    assert not LazyRequest(req).is_in_backoffice
+    req = HTTPRequest(None, {'SCRIPT_NAME': '/backoffice/test', 'SERVER_NAME': 'example.net'})
+    assert req.is_in_backoffice()
+    assert LazyRequest(req).is_in_backoffice
+
+
+def test_is_from_mobile(pub):
+    req = HTTPRequest(None, {'SCRIPT_NAME': '/', 'SERVER_NAME': 'example.net'})
+    assert not req.is_from_mobile()
+    assert not LazyRequest(req).is_from_mobile
+    req = HTTPRequest(None, {'SCRIPT_NAME': '/', 'SERVER_NAME': 'example.net', 'HTTP_USER_AGENT': 'bot/1.0'})
+    assert not req.is_from_mobile()
+    assert not LazyRequest(req).is_from_mobile
+    req = HTTPRequest(
+        None,
+        {'SCRIPT_NAME': '/', 'SERVER_NAME': 'example.net', 'HTTP_USER_AGENT': 'Mozilla/5.0 (Mobile) plop'},
+    )
+    assert req.is_from_mobile()
+    assert LazyRequest(req).is_from_mobile
+    req = HTTPRequest(
+        None,
+        {
+            'SCRIPT_NAME': '/',
+            'SERVER_NAME': 'example.net',
+            'HTTP_USER_AGENT': 'Mozilla/5.0 (Chrome) Mobile Safari',
+        },
+    )
+    assert req.is_from_mobile()
+    assert LazyRequest(req).is_from_mobile

tests/test_misc.py

@@ -20,7 +20,7 @@ from wcs.fields import StringField
 from wcs.qommon import evalutils, force_str
 from wcs.qommon.form import FileSizeWidget
 from wcs.qommon.http_request import HTTPRequest
-from wcs.qommon.humantime import humanduration2seconds, seconds2humanduration
+from wcs.qommon.humantime import humanduration2seconds, seconds2humanduration, timewords
 from wcs.qommon.misc import (
     _http_request,
     date_format,
@@ -108,6 +108,10 @@ def test_humantime_short(seconds, expected):
     assert seconds2humanduration(seconds, short=True) == expected
 
 
+def test_humantime_timewords():
+    assert timewords() == ['day(s)', 'hour(s)', 'minute(s)', 'second(s)', 'month(s)', 'year(s)']
+
+
 def test_parse_mimetypes():
     assert FileTypesDirectory.parse_mimetypes('application/pdf') == ['application/pdf']
     assert FileTypesDirectory.parse_mimetypes('.pdf') == ['application/pdf']

tests/test_publisher.py

@@ -26,7 +26,7 @@ from wcs.qommon.afterjobs import AfterJob
 from wcs.qommon.cron import CronJob
 from wcs.qommon.form import UploadedFile
 from wcs.qommon.http_request import HTTPRequest
-from wcs.qommon.publisher import Tenant
+from wcs.qommon.publisher import MaxSizeDict, Tenant
 from wcs.workflows import Workflow
 
 from .utilities import clean_temporary_pub, create_temporary_pub
@@ -526,6 +526,65 @@ def test_cron_command_rewind_jobs(settings, freezer):
         assert sorted(jobs) == ['job1', 'job2', 'job3']
 
 
+def test_cron_command_job_exception(settings):
+    create_temporary_pub()
+
+    def job1(pub, job=None):
+        raise Exception('Error')
+
+    @classmethod
+    def register_test_cronjobs(cls):
+        cls.register_cronjob(CronJob(job1, name='job1', days=[10]))
+
+    get_publisher().set_tenant_by_hostname('example.net')
+    sql.mark_cron_status('done')
+
+    with mock.patch('wcs.publisher.WcsPublisher.register_cronjobs', register_test_cronjobs):
+        get_publisher_class().cronjobs = []
+        clear_log_files()
+        call_command('cron', job_name='job1', domain='example.net')
+        assert get_logs('example.net') == [
+            'start',
+            "running jobs: ['job1']",
+            'exception running job job1: Error',
+        ]
+
+    clean_temporary_pub()
+
+
+def test_cron_command_job_log(settings):
+    pub = create_temporary_pub()
+
+    def job1(pub, job=None):
+        job.log('hello')
+        job.log_debug('debug')
+
+    @classmethod
+    def register_test_cronjobs(cls):
+        cls.register_cronjob(CronJob(job1, name='job1', days=[10]))
+
+    get_publisher().set_tenant_by_hostname('example.net')
+    sql.mark_cron_status('done')
+
+    with mock.patch('wcs.publisher.WcsPublisher.register_cronjobs', register_test_cronjobs):
+        get_publisher_class().cronjobs = []
+        clear_log_files()
+        call_command('cron', job_name='job1', domain='example.net')
+        assert get_logs('example.net') == ['start', "running jobs: ['job1']", 'hello']
+
+        pub.load_site_options()
+        pub.site_options.set('options', 'cron-log-level', 'debug')
+        with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+            pub.site_options.write(fd)
+
+        clear_log_files()
+        call_command('cron', job_name='job1', domain='example.net')
+        assert get_logs('example.net')[:3] == ['start', "running jobs: ['job1']", 'hello']
+        assert re.match(r'\(mem: .*\) debug', get_logs('example.net')[3])
+
+    clean_temporary_pub()
+
+
 def test_clean_afterjobs():
     pub = create_temporary_pub()
 
@@ -736,3 +795,17 @@ def test_get_site_language():
 
     req.environ['HTTP_ACCEPT_LANGUAGE'] = 'xy,fr,en;q=0.7,es;q=0.3'
     assert pub.get_site_language() == 'fr'
+
+
+def test_maxsize_dict():
+    d = MaxSizeDict()
+    with pytest.raises(KeyError):
+        d['a']  # noqa pylint: disable=pointless-statement
+    for i in range(256):
+        d[str(i)] = f'i : {i}'
+        try:
+            assert d['10']  # keep accessing low value
+        except KeyError:
+            pass
+    # kept keys are the recently added one + '10' that we kept accessing
+    assert set(d.keys()) == set(['10'] + [str(x) for x in range(129, 256)])

tests/test_snapshots.py

@@ -2,6 +2,7 @@ import io
 import os
 import shutil
 import xml.etree.ElementTree as ET
+from unittest import mock
 
 import pytest
 from quixote.http_request import Upload
@@ -189,6 +190,14 @@ def test_snapshot_instance(pub):
     snapshots = pub.snapshot_class.select_object_history(carddef)
     assert len(snapshots) == 1
 
+    # check that DeprecationsScan is not run on instance load
+    with mock.patch(
+        'wcs.backoffice.deprecations.DeprecationsScan.check_deprecated_elements_in_object'
+    ) as check:
+        snapshot = pub.snapshot_class.get_latest('formdef', formdef.id)
+        assert snapshot.instance
+        assert check.call_args_list == []
+
 
 def test_snapshot_user(pub):
     user = pub.user_class()

tests/test_sql.py

@@ -17,10 +17,12 @@ import wcs.sql_criterias as st
 from wcs import fields, sql
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
+from wcs.categories import Category
 from wcs.data_sources import NamedDataSource
 from wcs.formdata import Evolution
 from wcs.formdef import FormDef
 from wcs.qommon import force_str
+from wcs.testdef import TestDef
 from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem
 from wcs.workflows import (
     ActionsTracingEvolutionPart,
@@ -1183,6 +1185,45 @@ def test_sql_criteria_fts(pub):
     assert data_class.select([st.FtsMatch(formdata1.id_display)])[0].id_display == formdata1.id_display
 
 
+def test_search_tokens_purge(pub):
+    _, cur = sql.get_connection_and_cursor()
+
+    # purge garbage from other tests
+    sql.purge_obsolete_search_tokens()
+
+    cur.execute('SELECT count(*) FROM wcs_search_tokens;')
+    start = cur.fetchone()[0]
+
+    # define a new table
+    test_formdef = FormDef()
+    test_formdef.name = 'tableSelectFTStokens'
+    test_formdef.fields = [fields.StringField(id='3', label='string')]
+    test_formdef.store()
+    data_class = test_formdef.data_class(mode='sql')
+
+    cur.execute('SELECT count(*) FROM wcs_search_tokens;')
+    assert cur.fetchone()[0] == start + 1
+
+    t = data_class()
+    t.data = {'3': 'foofortokensofcourse'}
+    t.just_created()
+    t.store()
+
+    cur.execute('SELECT count(*) FROM wcs_search_tokens;')
+    assert cur.fetchone()[0] == start + 2
+
+    t.data = {'3': 'chaussettefortokensofcourse'}
+    t.store()
+
+    cur.execute('SELECT count(*) FROM wcs_search_tokens;')
+    assert cur.fetchone()[0] == start + 3
+
+    sql.purge_obsolete_search_tokens()
+
+    cur.execute('SELECT count(*) FROM wcs_search_tokens;')
+    assert cur.fetchone()[0] == start + 2
+
+
 def table_exists(cur, table_name):
     cur.execute(
         '''SELECT COUNT(*) FROM information_schema.tables
@@ -1576,6 +1617,51 @@ def test_all_forms_user_name_change(pub, formdef):
     conn.commit()
 
 
+def test_all_forms_category_change(pub, formdef):
+    Category.wipe()
+    FormDef.wipe()
+
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.store()
+
+    conn, cur = sql.get_connection_and_cursor()
+    cur.execute('SELECT category_id FROM wcs_all_forms WHERE formdef_id = %s', (formdef.id,))
+    row = cur.fetchone()
+    assert row[0] is None
+
+    category = Category()
+    category.name = 'Test'
+    category.store()
+
+    formdef.category_id = category.id
+    formdef.store()
+    cur.execute('SELECT category_id FROM wcs_all_forms WHERE formdef_id = %s', (formdef.id,))
+    row = cur.fetchone()
+    assert row[0] == int(category.id)
+
+    category2 = Category()
+    category2.name = 'Test2'
+    category2.store()
+    formdef.category_id = category2.id
+    formdef.store()
+    cur.execute('SELECT category_id FROM wcs_all_forms WHERE formdef_id = %s', (formdef.id,))
+    row = cur.fetchone()
+    assert row[0] == int(category2.id)
+
+    formdef.category_id = None
+    formdef.store()
+    cur.execute('SELECT category_id FROM wcs_all_forms WHERE formdef_id = %s', (formdef.id,))
+    row = cur.fetchone()
+    assert row[0] is None
+
+    cur.close()
+    conn.commit()
+
+
 def test_views_fts(pub):
     drop_formdef_tables()
     _, cur = sql.get_connection_and_cursor()
@@ -2375,7 +2461,7 @@ def test_migration_59_all_forms_table(pub):
     formdata.store()
 
     conn, cur = sql.get_connection_and_cursor()
-    cur.execute('DROP TABLE wcs_all_forms')
+    cur.execute('DROP TABLE wcs_all_forms CASCADE')
     cur.execute(
         'DROP TRIGGER %s ON %s' % (sql.get_formdef_trigger_name(formdef), sql.get_formdef_table_name(formdef))
     )
@@ -2937,3 +3023,77 @@ def test_sql_data_views(pub_with_views, formdef_class):
     assert column_exists_in_table(cur, f'{prefix}_test', 'geoloc_base_x')
     conn.commit()
     cur.close()
+
+
+def test_sql_integrity_errors(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.StringField(id='1', label='string'),
+    ]
+    formdef.store()
+    assert not formdef.sql_integrity_errors
+
+    formdef.fields = [
+        fields.FileField(id='1', label='string'),
+    ]
+    formdef.store()
+    assert formdef.sql_integrity_errors == {'1': {'got': 'character varying', 'expected': 'bytea'}}
+
+
+def test_testdef_user_uuid_migration(pub):
+    pub.user_class.wipe()
+
+    user = pub.user_class(name='new user')
+    user.email = 'new@example.com'
+    user.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.user_id = user.id
+
+    testdef = TestDef()
+    testdef.name = 'First test'
+    testdef.object_type = formdef.get_table_name()
+    testdef.object_id = formdef.id
+    testdef.data = {
+        'data': [],
+        'user': formdata.user.get_json_export_dict(),
+    }
+    testdef.store()
+
+    testdef2 = TestDef()
+    testdef2.name = 'First test'
+    testdef2.object_type = formdef.get_table_name()
+    testdef2.object_id = formdef.id
+    testdef2.data = {
+        'data': [],
+        'user': formdata.user.get_json_export_dict(),
+    }
+    testdef2.store()
+
+    conn, cur = sql.get_connection_and_cursor()
+    cur.execute('UPDATE wcs_meta SET value = 106 WHERE key = %s', ('sql_level',))
+
+    sql.migrate()
+    assert sql.is_reindex_needed('testdef', conn=conn, cur=cur) is True
+    assert pub.user_class.count() == 1
+    conn.commit()
+    cur.close()
+    sql.reindex()
+
+    assert pub.user_class.count() == 2
+    test_user = pub.user_class.select([st.NotNull('test_uuid')])[0]
+
+    testdef = TestDef.get(testdef.id)
+    assert not 'user' in testdef.data
+    assert testdef.user_uuid == test_user.test_uuid
+
+    testdef2 = TestDef.get(testdef2.id)
+    assert not 'user' in testdef2.data
+    assert testdef2.user_uuid == test_user.test_uuid

tests/test_storage.py

@@ -31,6 +31,15 @@ class Foobar(StorableObject):
     unique_value = None
 
 
+class Foobar2(StorableObject):
+    _names = 'tests%s' % random.randint(0, 100000)
+    _indexes = ['unique_value']
+    _hashed_indexes = ['value']
+
+    value = None
+    unique_value = None
+
+
 def test_store():
     test = Foobar()
     test.value = 'value'
@@ -307,3 +316,34 @@ def test_umask():
     cache_umask()
     test.store()
     assert (os.stat(test.get_object_filename()).st_mode % 0o1000) == 0o664
+
+
+def test_publisher_cache():
+    pub.reset_caches()
+
+    Foobar.wipe()
+    Foobar2.wipe()
+
+    test = Foobar()
+    test.value = 'value'
+    test.unique_value = 'unique-value'
+    test.store()
+
+    test2 = Foobar2()
+    test2.value = 'value'
+    test2.unique_value = 'unique-value'
+    test2.store()
+
+    test = Foobar.cached_get('1')
+    assert test.value == 'value'
+    assert Foobar.cached_get('1') is test  # same object
+
+    assert Foobar.get_on_index('unique-value', 'unique_value') is not test
+    assert Foobar.get_on_index('unique-value', 'unique_value', use_cache=True) is test
+
+    assert Foobar2.cached_get('1') is not test
+    assert Foobar2.cached_get('1') is Foobar2.get_on_index('unique-value', 'unique_value', use_cache=True)
+
+    with pytest.raises(KeyError):
+        Foobar2.get_on_index('unique-value', 'invalid', use_cache=True)
+    assert Foobar2.get_on_index('unique-value', 'invalid', use_cache=True, ignore_errors=True) is None

tests/test_templates.py

@@ -1819,3 +1819,22 @@ def test_temporary_access_url(pub):
     # removed formdata
     formdata.remove_self()
     assert Template('{% temporary_access_url %}').render(context) == ''
+
+
+def test_housenumber_templatefilters(pub):
+    assert Template('{{ "42"|housenumber_number }}').render() == '42'
+    assert Template('{{ "42"|housenumber_btq }}').render() == ''
+    assert Template('{{ "42bis"|housenumber_number }}').render() == '42'
+    assert Template('{{ "42bis"|housenumber_btq }}').render() == 'bis'
+    assert Template('{{ "  42  bis  "|housenumber_number }}').render() == '42'
+    assert Template('{{ "  42  bis  "|housenumber_btq }}').render() == 'bis'
+    assert Template('{{ "42 3 t "|housenumber_number }}').render() == '42'
+    assert Template('{{ "42 3 t "|housenumber_btq }}').render() == '3 t'
+    assert Template('{{ " bis "|housenumber_number }}').render() == ''
+    assert Template('{{ " bis "|housenumber_btq }}').render() == ''
+    assert Template('{{ 42|housenumber_number }}').render() == '42'
+    assert Template('{{ 42|housenumber_btq }}').render() == ''
+    assert Template('{{ ""|housenumber_number }}').render() == ''
+    assert Template('{{ ""|housenumber_btq }}').render() == ''
+    assert Template('{{ null|housenumber_number }}').render({'null': None}) == ''
+    assert Template('{{ null|housenumber_btq }}').render({'null': None}) == ''

tests/test_testdef.py

@@ -81,7 +81,7 @@ def test_testdef_export_to_xml(pub):
     assert testdef2.name == 'test'
     assert testdef2.object_type == 'formdefs'
     assert testdef2.object_id == str(formdef.id)
-    assert testdef2.data == {'fields': {'1': ['foo', 'baz'], '2': True}, 'user': None}
+    assert testdef2.data == {'fields': {'1': ['foo', 'baz'], '2': True}}
     assert testdef2.expected_error == 'xxx'
     assert testdef2.is_in_backoffice is False
 
@@ -1271,6 +1271,49 @@ def test_computed_field_forms_template_access(pub):
     assert testdef.recorded_errors == ['Invalid filter "unknown"']
 
 
+def test_numeric_field_support(pub):
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.fields = [
+        fields.PageField(
+            id='0',
+            label='1st page',
+            post_conditions=[
+                {'condition': {'type': 'django', 'value': 'form_var_foo == 13.12'}, 'error_message': ''}
+            ],
+        ),
+        fields.NumericField(
+            id='1', label='Numeric', varname='foo', restrict_to_integers=False, min_value=decimal.Decimal(10)
+        ),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.data['1'] = decimal.Decimal(13.12)
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.store()
+    testdef.run(formdef)
+
+    formdata.data['1'] = decimal.Decimal(9)
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+
+    with pytest.raises(TestError) as excinfo:
+        testdef.run(formdef)
+    assert (
+        str(excinfo.value)
+        == 'Invalid value "9" for field "Numeric": You should enter a number greater than or equal to 10.'
+    )
+
+    formdata.data['1'] = decimal.Decimal(42)
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+
+    with pytest.raises(TestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Page 1 post condition was not met (form_var_foo == 13.12).'
+
+
 def test_expected_error(pub):
     formdef = FormDef()
     formdef.name = 'test title'

tests/test_users.py

@@ -330,3 +330,24 @@ def test_clean_deleted_users(pub):
     call_command('cron', job_name='clean_deleted_users', domain='example.net')
 
     assert User.count() == 0
+
+
+def test_normal_users_test_users_isolation(pub):
+    pub.user_class.wipe()
+
+    user = pub.user_class()
+    user.name = 'Jean'
+    user.email = 'jean@example.com'
+    user.store()
+
+    user = pub.user_class()
+    user.name = 'Jean'
+    user.email = 'jean@example.com'
+    user.test_uuid = '42'
+    user.store()
+
+    assert len(pub.user_class.select()) == 1
+    assert pub.user_class.select()[0].test_uuid is None
+
+    assert len(pub.user_class.get_users_with_email('jean@example.com')) == 1
+    assert pub.user_class.get_users_with_email('jean@example.com')[0].test_uuid is None

tests/test_workflow_import.py

@@ -89,6 +89,17 @@ def test_status_forced_endpoint(pub):
     assert wf2.possible_status[1].forced_endpoint is False
 
 
+def test_status_with_loop(pub):
+    wf = Workflow(name='status')
+    st1 = wf.add_status('Status1', 'st1')
+    st2 = wf.add_status('Status2', 'st2')
+    st1.loop_items_template = '{{ "abc"|make_list }}'
+    st1.after_loop_status = str(st2.id)
+    wf2 = assert_import_export_works(wf)
+    assert wf2.possible_status[0].loop_items_template == '{{ "abc"|make_list }}'
+    assert wf2.possible_status[0].after_loop_status == wf2.possible_status[1].id
+
+
 def test_default_wf(pub):
     wf = Workflow.get_default_workflow()
     assert_import_export_works(wf)
@@ -494,9 +505,12 @@ def test_backoffice_fields(pub):
     wf = Workflow(name='bo fields')
     wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
     wf.backoffice_fields_formdef.fields = [
-        StringField(id='bo1', label='1st backoffice field', varname='backoffice_blah'),
+        StringField(
+            id='bo1', label='1st backoffice field', varname='backoffice_blah', display_locations=None
+        ),
     ]
-    assert_import_export_works(wf, True)
+    wf2 = assert_import_export_works(wf)
+    assert wf2.backoffice_fields_formdef.fields[0].display_locations == []
 
 
 def test_complex_dispatch_action(pub):

tests/test_workflow_tests.py

@@ -17,7 +17,7 @@ from wcs.workflows import (
 )
 
 from .backoffice_pages.test_all import create_user
-from .utilities import create_temporary_pub, get_app, login
+from .utilities import clean_temporary_pub, create_temporary_pub, get_app, login
 
 
 @pytest.fixture
@@ -36,10 +36,11 @@ def pub():
     return pub
 
 
-def test_workflow_tests_ignore_unsupported_items(pub, monkeypatch):
-    user = pub.user_class(name='test user')
-    user.store()
+def teardown_module(module):
+    clean_temporary_pub()
 
+
+def test_workflow_tests_ignore_unsupported_items(pub, monkeypatch):
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')
     end_status = workflow.add_status(name='End status')
@@ -58,7 +59,6 @@ def test_workflow_tests_ignore_unsupported_items(pub, monkeypatch):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
     testdef.workflow_tests.actions = [
         workflow_tests.AssertStatus(status_name='End status'),
     ]
@@ -73,9 +73,6 @@ def test_workflow_tests_ignore_unsupported_items(pub, monkeypatch):
 
 
 def test_workflow_tests_no_actions(pub):
-    user = pub.user_class(name='test user')
-    user.store()
-
     workflow = Workflow(name='Workflow One')
     workflow.add_status(name='New status')
 
@@ -90,7 +87,6 @@ def test_workflow_tests_no_actions(pub):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
     testdef.workflow_tests.actions = []
 
     with mock.patch('wcs.workflow_tests.WorkflowTests.run') as mocked_run:
@@ -99,9 +95,6 @@ def test_workflow_tests_no_actions(pub):
 
 
 def test_workflow_tests_action_not_configured(pub):
-    user = pub.user_class(name='test user')
-    user.store()
-
     workflow = Workflow(name='Workflow One')
     workflow.add_status(name='New status')
 
@@ -116,7 +109,6 @@ def test_workflow_tests_action_not_configured(pub):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
     testdef.workflow_tests.actions = [
         workflow_tests.ButtonClick(),
     ]
@@ -147,6 +139,7 @@ def test_workflow_tests_button_click(pub):
     role = pub.role_class(name='test role')
     role.store()
     user = pub.user_class(name='test user')
+    user.test_uuid = '42'
     user.roles = [role.id]
     user.store()
 
@@ -170,7 +163,7 @@ def test_workflow_tests_button_click(pub):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
+    testdef.agent_id = user.test_uuid
     testdef.workflow_tests.actions = [
         workflow_tests.ButtonClick(button_name='Go to end status'),
         workflow_tests.AssertStatus(status_name='End status'),
@@ -213,6 +206,7 @@ def test_workflow_tests_button_click_global_action(pub):
     role = pub.role_class(name='test role')
     role.store()
     user = pub.user_class(name='test user')
+    user.test_uuid = '42'
     user.roles = [role.id]
     user.store()
 
@@ -242,7 +236,7 @@ def test_workflow_tests_button_click_global_action(pub):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
+    testdef.agent_id = user.test_uuid
 
     testdef.workflow_tests.actions = [
         workflow_tests.AssertStatus(status_name='End status'),
@@ -272,11 +266,13 @@ def test_workflow_tests_button_click_who(pub):
     role = pub.role_class(name='test role')
     role.store()
     agent_user = pub.user_class(name='agent user')
+    agent_user.test_uuid = '42'
     agent_user.roles = [role.id]
     agent_user.store()
     other_role = pub.role_class(name='other test role')
     other_role.store()
     other_user = pub.user_class(name='other user')
+    other_user.test_uuid = '43'
     other_user.roles = [other_role.id]
     other_user.store()
 
@@ -319,13 +315,18 @@ def test_workflow_tests_button_click_who(pub):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = agent_user.id
+    testdef.agent_id = agent_user.test_uuid
     testdef.workflow_tests.actions = [
         workflow_tests.ButtonClick(button_name='Go to next status', who='receiver'),
         workflow_tests.AssertStatus(status_name='Jump by receiver'),
     ]
     testdef.run(formdef)
 
+    testdef.agent_id = None
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Broken, missing user'
+
     testdef.workflow_tests.actions = [
         workflow_tests.ButtonClick(button_name='Go to next status', who='submitter'),
         workflow_tests.AssertStatus(status_name='Jump by submitter'),
@@ -333,7 +334,7 @@ def test_workflow_tests_button_click_who(pub):
     testdef.run(formdef)
 
     testdef.workflow_tests.actions = [
-        workflow_tests.ButtonClick(button_name='Go to next status', who='other', who_id=other_user.id),
+        workflow_tests.ButtonClick(button_name='Go to next status', who='other', who_id=other_user.test_uuid),
         workflow_tests.AssertStatus(status_name='Jump by other user'),
     ]
     testdef.run(formdef)
@@ -364,7 +365,7 @@ def test_workflow_tests_button_click_who(pub):
     formdata.user = submitter_user
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = agent_user.id
+    testdef.agent_id = agent_user.test_uuid
     testdef.workflow_tests.actions = [
         workflow_tests.ButtonClick(button_name='Go to next status', who='submitter'),
         workflow_tests.AssertStatus(status_name='Jump by submitter'),
@@ -373,9 +374,6 @@ def test_workflow_tests_button_click_who(pub):
 
 
 def test_workflow_tests_automatic_jump(pub):
-    user = pub.user_class(name='test user')
-    user.store()
-
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')
     end_status = workflow.add_status(name='End status')
@@ -394,7 +392,6 @@ def test_workflow_tests_automatic_jump(pub):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
     testdef.workflow_tests.actions = [
         workflow_tests.AssertStatus(status_name='End status'),
     ]
@@ -414,9 +411,6 @@ def test_workflow_tests_automatic_jump(pub):
 
 
 def test_workflow_tests_automatic_jump_condition(pub):
-    user = pub.user_class(name='test user')
-    user.store()
-
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')
     frog_status = workflow.add_status(name='Frog status')
@@ -445,7 +439,6 @@ def test_workflow_tests_automatic_jump_condition(pub):
     formdata.data['1'] = 'frog'
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
     testdef.workflow_tests.actions = [
         workflow_tests.AssertStatus(status_name='Frog status'),
     ]
@@ -458,25 +451,11 @@ def test_workflow_tests_automatic_jump_condition(pub):
     assert str(excinfo.value) == 'Form should be in status "Frog status" but is in status "Bear status".'
 
 
-@pytest.mark.freeze_time('2024-02-19 12:00')
 def test_workflow_tests_automatic_jump_timeout(pub):
-    user = pub.user_class(name='test user')
-    user.store()
-
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')
     stalled_status = workflow.add_status(name='Stalled')
 
-    jump = new_status.add_action('jump')
-    jump.status = stalled_status.id
-    jump.timeout = 120 * 60  # 2 hours
-    jump.condition = {'type': 'django', 'value': 'form_receipt_datetime|age_in_days >= 1'}
-
-    sendmail = new_status.add_action('sendmail')
-    sendmail.to = ['test@example.org']
-    sendmail.subject = 'In new status'
-    sendmail.body = 'xxx'
-
     workflow.store()
 
     formdef = FormDef()
@@ -488,7 +467,27 @@ def test_workflow_tests_automatic_jump_timeout(pub):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
+
+    # no jumps configured, try skipping time anyway
+    testdef.workflow_tests.actions = [
+        workflow_tests.SkipTime(seconds=119 * 60),
+    ]
+    testdef.run(formdef)
+
+    # configure jump
+    jump = new_status.add_action('jump')
+    jump.status = stalled_status.id
+    jump.timeout = 120 * 60  # 2 hours
+    jump.condition = {'type': 'django', 'value': 'form_receipt_datetime|age_in_days >= 1'}
+
+    sendmail = new_status.add_action('sendmail')
+    sendmail.to = ['test@example.org']
+    sendmail.subject = 'In new status'
+    sendmail.body = 'xxx'
+
+    workflow.store()
+    formdef.refresh_from_storage()
+
     testdef.workflow_tests.actions = [
         workflow_tests.AssertStatus(status_name='New status'),
         workflow_tests.SkipTime(seconds=119 * 60),
@@ -518,11 +517,101 @@ def test_workflow_tests_automatic_jump_timeout(pub):
     testdef.run(formdef)
 
 
+@pytest.mark.freeze_time('2024-02-19 12:00')
+def test_workflow_tests_global_action_timeout(pub):
+    workflow = Workflow(name='Workflow One')
+    new_status = workflow.add_status(name='New status')
+    end_status = workflow.add_status(name='End status')
+
+    global_action = workflow.add_global_action('Go to end status')
+    trigger = global_action.append_trigger('timeout')
+    trigger.anchor = 'creation'
+    trigger.timeout = 1
+
+    jump = global_action.add_action('jump')
+    jump.status = end_status.id
+
+    # add choice so that new_status is not flagged as endpoint
+    choice = new_status.add_action('choice')
+    choice.label = 'Go to end status'
+    choice.status = end_status.id
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertStatus(status_name='New status'),
+        workflow_tests.SkipTime(seconds=60 * 60),  # 1 hour
+        workflow_tests.AssertStatus(status_name='New status'),
+        workflow_tests.SkipTime(seconds=24 * 60 * 60),  # 1 day
+        workflow_tests.AssertStatus(status_name='End status'),
+    ]
+
+    testdef.run(formdef)
+
+    trigger.anchor = '1st-arrival'
+    workflow.store()
+    formdef.refresh_from_storage()
+
+    testdef.run(formdef)
+
+    trigger.anchor = 'latest-arrival'
+    workflow.store()
+    formdef.refresh_from_storage()
+
+    testdef.run(formdef)
+
+    trigger.anchor = 'template'
+    trigger.anchor_template = '{{ form_receipt_date|date:"Y-m-d" }}'
+    workflow.store()
+    formdef.refresh_from_storage()
+
+    testdef.run(formdef)
+
+    trigger.anchor = 'finalized'
+    workflow.store()
+    formdef.refresh_from_storage()
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Form should be in status "End status" but is in status "New status".'
+
+    # remove choice so new status becomes endpoint
+    new_status.items = [x for x in new_status.items if x.id != choice.id]
+    workflow.store()
+    formdef.refresh_from_storage()
+
+    testdef.run(formdef)
+
+    trigger.anchor = 'anonymisation'
+    workflow.store()
+    formdef.refresh_from_storage()
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Form should be in status "End status" but is in status "New status".'
+
+    new_status.add_action('anonymise')
+    workflow.store()
+    formdef.refresh_from_storage()
+
+    testdef.run(formdef)
+
+
 @mock.patch('wcs.qommon.emails.send_email')
 def test_workflow_tests_sendmail(mocked_send_email, pub):
     role = pub.role_class(name='test role')
     role.store()
     user = pub.user_class(name='test user')
+    user.test_uuid = '42'
     user.roles = [role.id]
     user.store()
 
@@ -556,7 +645,7 @@ def test_workflow_tests_sendmail(mocked_send_email, pub):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
+    testdef.agent_id = user.test_uuid
     testdef.workflow_tests.actions = [
         workflow_tests.AssertEmail(
             addresses=['test@example.org'], subject_strings=['In new status'], body_strings=['xxx']
@@ -614,9 +703,6 @@ def test_workflow_tests_sendmail(mocked_send_email, pub):
 def test_workflow_tests_sms(pub):
     pub.cfg['sms'] = {'sender': 'xxx', 'passerelle_url': 'http://passerelle.invalid/'}
 
-    user = pub.user_class(name='test user')
-    user.store()
-
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')
 
@@ -635,7 +721,12 @@ def test_workflow_tests_sms(pub):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertSMS(),
+    ]
+
+    testdef.run(formdef)
+
     testdef.workflow_tests.actions = [
         workflow_tests.AssertSMS(phone_numbers=['0123456789'], body='Hello'),
     ]
@@ -668,9 +759,6 @@ def test_workflow_tests_sms(pub):
 
 
 def test_workflow_tests_anonymise(pub):
-    user = pub.user_class(name='test user')
-    user.store()
-
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')
     workflow.store()
@@ -684,7 +772,6 @@ def test_workflow_tests_anonymise(pub):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
     testdef.workflow_tests.actions = [
         workflow_tests.AssertAnonymise(),
     ]
@@ -713,9 +800,6 @@ def test_workflow_tests_anonymise(pub):
 
 
 def test_workflow_tests_redirect(pub):
-    user = pub.user_class(name='test user')
-    user.store()
-
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')
     workflow.store()
@@ -729,7 +813,6 @@ def test_workflow_tests_redirect(pub):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
     testdef.workflow_tests.actions = [
         workflow_tests.AssertRedirect(url='https://example.com/'),
     ]
@@ -758,9 +841,6 @@ def test_workflow_tests_redirect(pub):
 
 
 def test_workflow_tests_history_message(pub):
-    user = pub.user_class(name='test user')
-    user.store()
-
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')
     workflow.store()
@@ -774,7 +854,6 @@ def test_workflow_tests_history_message(pub):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
     testdef.workflow_tests.actions = [
         workflow_tests.AssertHistoryMessage(message='Hello 42'),
     ]
@@ -802,9 +881,6 @@ def test_workflow_tests_history_message(pub):
 
 
 def test_workflow_tests_alert(pub):
-    user = pub.user_class(name='test user')
-    user.store()
-
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')
     workflow.store()
@@ -818,7 +894,6 @@ def test_workflow_tests_alert(pub):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
     testdef.workflow_tests.actions = [
         workflow_tests.AssertAlert(message='Hello 42'),
     ]
@@ -848,9 +923,6 @@ def test_workflow_tests_alert(pub):
 
 
 def test_workflow_tests_criticality(pub):
-    user = pub.user_class(name='test user')
-    user.store()
-
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')
     green_level = WorkflowCriticalityLevel(name='green')
@@ -867,7 +939,6 @@ def test_workflow_tests_criticality(pub):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
     testdef.workflow_tests.actions = [
         workflow_tests.AssertCriticality(level_id=red_level.id),
     ]
@@ -892,9 +963,6 @@ def test_workflow_tests_criticality(pub):
 
 
 def test_workflow_tests_backoffice_fields(pub):
-    user = pub.user_class(name='test user')
-    user.store()
-
     workflow = Workflow(name='Workflow One')
     workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
     workflow.backoffice_fields_formdef.fields = [
@@ -921,7 +989,6 @@ def test_workflow_tests_backoffice_fields(pub):
     formdata.data['1'] = 'abc'
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
     testdef.workflow_tests.actions = [
         workflow_tests.AssertBackofficeFieldValues(id='1', fields=[{'field_id': 'bo2', 'value': 'abc'}]),
     ]
@@ -944,9 +1011,6 @@ def test_workflow_tests_backoffice_fields(pub):
 
 
 def test_workflow_tests_webservice(pub):
-    user = pub.user_class(name='test user')
-    user.store()
-
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')
     end_status = workflow.add_status(name='End status')
@@ -977,7 +1041,6 @@ def test_workflow_tests_webservice(pub):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
     testdef.store()
 
     response = WebserviceResponse()
@@ -1058,9 +1121,6 @@ def test_workflow_tests_webservice(pub):
 
 
 def test_workflow_tests_webservice_status_jump(pub):
-    user = pub.user_class(name='test user')
-    user.store()
-
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')
     end_status = workflow.add_status(name='Error status')
@@ -1081,7 +1141,6 @@ def test_workflow_tests_webservice_status_jump(pub):
     formdata.just_created()
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.agent_id = user.id
     testdef.store()
 
     response = WebserviceResponse()
@@ -1108,9 +1167,13 @@ def test_workflow_tests_webservice_status_jump(pub):
 
 
 def test_workflow_tests_create_from_formdata(pub, http_requests, freezer):
+    pub.cfg['sms'] = {'sender': 'xxx', 'passerelle_url': 'http://passerelle.invalid/'}
+    pub.write_cfg()
+
     role = pub.role_class(name='test role')
     role.store()
     user = create_user(pub, is_admin=True)
+    user.test_uuid = '42'
     user.roles = [role.id]
     user.store()
 
@@ -1200,6 +1263,7 @@ def test_workflow_tests_create_from_formdata(pub, http_requests, freezer):
     assert formdata.status == 'wf-end-status'
 
     testdef = TestDef.create_from_formdata(formdef, formdata, add_workflow_tests=True)
+    testdef.agent_id = user.test_uuid
     testdef.run(formdef)
 
     actions = testdef.workflow_tests.actions
@@ -1234,3 +1298,71 @@ def test_workflow_tests_create_from_formdata(pub, http_requests, freezer):
 
     assert actions[-1].key == 'assert-status'
     assert actions[-1].status_name == 'End status'
+
+
+def test_workflow_tests_create_from_formdata_multiple_buttons(pub, http_requests):
+    role = pub.role_class(name='test role')
+    role.store()
+    user = create_user(pub, is_admin=True)
+    user.test_uuid = '42'
+    user.roles = [role.id]
+    user.store()
+
+    workflow = Workflow(name='Workflow One')
+    new_status = workflow.add_status('New status', 'new-status')
+    middle_status = workflow.add_status('Middle status', 'middle-status')
+    end_status = workflow.add_status('End status', 'end-status')
+
+    choice = new_status.add_action('choice')
+    choice.label = 'Go to middle status'
+    choice.status = middle_status.id
+    choice.by = [role.id]
+
+    choice = middle_status.add_action('choice')
+    choice.label = 'Go to end status'
+    choice.status = end_status.id
+    choice.by = [role.id]
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.user_id = user.id
+    formdata.just_created()
+    formdata.store()
+
+    formdata.perform_workflow()
+    formdata.store()
+
+    app = login(get_app(pub))
+    resp = app.get(formdata.get_url())
+    resp = resp.form.submit('button1').follow()
+    resp = resp.form.submit('button1').follow()
+    formdata.refresh_from_storage()
+    assert formdata.status == 'wf-end-status'
+
+    testdef = TestDef.create_from_formdata(formdef, formdata, add_workflow_tests=True)
+    testdef.agent_id = user.test_uuid
+    testdef.run(formdef)
+
+    actions = testdef.workflow_tests.actions
+    assert len(actions) == 5
+
+    assert actions[0].key == 'assert-status'
+    assert actions[0].status_name == 'New status'
+
+    assert actions[1].key == 'button-click'
+    assert actions[1].button_name == 'Go to middle status'
+
+    assert actions[2].key == 'assert-status'
+    assert actions[2].status_name == 'Middle status'
+
+    assert actions[3].key == 'button-click'
+    assert actions[3].button_name == 'Go to end status'
+
+    assert actions[4].key == 'assert-status'
+    assert actions[4].status_name == 'End status'

tests/utilities.py

@@ -146,6 +146,7 @@ def create_temporary_pub(pickle_mode=False, lazy_mode=False):
             sql.Audit.wipe()
             sql_mark_current_test()
         pub.write_cfg()
+        pub.reset_caches()
         return pub
 
     os.symlink(os.path.join(os.path.dirname(__file__), 'templates'), os.path.join(pub.app_dir, 'templates'))

tests/workflow/test_create_formdata.py

@@ -127,6 +127,7 @@ def test_create_formdata(pub):
     # now we want one
     target_formdef.enable_tracking_codes = True
     target_formdef.store()
+    pub.reset_caches()
     target_formdef.data_class().wipe()
     formdata.perform_workflow()
     # and a tracking code is created
@@ -558,6 +559,7 @@ def test_recursive_create_formdata_with_subformdata(pub):
 
     formdef.workflow_id = wf.id
     formdef.store()
+    pub.reset_caches()
 
     formdata = formdef.data_class()()
     formdata.data = {}
@@ -579,6 +581,7 @@ def test_recursive_create_formdata_with_subformdata(pub):
         StringField(id='0', label='string', varname='foo_string'),
     ]
     subsubformdef.store()
+    pub.reset_caches()
 
     subwf = Workflow(name='create-formdata-again')
     subwf.possible_status = Workflow.get_default_workflow().possible_status[:]
@@ -591,6 +594,7 @@ def test_recursive_create_formdata_with_subformdata(pub):
 
     subformdef.workflow_id = subwf.id
     subformdef.store()
+    pub.reset_caches()
 
     formdata = formdef.data_class()()
     formdata.data = {}

tests/workflow/test_jump.py

@@ -1,7 +1,9 @@
 import datetime
+import os
 from unittest import mock
 
 import pytest
+from django.core.management import call_command
 from pyquery import PyQuery
 from quixote import cleanup
 
@@ -11,6 +13,7 @@ from wcs.qommon.http_request import HTTPRequest
 from wcs.wf.jump import JumpWorkflowStatusItem, _apply_timeouts
 from wcs.workflows import Workflow, perform_items
 
+from ..test_publisher import get_logs
 from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
 from .test_all import admin_user  # noqa pylint: disable=unused-import
 
@@ -629,3 +632,44 @@ def test_jump_self_timeout(pub):
     formdata.store()
     formdata.record_workflow_event('backoffice-created')
     _apply_timeouts(pub)
+
+
+def test_timeout_cron_debug_log(pub):
+    FormDef.wipe()
+    Workflow.wipe()
+
+    workflow = Workflow(name='timeout')
+    st1 = workflow.add_status('Status1', 'st1')
+    workflow.add_status('Status2', 'st2')
+
+    jump = st1.add_action('jump', id='_jump')
+    jump.by = ['_submitter', '_receiver']
+    jump.timeout = 30 * 60  # 30 minutes
+    jump.status = 'st2'
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.workflow_id = workflow.id
+    assert formdef.get_workflow().id == workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    rewind(formdata, seconds=40 * 60)
+    formdata.store()
+    formdata_id = formdata.id
+
+    pub.load_site_options()
+    pub.site_options.set('options', 'cron-log-level', 'debug')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+
+    call_command('cron', job_name='evaluate_jumps', domain='example.net', force_job=True)
+
+    assert formdef.data_class().get(formdata_id).status == 'wf-st2'
+    assert get_logs('example.net')[:2] == ['start', "running jobs: ['evaluate_jumps']"]
+    assert 'applying timeouts on baz' in get_logs('example.net')[2]
+    assert 'event: timeout-jump' in get_logs('example.net')[3]

tests/workflow/test_register_comment.py

@@ -607,3 +607,22 @@ def test_register_comment_to_with_attachment(pub):
     assert 'to-role.txt' in display_parts()[2]
     assert 'to-submitter.txt' in display_parts()[4]
     assert 'to-role-or-submitter.txt' in display_parts()[6]
+
+
+def test_register_comment_fts(pub):
+    pub.substitutions.feed(MockSubstitutionVariables())
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    item = RegisterCommenterWorkflowStatusItem()
+    item.comment = 'Hello\x00\nworld'
+    item.perform(formdata)
+    assert formdata.evolution[-1].parts[-1].content == '<p>Hello\x00\nworld</p>'  # kept
+    assert formdata.evolution[-1].parts[-1].render_for_fts() == 'Hello  world'  # not kept

tox.ini

@@ -77,6 +77,7 @@ deps =
     schwifty
 allowlist_externals =
     ./getlasso3.sh
+    ./pylint.sh
 commands =
     ./getlasso3.sh
     ./pylint.sh wcs/ tests/

wcs/admin/api_access.py

@@ -201,7 +201,7 @@ class ApiAccessDirectory(Directory):
             templates=['wcs/backoffice/api_accesses.html'],
             context={
                 'view': self,
-                'api_accesses': ApiAccess.select(order_by='name'),
+                'api_accesses': [x for x in ApiAccess.select(order_by='name') if not x.idp_api_client],
                 'api_manage_url': api_manage_url,
             },
         )

wcs/admin/blocks.py

@@ -467,7 +467,7 @@ class BlocksDirectory(Directory):
 
         error, reason = False, None
         try:
-            blockdef = BlockDef.import_from_xml(fp)
+            blockdef = BlockDef.import_from_xml(fp, check_deprecated=True)
         except BlockdefImportError as e:
             error = True
             reason = _(e.msg) % e.msg_args

wcs/admin/categories.py

@@ -469,7 +469,7 @@ class CategoriesDirectory(Directory):
         fp = form.get_widget('file').parse().fp
 
         try:
-            category = self.category_class.import_from_xml(fp)
+            category = self.category_class.import_from_xml(fp, check_deprecated=True)
             get_session().message = ('info', _('This category has been successfully imported.'))
         except ValueError as e:
             form.set_error('file', _('Invalid File'))

wcs/admin/comment_templates.py

@@ -150,7 +150,7 @@ class CommentTemplatesDirectory(Directory):
 
         error = False
         try:
-            comment_template = CommentTemplate.import_from_xml(fp)
+            comment_template = CommentTemplate.import_from_xml(fp, check_deprecated=True)
             get_session().message = ('info', _('This comment template has been successfully imported.'))
         except ValueError:
             error = True

wcs/admin/data_sources.py

@@ -670,7 +670,7 @@ class NamedDataSourcesDirectory(Directory):
 
         error, reason = False, None
         try:
-            datasource = NamedDataSource.import_from_xml(fp)
+            datasource = NamedDataSource.import_from_xml(fp, check_deprecated=True)
             get_session().message = ('info', _('This datasource has been successfully imported.'))
         except NamedDataSourceImportError as e:
             error = True

wcs/admin/fields.py

@@ -387,6 +387,18 @@ class FieldsDirectory(Directory):
                 r += htmltext(' ')
                 r += htmltext(_('It is close to the system limits and no new fields should be added.'))
                 r += htmltext('</div>')
+            elif (
+                hasattr(self.objectdef, 'get_total_count_data_fields')
+                and self.objectdef.get_total_count_data_fields() > 2000
+            ):
+                # warn before DATA_UPLOAD_MAX_NUMBER_FIELDS
+                r += htmltext('<div class="warningnotice">')
+                r += htmltext('<p>%s %s</p>') % (
+                    _('There are at least %d data fields, including fields in blocks.')
+                    % self.objectdef.get_total_count_data_fields(),
+                    _('It is close to the system limits and no new fields should be added.'),
+                )
+                r += htmltext('</div>')
 
             if [x for x in self.objectdef.fields if x.key == 'page']:
                 if self.objectdef.fields[0].key != 'page':

wcs/admin/forms.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+import datetime
 import difflib
 import io
 import xml.etree.ElementTree as ET
@@ -28,6 +29,7 @@ from wcs.backoffice.deprecations import DeprecationsDirectory
 from wcs.backoffice.snapshots import SnapshotsDirectory
 from wcs.carddef import CardDef
 from wcs.categories import Category
+from wcs.fields import PageField
 from wcs.formdef import (
     DRAFTS_DEFAULT_LIFESPAN,
     DRAFTS_DEFAULT_MAX_PER_USER,
@@ -60,7 +62,7 @@ from wcs.qommon.form import (
 )
 from wcs.qommon.misc import localstrftime
 from wcs.roles import get_user_roles, logged_users_role
-from wcs.sql_criterias import Equal, Null, StrictNotEqual
+from wcs.sql_criterias import Equal, GreaterOrEqual, Null, StrictNotEqual
 from wcs.workflows import Workflow
 
 from . import utils
@@ -516,6 +518,7 @@ class OptionsDirectory(Directory):
                 'drafts_max_per_user',
                 'user_support',
                 'management_sidebar_items',
+                'history_pane_default_mode',
             ]
             for attr in attrs:
                 widget = form.get_widget(attr)
@@ -526,8 +529,8 @@ class OptionsDirectory(Directory):
                             continue
                     new_value = widget.parse()
                     if attr == 'management_sidebar_items':
-                        new_value = set(new_value)
-                        if new_value == self.formdef.__class__.management_sidebar_items:
+                        new_value = set(new_value or [])
+                        if new_value == self.formdef.get_default_management_sidebar_items():
                             new_value = {'__default__'}
                     if attr == 'digest_template':
                         if self.formdef.default_digest_template != new_value:
@@ -630,7 +633,6 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
         'enable',
         'workflow',
         'role',
-        ('workflow-options', 'workflow_options'),
         ('workflow-variables', 'workflow_variables'),
         ('workflow-status-remapping', 'workflow_status_remapping'),
         'roles',
@@ -805,7 +807,8 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
                 _('Custom')
                 if (
                     self.formdef.skip_from_360_view
-                    or self.formdef.management_sidebar_items != {'__default__'}
+                    or self.formdef.management_sidebar_items
+                    not in ({'__default__'}, self.formdef.get_default_management_sidebar_items())
                 )
                 else _('Default'),
             ),
@@ -855,17 +858,8 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
             )
 
         options['workflow_options'] = ''
-        if self.formdef.workflow_id:
-            pristine_workflow = Workflow.get(self.formdef.workflow_id, ignore_errors=True)
-            if pristine_workflow and pristine_workflow.variables_formdef:
-                options['workflow_options'] = self.add_option_line('workflow-variables', _('Options'), '')
-            elif self.formdef.workflow_options and get_publisher().has_site_option(
-                'enable-workflow-variable-parameter'
-            ):
-                # there are no variables defined but there are some values
-                # in workflow_options, this is probably the legacy stuff.
-                if any(x for x in self.formdef.workflow_options if '*' in x):
-                    options['workflow_options'] = self.add_option_line('workflow-options', _('Options'), '')
+        if self.formdef.workflow and self.formdef.workflow.variables_formdef:
+            options['workflow_options'] = self.add_option_line('workflow-variables', _('Options'), '')
 
         options['workflow_roles_list'] = []
         if self.formdef.workflow.roles:
@@ -1454,7 +1448,7 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
 
         error, reason = False, None
         try:
-            new_formdef = self.formdef_class.import_from_xml(fp, include_id=True)
+            new_formdef = self.formdef_class.import_from_xml(fp, include_id=True, check_deprecated=True)
         except FormdefImportError as e:
             error = True
             reason = _(e.msg) % e.msg_args
@@ -1701,55 +1695,6 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
         r += form.render()
         return r.getvalue()
 
-    def workflow_options(self):
-        request = get_request()
-        if request.get_method() == 'GET' and request.form.get('file'):
-            value = self.formdef.workflow_options.get(request.form.get('file'))
-            if value:
-                return value.build_response()
-
-        get_response().set_title(title=_('Workflow Options'))
-        form = Form(enctype='multipart/form-data')
-        pristine_workflow = Workflow.get(self.formdef.workflow_id)
-        for status in self.formdef.workflow.possible_status:
-            had_options = False
-            for item in status.items:
-                prefix = '%s*%s*' % (status.id, item.id)
-                pristine_item = pristine_workflow.get_status(status.id).get_item(item.id)
-                parameters = [x for x in item.get_parameters() if not getattr(pristine_item, x)]
-                if not parameters:
-                    continue
-                if not had_options:
-                    form.widgets.append(HtmlWidget('<h3>%s</h3>' % status.name))
-                    had_options = True
-                label = getattr(item, 'label', None) or _(item.description)
-                form.widgets.append(HtmlWidget('<h4>%s</h4>' % label))
-                item.add_parameters_widgets(form, parameters, prefix=prefix, formdef=self.formdef)
-        form.add_submit('submit', _('Submit'))
-        form.add_submit('cancel', _('Cancel'))
-
-        if form.get_widget('cancel').parse():
-            return redirect('.')
-
-        if form.is_submitted() and not form.has_errors():
-            self.workflow_options_submit(form)
-            return redirect('.')
-
-        r = TemplateIO(html=True)
-        r += htmltext('<h2>%s</h2>') % _('Workflow Options')
-        r += form.render()
-        return r.getvalue()
-
-    def workflow_options_submit(self, form):
-        self.formdef.workflow_options = {}
-        for widget in form.get_all_widgets():
-            if widget in form.get_submit_widgets():
-                continue
-            if widget.name.startswith('_'):
-                continue
-            self.formdef.workflow_options[widget.name] = widget.parse()
-        self.formdef.store(comment=_('Change in workflow options'))
-
     def inspect(self):
         get_response().set_title(self.formdef.name)
         get_response().breadcrumb.append(('inspect', _('Inspector')))
@@ -1812,6 +1757,56 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
                 f'{self.formdef.xml_root_node}:{self.formdef.id}'
             )
             context['deprecation_titles'] = deprecations.titles
+
+        receipt_time_criteria = GreaterOrEqual(
+            'receipt_time',
+            datetime.datetime.now() - datetime.timedelta(days=self.formdef.get_drafts_lifespan()),
+        )
+
+        temp_drafts = defaultdict(int)
+        for formdata in self.formdef.data_class().select_iterator(
+            clause=[Equal('status', 'draft'), receipt_time_criteria], itersize=200
+        ):
+            page_id = formdata.page_id if formdata.page_id is not None else '_unknown'
+            temp_drafts[page_id] += 1
+
+        total_drafts = sum(temp_drafts.values()) if temp_drafts else 0
+        drafts = {}
+        special_page_index_mapping = {
+            '_first_page': -1000,  # first
+            '_unknown': 1000,  # last
+            '_confirmation_page': 999,  # second to last
+        }
+        if total_drafts:
+            for page_id, page_index in special_page_index_mapping.items():
+                try:
+                    page_total = temp_drafts.pop(page_id)
+                except KeyError:
+                    page_total = 0
+                drafts[page_id] = {'total': page_total, 'field': None, 'page_index': page_index}
+            for page_id, page_total in temp_drafts.items():
+                for index, field in enumerate(self.formdef.iter_fields(with_backoffice_fields=False)):
+                    if page_id == field.id and isinstance(field, PageField):
+                        drafts[page_id] = {
+                            'total': page_total,
+                            'field': field,
+                            'page_index': index,
+                        }
+                        break
+                else:
+                    drafts['_unknown']['total'] += page_total
+
+            for draft_data in drafts.values():
+                draft_data['percent'] = 100 * draft_data['total'] / total_drafts
+
+            total_formdata = self.formdef.data_class().count([receipt_time_criteria])
+            context['drafts'] = sorted(drafts.items(), key=lambda x: x[1]['page_index'])
+            context['percent_submitted_formdata'] = 100 * (total_formdata - total_drafts) / total_formdata
+            context['total_formdata'] = total_formdata
+
+        context['total_drafts'] = total_drafts
+        context['is_carddef'] = isinstance(self.formdef, CardDef)
+
         return template.QommonTemplateResponse(
             templates=[self.inspect_template_name],
             context=context,
@@ -1839,6 +1834,7 @@ class FormsDirectory(AccessControlled, Directory):
         'categories',
         ('data-sources', 'data_sources'),
         ('application', 'applications_dir'),
+        ('test-users', 'test_users'),
     ]
 
     category_class = Category
@@ -1871,6 +1867,12 @@ class FormsDirectory(AccessControlled, Directory):
         super().__init__(*args, **kwargs)
         self.applications_dir = ApplicationsDirectory(self.formdef_class)
 
+    @property
+    def test_users(self):
+        from wcs.admin.tests import TestUsersDirectory
+
+        return TestUsersDirectory()
+
     def _q_traverse(self, path):
         get_response().breadcrumb.append(('%s/' % self.section, self.top_title))
         get_response().set_backoffice_section(self.section)
@@ -2024,7 +2026,7 @@ class FormsDirectory(AccessControlled, Directory):
         error, reason = False, None
         try:
             try:
-                formdef = self.formdef_class.import_from_xml(fp)
+                formdef = self.formdef_class.import_from_xml(fp, check_deprecated=True)
                 get_session().message = ('info', str(self.import_success_message))
             except FormdefImportRecoverableError:
                 fp.seek(0)

wcs/admin/logged_errors.py

@@ -317,7 +317,12 @@ class LoggedErrorsDirectory(AccessControlled, Directory):
                 if 'others' in form.get_widget('types').parse():
                     criterias.append(Null('formdef_class'))
                 criterias = [Or(criterias)]
-            criterias.append(Less('latest_occurence_timestamp', form.get_widget('latest_occurence').parse()))
+            criterias.append(
+                Less(
+                    'latest_occurence_timestamp',
+                    misc.get_as_datetime(form.get_widget('latest_occurence').parse()),
+                )
+            )
             get_publisher().loggederror_class.wipe(clause=criterias)
             return redirect('.')
 

wcs/admin/mail_templates.py

@@ -150,7 +150,7 @@ class MailTemplatesDirectory(Directory):
 
         error = False
         try:
-            mail_template = MailTemplate.import_from_xml(fp)
+            mail_template = MailTemplate.import_from_xml(fp, check_deprecated=True)
             get_session().message = ('info', _('This mail template has been successfully imported.'))
         except ValueError:
             error = True

wcs/admin/tests.py

@@ -17,6 +17,7 @@
 import collections
 import copy
 import json
+import uuid
 
 from django.template.loader import render_to_string
 from django.utils.timezone import now
@@ -35,14 +36,16 @@ from wcs.qommon.errors import TraversalError
 from wcs.qommon.form import (
     FileWidget,
     Form,
+    JsonpSingleSelectWidget,
     RadiobuttonsWidget,
     SingleSelectWidget,
     StringWidget,
     TextWidget,
     UrlWidget,
     WidgetDict,
+    WidgetList,
 )
-from wcs.sql_criterias import Equal, Null, StrictNotEqual
+from wcs.sql_criterias import Equal, NotNull, Null, StrictNotEqual
 from wcs.testdef import TestDef, TestError, TestResult, WebserviceResponse
 from wcs.workflow_tests import WorkflowTestError
 from wcs.workflow_traces import WorkflowTrace
@@ -68,11 +71,8 @@ class TestEditPage(FormBackofficeEditPage):
         return super()._q_index()
 
     def create_form(self, *args, **kwargs):
-        # FormBackofficeEditPage.create_form is relevant only for forms, skip it for cards
-        if self.testdef.object_type == 'formdefs':
-            form = super().create_form(*args, **kwargs)
-        else:
-            form = super(FormBackofficeEditPage, self).create_form(*args, **kwargs)
+        form = super().create_form(*args, **kwargs)
+        form.attrs['data-live-url'] = self.testdef.get_admin_url() + 'edit-data/live'
         return form
 
     def modify_filling_context(self, context, *args, **kwargs):
@@ -228,13 +228,14 @@ class TestPage(FormBackOfficeStatusPage):
         form.add(StringWidget, 'name', title=_('Name'), required=True, size=50, value=self.testdef.name)
 
         user_options = [('', '---', '')] + [
-            (x.id, str(x), x.id) for x in get_publisher().user_class.select(order_by='name')
+            (x.test_uuid, str(x), x.test_uuid)
+            for x in get_publisher().user_class.select([NotNull('test_uuid')], order_by='name')
         ]
         form.add(
             SingleSelectWidget,
             'user',
             title=_('User'),
-            value=self.testdef.data['user'].get('id', '') if self.testdef.data['user'] else '',
+            value=self.testdef.user_uuid or '',
             options=user_options,
             **{'data-autocomplete': 'true'},
         )
@@ -252,13 +253,7 @@ class TestPage(FormBackOfficeStatusPage):
             return r.getvalue()
         else:
             self.testdef.name = form.get_widget('name').parse()
-
-            user_id = form.get_widget('user').parse()
-            if user_id:
-                user = get_publisher().user_class.get(user_id)
-                self.testdef.data['user'] = user.get_json_export_dict()
-            else:
-                self.testdef.data['user'] = None
+            self.testdef.user_uuid = form.get_widget('user').parse()
 
             self.testdef.store()
             return redirect('.')
@@ -387,11 +382,14 @@ class TestsDirectory(Directory):
             r += form.render()
             return r.getvalue()
 
+        agent_user = get_publisher().user_class.get(get_session().user)
+        test_agent_user, dummy = TestDef.get_or_create_test_user(agent_user)
+
         creation_mode_widget = form.get_widget('creation_mode')
         if not creation_mode_widget or creation_mode_widget.parse() == 'empty':
             testdef = TestDef.create_from_formdata(self.objectdef, self.objectdef.data_class()())
             testdef.name = form.get_widget('name').parse()
-            testdef.agent_id = get_session().user
+            testdef.agent_id = test_agent_user.test_uuid
             testdef.store()
             return redirect(testdef.get_admin_url() + 'edit-data/')
         else:
@@ -404,7 +402,7 @@ class TestsDirectory(Directory):
                 add_workflow_tests=bool(creation_mode_widget.parse() == 'formdata-wf'),
             )
             testdef.name = form.get_widget('name').parse()
-            testdef.agent_id = get_session().user
+            testdef.agent_id = test_agent_user.test_uuid
             testdef.store()
             return redirect(testdef.get_admin_url())
 
@@ -651,12 +649,13 @@ class TestResultsDirectory(Directory):
 
 
 class TestsAfterJob(AfterJob):
-    def __init__(self, objectdef, reason, snapshot=None, **kwargs):
+    def __init__(self, objectdef, reason, snapshot=None, triggered_by='', **kwargs):
         super().__init__(
             objectdef_class=objectdef.__class__,
             objectdef_id=objectdef.id,
-            reason=reason,
+            reason=str(reason or ''),
             snapshot_id=snapshot.id if snapshot else None,
+            triggered_by=triggered_by,
             **kwargs,
         )
 
@@ -667,7 +666,7 @@ class TestsAfterJob(AfterJob):
             return
         reason = self.kwargs['reason']
 
-        result = self.run_tests(objectdef, reason)
+        result = self.run_tests(objectdef, reason, self.kwargs.get('triggered_by', ''))
 
         if result and self.kwargs['snapshot_id'] is not None:
             snapshot = get_publisher().snapshot_class.get(self.kwargs['snapshot_id'])
@@ -675,11 +674,14 @@ class TestsAfterJob(AfterJob):
             snapshot.store()
 
     @staticmethod
-    def run_tests(objectdef, reason):
+    def run_tests(objectdef, reason, triggered_by=''):
         testdefs = TestDef.select_for_objectdef(objectdef)
         if not testdefs:
             return
 
+        if triggered_by == 'workflow-change' and not any(x.workflow_tests.actions for x in testdefs):
+            return
+
         for test in testdefs:
             try:
                 test.run(objectdef)
@@ -923,3 +925,164 @@ class WebserviceResponseDirectory(Directory):
         webservice_response.store()
 
         return redirect(self.testdef.get_admin_url() + 'webservice-responses/%s/' % webservice_response.id)
+
+
+class TestUserPage(Directory):
+    _q_exports = ['', 'delete']
+
+    def __init__(self, component):
+        try:
+            self.user = get_publisher().user_class.get(component)
+        except IndexError:
+            raise TraversalError()
+
+        if not self.user.test_uuid:
+            raise TraversalError()
+
+    def _q_index(self):
+        form = Form(enctype='multipart/form-data')
+
+        formdef = get_publisher().user_class.get_formdef()
+        form.add(
+            StringWidget, 'name', title=_('Test user label'), required=True, size=30, value=self.user.name
+        )
+        roles = list(get_publisher().role_class.select(order_by='name'))
+        form.add(
+            WidgetList,
+            'roles',
+            title=_('Roles'),
+            element_type=SingleSelectWidget,
+            value=self.user.roles,
+            add_element_label=_('Add Role'),
+            element_kwargs={
+                'render_br': False,
+                'options': [(None, '---', None)]
+                + [(x.id, x.name, x.id) for x in roles if not x.is_internal()],
+            },
+        )
+        formdef.add_fields_to_form(form, form_data=self.user.form_data)
+
+        form.add_submit('submit', _('Submit'))
+        form.add_submit('cancel', _('Cancel'))
+        form.add_media()
+
+        if form.get_widget('cancel').parse():
+            return redirect('.')
+
+        if form.get_submit() == 'submit' and not form.has_errors():
+            formdef = get_publisher().user_class.get_formdef()
+            data = formdef.get_data(form)
+            self.user.set_attributes_from_formdata(data)
+            self.user.form_data = data
+
+            if get_publisher().user_class.count(
+                [Equal('email', self.user.email), NotNull('test_uuid'), StrictNotEqual('id', self.user.id)]
+            ):
+                form.add_global_errors([_('A test user with this email already exists.')])
+            else:
+                self.user.name = form.get_widget('name').parse()
+                self.user.roles = form.get_widget('roles').parse()
+                self.user.store()
+
+                return redirect('..')
+
+        get_response().breadcrumb.append(('edit', _('Edit test user')))
+        r = TemplateIO(html=True)
+        r += htmltext('<h2>%s</h2>') % (_('Edit test user'))
+        r += form.render()
+        return r.getvalue()
+
+    def delete(self):
+        form = Form(enctype='multipart/form-data')
+        form.add_submit('delete', _('Delete'))
+        form.add_submit('cancel', _('Cancel'))
+
+        if form.get_widget('cancel').parse():
+            return redirect('.')
+        if not form.is_submitted() or form.has_errors():
+            get_response().breadcrumb.append(('delete', _('Delete')))
+            r = TemplateIO(html=True)
+            r += htmltext('<h2>%s %s</h2>') % (_('Deleting:'), self.user)
+            r += form.render()
+            return r.getvalue()
+
+        self.user.remove_object(self.user.id)
+        return redirect('..')
+
+
+class TestUsersDirectory(Directory):
+    _q_exports = ['', 'new']
+
+    def _q_traverse(self, path):
+        get_response().breadcrumb.append(('test-users/', _('Test users')))
+        return super()._q_traverse(path)
+
+    def _q_lookup(self, component):
+        return TestUserPage(component)
+
+    def _q_index(self):
+        context = {
+            'users': get_publisher().user_class.select([NotNull('test_uuid')]),
+            'has_sidebar': True,
+        }
+        get_response().add_javascript(['popup.js', 'select2.js'])
+        get_response().set_title(_('Test users'))
+        return template.QommonTemplateResponse(
+            templates=['wcs/backoffice/test-users.html'],
+            context=context,
+            is_django_native=True,
+        )
+
+    def new(self):
+        form = Form(enctype='multipart/form-data')
+        form.add(StringWidget, 'name', title=_('Name'), required=True, size=50)
+
+        creation_options = [
+            ('empty', _('Empty user'), 'empty'),
+            ('copy', _('Copy existing user'), 'copy'),
+        ]
+        form.add(
+            RadiobuttonsWidget,
+            'creation_mode',
+            options=creation_options,
+            value='empty',
+            attrs={'data-dynamic-display-parent': 'true'},
+        )
+        form.attrs['data-enable-select2'] = 'on'
+        form.add(
+            JsonpSingleSelectWidget,
+            'user_id',
+            url='/api/users/',
+            attrs={
+                'data-dynamic-display-child-of': 'creation_mode',
+                'data-dynamic-display-value-in': 'copy',
+            },
+        )
+
+        form.add_submit('submit', _('Submit'))
+        form.add_submit('cancel', _('Cancel'))
+
+        if form.get_widget('cancel').parse():
+            return redirect('.')
+
+        if form.is_submitted() and not form.has_errors():
+            if form.get_widget('creation_mode').parse() == 'empty':
+                user = get_publisher().user_class()
+                user.test_uuid = str(uuid.uuid4())
+            else:
+                user = get_publisher().user_class.get(form.get_widget('user_id').parse())
+                user, created = TestDef.get_or_create_test_user(user)
+                if not created:
+                    form.get_widget('user_id').set_error(_('A test user with this email already exists.'))
+
+            if not form.has_errors():
+                user.name = form.get_widget('name').parse()
+                user.store()
+                return redirect('.')
+
+        get_response().breadcrumb.append(('new', _('New')))
+        get_response().set_title(_('New test user'))
+        r = TemplateIO(html=True)
+        r += htmltext('<h2>%s</h2>') % _('New test user')
+        r += form.render()
+        return r.getvalue()

wcs/admin/workflow_tests.py

@@ -24,6 +24,7 @@ from quixote.html import TemplateIO, htmltext
 from wcs.qommon import _, template
 from wcs.qommon.errors import TraversalError
 from wcs.qommon.form import Form, SingleSelectWidget
+from wcs.sql_criterias import NotNull
 from wcs.workflow_tests import get_test_action_class_by_type, get_test_action_options
 
 
@@ -95,7 +96,8 @@ class WorkflowTestActionPage(Directory):
     def duplicate(self):
         new_action = copy.deepcopy(self.action)
         new_action.id = self.testdef.workflow_tests.get_new_action_id()
-        self.testdef.workflow_tests.actions.append(new_action)
+        action_position = self.testdef.workflow_tests.actions.index(self.action)
+        self.testdef.workflow_tests.actions.insert(action_position + 1, new_action)
         self.testdef.store()
         return redirect('..')
 
@@ -147,7 +149,8 @@ class WorkflowTestsDirectory(Directory):
         form = Form(enctype='multipart/form-data')
 
         user_options = [('', '---', '')] + [
-            (str(x.id), str(x), str(x.id)) for x in get_publisher().user_class.select(order_by='name')
+            (str(x.test_uuid), str(x), str(x.test_uuid))
+            for x in get_publisher().user_class.select([NotNull('test_uuid')], order_by='name')
         ]
         form.add(
             SingleSelectWidget,

wcs/admin/workflows.py

@@ -40,7 +40,6 @@ from wcs.qommon.afterjobs import AfterJob
 from wcs.qommon.form import (
     CheckboxWidget,
     ColourWidget,
-    CompositeWidget,
     ComputedExpressionWidget,
     FileWidget,
     Form,
@@ -51,7 +50,6 @@ from wcs.qommon.form import (
     SlugWidget,
     StringWidget,
     UrlWidget,
-    VarnameWidget,
 )
 from wcs.sql_criterias import Equal
 from wcs.workflows import (
@@ -1083,80 +1081,15 @@ class WorkflowStatusDirectory(Directory):
         return r.getvalue()
 
 
-class WorkflowVariableWidget(CompositeWidget):
-    def __init__(self, name, value=None, workflow=None, **kwargs):
-        CompositeWidget.__init__(self, name, **kwargs)
-        if value and '*' in value:
-            varname = None
-        else:
-            varname = value
-        self.add(VarnameWidget, 'name', render_br=False, value=varname)
-        if not get_publisher().has_site_option('enable-workflow-variable-parameter'):
-            return
-        options = []
-        if workflow:
-            excluded_parameters = ['backoffice_info_text']
-            for status in workflow.possible_status:
-                for item in status.items:
-                    prefix = '%s*%s*' % (status.id, item.id)
-                    parameters = [
-                        x
-                        for x in item.get_parameters()
-                        if not getattr(item, x) and x not in excluded_parameters
-                    ]
-                    label = getattr(item, 'label', None) or item.description
-                    for parameter in parameters:
-                        key = prefix + parameter
-                        fake_form = Form()
-                        item.add_parameters_widgets(fake_form, [parameter], orig='variable_widget')
-                        if not fake_form.widgets:
-                            continue
-                        parameter_label = fake_form.widgets[0].title
-                        option_value = '%s / %s / %s' % (status.name, label, parameter_label)
-                        options.append((key, option_value, key))
-        if not options:
-            return
-        options = [('', '---', '')] + options
-        self.widgets.append(
-            HtmlWidget(_('or you can use this field to directly replace a workflow parameter:'))
-        )
-        self.add(
-            SingleSelectWidget,
-            'select',
-            options=options,
-            value=value,
-            hint=_('This takes priority over a variable name'),
-            attrs={'data-dynamic-display-parent': 'true'},
-            render_br=False,
-        )
-
-    def _parse(self, request):
-        super()._parse(request)
-        if self.get('select'):
-            self.value = self.get('select')
-        elif self.get('name'):
-            self.value = self.get('name')
-
-
 class WorkflowVariablesFieldDefPage(FieldDefPage):
     section = 'workflows'
     blacklisted_attributes = ['condition', 'prefill', 'display_locations', 'anonymise']
 
     def form(self):
         form = super().form()
-        form.remove('varname')
-        form.add(
-            WorkflowVariableWidget,
-            'varname',
-            title=_('Variable'),
-            value=self.field.varname,
-            advanced=False,
-            required=True,
-            workflow=self.objectdef.workflow,
-        )
         # add default value widget
         if self.field.key in ('string', 'email', 'text', 'date'):
-            widget = form.add(
+            form.add(
                 self.field.widget_class,
                 'default_value',
                 title=_('Default Value'),
@@ -1167,11 +1100,6 @@ class WorkflowVariablesFieldDefPage(FieldDefPage):
                 ),
                 value=getattr(self.field, 'default_value', None),
             )
-            if get_publisher().has_site_option('enable-workflow-variable-parameter'):
-                widget.attrs = {
-                    'data-dynamic-display-child-of': 'varname$select',
-                    'data-dynamic-display-value': '',
-                }
         return form
 
     def submit(self, form):
@@ -2222,7 +2150,7 @@ class WorkflowsDirectory(Directory):
 
         error, reason = False, None
         try:
-            workflow = Workflow.import_from_xml(fp)
+            workflow = Workflow.import_from_xml(fp, check_deprecated=True)
         except WorkflowImportError as e:
             error = True
             reason = _(e.msg) % e.msg_args

wcs/admin/wscalls.py

@@ -319,7 +319,7 @@ class NamedWsCallsDirectory(Directory):
 
         error, reason = False, None
         try:
-            wscall = NamedWsCall.import_from_xml(fp)
+            wscall = NamedWsCall.import_from_xml(fp, check_deprecated=True)
             get_session().message = ('info', _('This webservice call has been successfully imported.'))
         except NamedWsCallImportError as e:
             error = True

wcs/api_access.py

@@ -33,6 +33,7 @@ class ApiAccess(XmlStorableObject):
     access_key = None
     description = None
     restrict_to_anonymised_data = False
+    idp_api_client = False
     _roles = None
     _role_ids = Ellipsis
 
@@ -44,6 +45,7 @@ class ApiAccess(XmlStorableObject):
         ('access_key', 'str'),
         ('restrict_to_anonymised_data', 'bool'),
         ('roles', 'roles'),
+        ('idp_api_client', 'bool'),
     ]
 
     @classmethod
@@ -98,7 +100,7 @@ class ApiAccess(XmlStorableObject):
     @classmethod
     def get_with_credentials(cls, username, password):
         api_access = cls.get_by_identifier(username)
-        if not api_access or api_access.access_key != password:
+        if not api_access or api_access.access_key != password or api_access.idp_api_client:
             api_access = cls.get_from_idp(username, password)
             if not api_access:
                 raise KeyError
@@ -143,11 +145,18 @@ class ApiAccess(XmlStorableObject):
         if data.get('err', 1) != 0:
             return None
 
-        api_access = cls.volatile()
+        # cache api client locally, it is necessary for serialization for afterjobs
+        # in uwsgi spooler.
+        access_identifier = f'_idp_{username}'
+        api_access = cls.get_by_identifier(access_identifier) or cls()
+        api_access.idp_api_client = True
+        api_access.access_identifier = access_identifier
+        role_class = get_publisher().role_class
         try:
             api_access.restrict_to_anonymised_data = data['data']['restrict_to_anonymised_data']
-            api_access._role_ids = data['data']['roles']
+            api_access.roles = [role_class.get(x, ignore_errors=True) for x in data['data']['roles']]
+            api_access.roles = [x for x in api_access.roles if x is not None]
         except KeyError:
             return None
-
+        api_access.store()
         return api_access

wcs/api_export_import.py

@@ -269,9 +269,9 @@ def object_dependencies(request, objects, slug):
 
 @signature_required
 def bundle_check(request):
-    tar_io = io.BytesIO(request.body)
+    bundle = request.FILES['bundle']
     try:
-        with tarfile.open(fileobj=tar_io) as tar:
+        with tarfile.open(fileobj=bundle) as tar:
             try:
                 manifest = json.loads(tar.extractfile('manifest.json').read().decode())
             except KeyError:
@@ -529,7 +529,10 @@ class BundleImportJob(AfterJob):
                     'Invalid tar file, missing component %s/%s.' % (element['type'], element['slug'])
                 )
             new_object = element_klass.import_from_xml_tree(
-                ET.fromstring(element_content), include_id=False, check_datasources=False
+                ET.fromstring(element_content),
+                include_id=False,
+                check_datasources=False,
+                check_deprecated=True,
             )
             if not finalize and element_klass in category_classes:
                 # for categories, keep positions of imported objects
@@ -631,7 +634,7 @@ class BundleImportJob(AfterJob):
 
 @signature_required
 def bundle_import(request):
-    job = BundleImportJob(tar_content=request.body)
+    job = BundleImportJob(tar_content=request.FILES['bundle'].read())
     job.store()
     job.run(spool=True)
     return JsonResponse({'err': 0, 'url': job.get_api_status_url()})
@@ -689,7 +692,7 @@ class BundleDeclareJob(BundleImportJob):
 
 @signature_required
 def bundle_declare(request):
-    job = BundleDeclareJob(tar_content=request.body)
+    job = BundleDeclareJob(tar_content=request.FILES['bundle'].read())
     job.store()
     job.run(spool=True)
     return JsonResponse({'err': 0, 'url': job.get_api_status_url()})

wcs/backoffice/cards.py

@@ -31,7 +31,7 @@ from wcs.categories import CardDefCategory
 from wcs.sql_criterias import Null, StrictNotEqual
 
 from ..qommon import _, pgettext_lazy
-from ..qommon.form import ComputedExpressionWidget, StringWidget
+from ..qommon.form import CheckboxesWidget, ComputedExpressionWidget, Form, RadiobuttonsWidget, StringWidget
 
 
 class CardDefUI(FormDefUI):
@@ -71,6 +71,26 @@ class CardDefOptionsDirectory(OptionsDirectory):
         )
         return form
 
+    def management(self):
+        form = Form(enctype='multipart/form-data')
+        form.add(
+            CheckboxesWidget,
+            'management_sidebar_items',
+            title=_('Sidebar elements'),
+            options=[(x[0], x[1], x[0]) for x in self.formdef.get_management_sidebar_available_items()],
+            value=self.formdef.get_management_sidebar_items(),
+            inline=False,
+        )
+        form.add(
+            RadiobuttonsWidget,
+            'history_pane_default_mode',
+            title=_('History pane default mode'),
+            options=[('collapsed', _('Collapsed'), 'collapsed'), ('expanded', _('Expanded'), 'expanded')],
+            value=self.formdef.history_pane_default_mode,
+            extra_css_class='widget-inline-radio',
+        )
+        return self.handle(form, pgettext_lazy('cards', 'Management'))
+
 
 class CardFieldDefPage(FormFieldDefPage):
     section = 'cards'
@@ -140,6 +160,15 @@ class CardDefPage(FormDefPage):
         options['user_support'] = self.add_option_line(
             'options/user_support', _('User support'), user_support_status
         )
+        options['management'] = self.add_option_line(
+            'options/management',
+            pgettext_lazy('cards', 'Management'),
+            _('Custom')
+            if self.formdef.history_pane_default_mode != 'collapsed'
+            or self.formdef.management_sidebar_items
+            not in ({'__default__'}, self.formdef.get_default_management_sidebar_items())
+            else _('Default'),
+        )
         return options
 
     def get_sorted_usage_in_formdefs(self):

wcs/backoffice/data_management.py

@@ -441,9 +441,6 @@ class CardBackOfficeStatusPage(FormBackOfficeStatusPage):
     def should_fold_summary(self, mine, request_user):
         return False
 
-    def should_fold_history(self):
-        return True
-
 
 class ImportFromCsvAfterJob(AfterJob):
     def __init__(self, carddef, data_lines, update_existing_cards, submission_agent_id):

wcs/backoffice/deprecations.py

@@ -416,6 +416,7 @@ class DeprecationsScan(AfterJob):
             )
 
     def check_deprecated_elements_in_object(self, obj):
+        self.id = None  # to avoid store of afterjob
         if not get_publisher().has_site_option('forbid-new-python-expressions'):
             # for perfs, don't check object if nothing is forbidden
             return

wcs/backoffice/management.py

@@ -4513,11 +4513,10 @@ class MassActionAfterJob(AfterJob):
             # action not found
             return
 
-        if item_ids:
-            oldest_lazy_form = formdef.data_class().get(item_ids[0]).get_as_lazy()
         self.total_count = len(item_ids)
         self.store()
 
+        oldest_lazy_form = None
         publisher = get_publisher()
         for i, formdata_id in enumerate(item_ids):
             # do not load all formdatas at once as they can be modified during the loop
@@ -4525,6 +4524,8 @@ class MassActionAfterJob(AfterJob):
             formdata = formdef.data_class().get(formdata_id, ignore_errors=True)
             if not formdata:
                 continue
+            if oldest_lazy_form is None:
+                oldest_lazy_form = formdata.get_as_lazy()
             publisher.reset_formdata_state()
             publisher.substitutions.feed(user)
             publisher.substitutions.feed(formdef)

wcs/backoffice/submission.py

@@ -504,7 +504,6 @@ class SubmissionDirectory(Directory):
             if redirect_url:
                 return redirect(redirect_url)
 
-        get_response().breadcrumb.append(('submission/', _('Submission')))
         get_response().set_title(_('Submission'))
 
         list_forms = self.get_submittable_formdefs(prefetch=False)
@@ -587,7 +586,7 @@ class SubmissionDirectory(Directory):
 
         if get_request().form.get('ajax') == 'true':
             get_request().ignore_session = True
-            get_response().filter = {'raw': True}
+            get_response().raw = True
             return r.getvalue()
 
         rt = TemplateIO(html=True)

wcs/blocks.py

@@ -179,7 +179,7 @@ class BlockDef(StorableObject):
         return root
 
     @classmethod
-    def import_from_xml(cls, fd, include_id=False, check_datasources=True, check_deprecated=True):
+    def import_from_xml(cls, fd, include_id=False, check_datasources=True, check_deprecated=False):
         try:
             tree = ET.parse(fd)
         except Exception:
@@ -203,7 +203,7 @@ class BlockDef(StorableObject):
 
     @classmethod
     def import_from_xml_tree(
-        cls, tree, include_id=False, check_datasources=True, check_deprecated=True, **kwargs
+        cls, tree, include_id=False, check_datasources=True, check_deprecated=False, **kwargs
     ):
         from wcs.backoffice.deprecations import DeprecatedElementsDetected, DeprecationsScan
 
@@ -498,6 +498,17 @@ class BlockWidget(WidgetList):
             **kwargs,
         )
 
+    @property
+    def a11y_labelledby(self):
+        return bool(self.a11y_role)
+
+    @property
+    def a11y_role(self):
+        # don't mark block as a group if it has no label
+        if self.label_display != 'hidden':
+            return 'group'
+        return None
+
     def set_value(self, value):
         from .fields.block import BlockRowValue
 
@@ -566,7 +577,9 @@ class BlockWidget(WidgetList):
     def render_title(self, title):
         attrs = {'id': 'form_label_%s' % self.get_name_for_id()}
         if not title or self.label_display == 'hidden':
-            return htmltag('span', **attrs) + htmltext('</span>')
+            # add a tag even if there's no label to display as it's used as an anchor point
+            # for links to errors.
+            return htmltag('div', **attrs) + htmltext('</div>')
 
         if self.label_display == 'normal':
             return super().render_title(title)

wcs/carddata.py

@@ -29,7 +29,7 @@ class CardData(FormData):
     def get_data_source_structured_item(
         self, digest_key='default', group_by=None, with_related_urls=False, with_files_urls=False
     ):
-        if self.digests is None:
+        if not self.digests:
             if digest_key == 'default':
                 summary = _('Digest (default) not defined')
             else:
@@ -122,11 +122,18 @@ class CardData(FormData):
         return '/api/card-file-by-token/%s' % token.id
 
     def update_related(self):
+        if self.is_draft():
+            return
         if self.formdef.reverse_relations:
             job = UpdateRelationsAfterJob(carddata=self)
             if get_response():
-                job.store()
-                get_response().add_after_job(job)
+                job._update_key = (self._formdef.id, self.id)
+                # do not register/run job if an identical job is already planned
+                if job._update_key not in (
+                    getattr(x, '_update_key', None) for x in get_response().after_jobs or []
+                ):
+                    job.store()
+                    get_response().add_after_job(job)
             else:
                 job.execute()
         self._has_changed_digest = False
@@ -149,7 +156,7 @@ class UpdateRelationsAfterJob(AfterJob):
         update_related_seen = get_publisher()._update_related_seen
 
         try:
-            carddef = CardDef.get(self.kwargs['carddef_id'])
+            carddef = CardDef.cached_get(self.kwargs['carddef_id'])
             carddata = carddef.data_class().get(self.kwargs['carddata_id'])
         except KeyError:
             # card got removed (probably the afterjob met some unexpected delay), ignore.
@@ -162,7 +169,7 @@ class UpdateRelationsAfterJob(AfterJob):
             obj_type, obj_slug = obj_ref.split(':')
             obj_class = klass.get(obj_type)
             try:
-                objdef = obj_class.get_by_slug(obj_slug)
+                objdef = obj_class.get_by_slug(obj_slug, use_cache=True)
             except KeyError:
                 continue
             criterias = []

wcs/carddef.py

@@ -49,6 +49,7 @@ class CardDef(FormDef):
     item_name_plural = pgettext_lazy('item', 'cards')
 
     confirmation = False
+    history_pane_default_mode = 'collapsed'
 
     # users are not allowed to access carddata where they're submitter.
     user_allowed_to_access_own_data = False
@@ -143,6 +144,10 @@ class CardDef(FormDef):
         self.roles = self.backoffice_submission_roles
         return super().store(comment=comment, *args, **kwargs)
 
+    def update_category_reference(self):
+        # only relevant for formdefs
+        pass
+
     @classmethod
     def get_carddefs_as_data_source(cls):
         carddefs_by_id = {}
@@ -199,7 +204,7 @@ class CardDef(FormDef):
         assert data_source_id.startswith('carddef:')
         parts = data_source_id.split(':')
         try:
-            carddef = cls.get_by_urlname(parts[1])
+            carddef = cls.get_by_urlname(parts[1], use_cache=True)
         except KeyError:
             return []
         criterias = [StrictNotEqual('status', 'draft'), Null('anonymised')]
@@ -294,7 +299,7 @@ class CardDef(FormDef):
         if len(parts) != 3:
             return []
         try:
-            carddef = cls.get_by_urlname(parts[1])
+            carddef = cls.get_by_urlname(parts[1], use_cache=True)
         except KeyError:
             return []
         custom_view = cls.get_data_source_custom_view(data_source_id, carddef=carddef)
@@ -311,6 +316,24 @@ class CardDef(FormDef):
                 return True
         return False
 
+    def get_default_management_sidebar_items(self):
+        management_sidebar_items = {
+            'general',
+            'submission-context',
+            'user',
+            'geolocation',
+            'custom-template',
+        }
+        if not self.user_support:
+            management_sidebar_items.remove('user')
+        return management_sidebar_items
+
+    def get_management_sidebar_available_items(self):
+        excluded_parts = ['pending-forms']
+        if not self.user_support:
+            excluded_parts.append('user')
+        return [x for x in super().get_management_sidebar_available_items() if x[0] not in excluded_parts]
+
 
 def get_cards_graph(category=None, show_orphans=False):
     out = io.StringIO()

wcs/ctl/management/commands/hobo_deploy.py

@@ -427,7 +427,7 @@ class Command(TenantCommand):
 
     def configure_site_options(self, current_service, pub, ignore_timestamp=False):
         # configure site-options.cfg
-        config = configparser.RawConfigParser()
+        config = configparser.ConfigParser(interpolation=None)
         site_options_filepath = os.path.join(pub.app_dir, 'site-options.cfg')
         if os.path.exists(site_options_filepath):
             config.read(site_options_filepath)

wcs/custom_views.py

@@ -58,9 +58,9 @@ class CustomView(StorableObject):
     @property
     def formdef(self):
         if self.formdef_type == 'formdef':
-            return FormDef.get(self.formdef_id)
+            return FormDef.cached_get(self.formdef_id)
         else:
-            return CardDef.get(self.formdef_id)
+            return CardDef.cached_get(self.formdef_id)
 
     @formdef.setter
     def formdef(self, value):

wcs/data_sources.py

@@ -917,7 +917,7 @@ class NamedDataSource(XmlStorableObject):
         return root
 
     @classmethod
-    def import_from_xml_tree(cls, tree, include_id=False, check_deprecated=True, **kwargs):
+    def import_from_xml_tree(cls, tree, include_id=False, check_deprecated=False, **kwargs):
         from wcs.backoffice.deprecations import DeprecatedElementsDetected, DeprecationsScan
 
         data_source = super().import_from_xml_tree(
@@ -940,8 +940,7 @@ class NamedDataSource(XmlStorableObject):
         data_source = super().get_by_slug(slug, ignore_errors=ignore_errors)
         if data_source is None:
             if stub_fallback:
-                if slug != 'inspect_collapse':
-                    get_logger().warning("data source '%s' does not exist" % slug)
+                get_logger().warning("data source '%s' does not exist" % slug)
                 return StubNamedDataSource(name=slug)
         return data_source
 
@@ -1219,6 +1218,8 @@ class StubNamedDataSource(NamedDataSource):
 
 class DataSourcesSubstitutionProxy:
     def __getattr__(self, attr):
+        if attr == 'inspect_collapse':
+            return True
         return DataSourceProxy(attr)
 
     def inspect_keys(self):

wcs/fields/base.py

@@ -443,6 +443,11 @@ class Field:
                 if xml_node_text(node.find('locked')) == 'True':
                     self.prefill['locked'] = True
 
+    def display_locations_export_to_xml(self, node, include_id=False):
+        display_locations_node = ET.SubElement(node, 'display_locations')
+        for v in self.display_locations or []:
+            ET.SubElement(display_locations_node, 'item').text = force_str(v)
+
     def get_rst_view_value(self, value, indent=''):
         return indent + self.get_view_value(value)
 

wcs/fields/block.py

@@ -142,7 +142,10 @@ class BlockField(WidgetField):
 
     def get_dependencies(self):
         yield from super().get_dependencies()
-        yield self.block
+        try:
+            yield self.block
+        except KeyError:
+            pass
 
     def add_to_form(self, form, value=None):
         try:

wcs/fields/file.py

@@ -16,6 +16,7 @@
 
 import base64
 import os
+import urllib.parse
 import xml.etree.ElementTree as ET
 
 from django.utils.encoding import force_bytes, force_str
@@ -139,6 +140,20 @@ class FileField(WidgetField):
             upload = PicklableUpload(value.filename, value.content_type)
             upload.receive([value.content])
             return upload
+
+        value = misc.unlazy(value)
+        if isinstance(value, str) and urllib.parse.urlparse(value).scheme in ('http', 'https'):
+            try:
+                response, dummy, data, dummy = misc.http_get_page(value, raise_on_http_errors=True)
+            except misc.ConnectionError:
+                pass
+            else:
+                value = {
+                    'filename': os.path.basename(urllib.parse.urlparse(value).path) or _('file.bin'),
+                    'content': data,
+                    'content_type': response.headers.get('content-type'),
+                }
+
         if isinstance(value, dict):
             # if value is a dictionary we expect it to have a content or
             # b64_content key and a filename keys and an optional

wcs/fields/numeric.py

@@ -78,6 +78,9 @@ class NumericField(WidgetField):
             return value
         return django_number_format(value, use_l10n=True)
 
+    def get_json_value(self, value, **kwargs):
+        return str(value)
+
     def from_json_value(self, value):
         try:
             return misc.parse_decimal(value, do_raise=True, keep_none=True)

wcs/formdata.py

@@ -514,7 +514,7 @@ class FormData(StorableObject):
             # response.
             fields['id_display'] = self.formdef.get_display_id_format().strip()
 
-        changed = False
+        changed = set()
 
         def get_all_fields(with_backoffice_fields=False):
             fields = self.formdef.get_all_fields() if with_backoffice_fields else self.formdef.fields
@@ -551,7 +551,7 @@ class FormData(StorableObject):
             user_object.set_attributes_from_formdata(form_user_data)
             if user_object.name != self.user_label:
                 self.user_label = user_object.name
-                changed = True
+                changed.add('user_label')
 
         if any(fields.values()):
             context = self.get_substitution_variables()
@@ -593,6 +593,7 @@ class FormData(StorableObject):
                 if attribute.startswith('template:'):
                     key = attribute[9:]
                     if new_value != (self.digests or {}).get(key):
+                        changed.add('digests')
                         digests[key] = new_value
 
                     if i18n_enabled and template and '|translate' in template and new_value != 'ERROR':
@@ -605,15 +606,14 @@ class FormData(StorableObject):
                                 except Exception:
                                     continue
                             if new_value != (self.digests or {}).get(key):
+                                changed.add('digests')
                                 digests[key] = new_value
 
                 else:
                     if new_value != getattr(self, attribute, None):
                         setattr(self, attribute, new_value)
-                        changed = True
-            if digests:
-                self.digests = digests
-                changed = True
+                        changed.add('digests')
+            self.digests = digests
 
         new_statistics_data = {}
         for field in get_all_fields(with_backoffice_fields=True):
@@ -641,7 +641,7 @@ class FormData(StorableObject):
 
         if new_statistics_data != self.statistics_data:
             self.statistics_data = new_statistics_data
-            changed = True
+            changed.add('statistics_data')
 
         new_relations_data = collections.defaultdict(set)
         for relation in self.iter_target_datas():
@@ -653,7 +653,7 @@ class FormData(StorableObject):
         new_relations_data = {k: list(v) for k, v in sorted(new_relations_data.items())}
         if new_relations_data != self.relations_data:
             self.relations_data = new_relations_data
-            changed = True
+            changed.add('relations_data')
 
         return changed
 
@@ -1940,7 +1940,7 @@ class FormData(StorableObject):
                     elif obj_type == 'carddef':
                         obj_class = CardDef
                     try:
-                        _objectdef = obj_class.get_by_urlname(slug)
+                        _objectdef = obj_class.get_by_urlname(slug, use_cache=True)
                     except KeyError:
                         yield (
                             _('Linked object def by id %(object_id)s') % {'object_id': slug},

wcs/formdef.py

@@ -176,14 +176,7 @@ class FormDef(StorableObject):
     expiration_date = None
     has_captcha = False
     skip_from_360_view = False
-    management_sidebar_items = {
-        'general',
-        'submission-context',
-        'user',
-        'geolocation',
-        'custom-template',
-        'pending-forms',
-    }
+    management_sidebar_items = {'__default__'}
     include_download_all_button = False
     appearance_keywords = None
     digest_templates = None
@@ -195,6 +188,8 @@ class FormDef(StorableObject):
     user_support = None
 
     geolocations = None
+    history_pane_default_mode = 'expanded'
+    sql_integrity_errors = None
 
     # store reverse relations
     reverse_relations = None
@@ -242,7 +237,6 @@ class FormDef(StorableObject):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.fields = []
-        self.management_sidebar_items = {'__default__'}
 
     def __eq__(self, other):
         return bool(
@@ -280,7 +274,7 @@ class FormDef(StorableObject):
                     break
 
         if self.include_download_all_button:  # 2023-12-30
-            self.management_sidebar_items = self.__class__.management_sidebar_items.copy()
+            self.management_sidebar_items = self.get_default_management_sidebar_items()
             self.management_sidebar_items.add('download-files')
             self.include_download_all_button = False
             changed = True
@@ -305,6 +299,16 @@ class FormDef(StorableObject):
                 sql.clean_global_views(conn, cur)
             cur.close()
 
+    def get_default_management_sidebar_items(self):
+        return {
+            'general',
+            'submission-context',
+            'user',
+            'geolocation',
+            'custom-template',
+            'pending-forms',
+        }
+
     def get_management_sidebar_available_items(self):
         return [
             ('general', _('General Information')),
@@ -325,7 +329,7 @@ class FormDef(StorableObject):
 
     def get_management_sidebar_items(self):
         if self.management_sidebar_items == {'__default__'}:
-            return self.__class__.management_sidebar_items
+            return self.get_default_management_sidebar_items()
         return self.management_sidebar_items or []
 
     @property
@@ -474,6 +478,14 @@ class FormDef(StorableObject):
             self.update_storage()
             self.store_related_custom_views()
             self.update_searchable_formdefs_table()
+            self.update_category_reference()
+
+    def update_category_reference(self):
+        if getattr(self, '_onload_category_id', None) != self.category_id:
+            from . import sql
+
+            sql.update_global_view_formdef_category(self)
+            self._onload_category_id = self.category_id
 
     def has_captcha_enabled(self):
         return self.has_captcha and get_publisher().has_site_option('formdef-captcha-option')
@@ -576,6 +588,23 @@ class FormDef(StorableObject):
     def get_all_fields(self):
         return (self.fields or []) + self.workflow.get_backoffice_fields()
 
+    def get_all_fields_dict(self):
+        return {x.id: x for x in self.get_all_fields()}
+
+    def get_total_count_data_fields(self):
+        count = len([x for x in self.fields or [] if not x.is_no_data_field and not x.key == 'block'])
+        for field in self.fields or []:
+            if not field.key == 'block':
+                continue
+            try:
+                count += (
+                    len([x for x in field.block.fields or [] if not x.is_no_data_field])
+                    * field.default_items_count
+                )
+            except KeyError:
+                continue
+        return count
+
     def iter_fields(self, include_block_fields=False, with_backoffice_fields=True, with_no_data_fields=True):
         def _iter_fields(fields, block_field=None):
             for field in fields:
@@ -651,10 +680,9 @@ class FormDef(StorableObject):
 
         if self.workflow_id:
             try:
-                workflow = Workflow.get(self.workflow_id)
+                self._workflow = Workflow.get(self.workflow_id)
             except KeyError:
                 return Workflow.get_unknown_workflow()
-            self._workflow = self.get_workflow_with_options(workflow)
             return self._workflow
         else:
             self._workflow = self.get_default_workflow()
@@ -666,20 +694,6 @@ class FormDef(StorableObject):
 
         return Workflow.get_default_workflow()
 
-    def get_workflow_with_options(self, workflow):
-        # this needs to be kept in sync with admin/forms.ptl,
-        # FormDefPage::workflow
-        if not self.workflow_options:
-            return workflow
-        for status in workflow.possible_status:
-            for item in status.items:
-                prefix = '%s*%s*' % (status.id, item.id)
-                for parameter in item.get_parameters():
-                    value = self.workflow_options.get(prefix + parameter)
-                    if value:
-                        setattr(item, parameter, value)
-        return workflow
-
     def set_workflow(self, workflow):
         if workflow and workflow.id not in ['_carddef_default', '_default']:
             self.workflow_id = workflow.id
@@ -774,9 +788,13 @@ class FormDef(StorableObject):
         self.workflow_options.update(variables)
 
     @classmethod
-    def get_by_urlname(cls, url_name, ignore_migration=False, ignore_errors=False):
+    def get_by_urlname(cls, url_name, ignore_migration=False, ignore_errors=False, use_cache=False):
         return cls.get_on_index(
-            url_name, 'url_name', ignore_migration=ignore_migration, ignore_errors=ignore_errors
+            url_name,
+            'url_name',
+            ignore_migration=ignore_migration,
+            ignore_errors=ignore_errors,
+            use_cache=use_cache,
         )
 
     get_by_slug = get_by_urlname
@@ -1474,7 +1492,7 @@ class FormDef(StorableObject):
 
     @classmethod
     def import_from_xml(
-        cls, fd, include_id=False, fix_on_error=False, check_datasources=True, check_deprecated=True
+        cls, fd, include_id=False, fix_on_error=False, check_datasources=True, check_deprecated=False
     ):
         try:
             tree = ET.parse(fd)
@@ -1514,7 +1532,7 @@ class FormDef(StorableObject):
         fix_on_error=False,
         snapshot=False,
         check_datasources=True,
-        check_deprecated=True,
+        check_deprecated=False,
     ):
         from wcs.backoffice.deprecations import DeprecatedElementsDetected, DeprecationsScan
         from wcs.carddef import CardDef
@@ -2015,6 +2033,8 @@ class FormDef(StorableObject):
             del odict['_custom_views']
         if '_import_orig_slug' in odict:
             del odict['_import_orig_slug']
+        if '_onload_category_id' in odict:
+            del odict['_onload_category_id']
         return odict
 
     def __setstate__(self, dict):
@@ -2029,6 +2049,7 @@ class FormDef(StorableObject):
     @classmethod
     def storage_load(cls, fd, **kwargs):
         o = super().storage_load(fd)
+        o._onload_category_id = o.category_id  # keep track of category, to update wcs_all_forms if changed
         if kwargs.get('lightweight'):
             o.fields = Ellipsis
             return o
@@ -2306,6 +2327,10 @@ def update_storage_all_formdefs(publisher, **kwargs):
 
     for formdef in itertools.chain(FormDef.select(), CardDef.select()):
         formdef.update_storage()
+        if formdef.sql_integrity_errors:
+            # print errors, this will get them in the cron output, that hopefully
+            # a sysadmin will read.
+            print(f'! Integrity errors in {formdef.get_admin_url()}')
 
 
 def get_formdefs_of_all_kinds(**kwargs):
@@ -2360,7 +2385,7 @@ class UpdateDigestAfterJob(AfterJob):
     def execute(self):
         for formdef_class, formdef_id in self.kwargs['formdefs']:
             formdef = formdef_class.get(formdef_id)
-            for formdata in formdef.data_class().select(order_by='id'):
+            for formdata in formdef.data_class().select_iterator(order_by='id', itersize=200):
                 formdata.store()
 
 

wcs/forms/actions.py

@@ -31,15 +31,17 @@ from wcs.wf.jump import jump_and_perform
 from wcs.workflows import perform_items, push_perform_workflow
 
 
-class MissingOrExpiredToken(PublishError):
+class InvalidActionLink(PublishError):
     status_code = 404
     title = _('Error')
+    description = _('This action link is no longer valid.')
+
+
+class MissingOrExpiredToken(InvalidActionLink):
     description = _('This action link has already been used or has expired.')
 
 
-class MissingFormdata(PublishError):
-    status_code = 404
-    title = _('Error')
+class MissingFormdata(InvalidActionLink):
     description = _('This action link is no longer valid as the attached form has been removed.')
 
 
@@ -78,6 +80,9 @@ class ActionDirectory(Directory, FormTemplateMixin):
             raise MissingFormdata()
         self.action = None
         status = self.formdata.get_status()
+        if not status or not status.items:
+            # unknown status or workflow change and no actions anymore
+            raise InvalidActionLink()
         for item in status.items:
             if getattr(item, 'identifier', None) == self.token.context['action_id']:
                 self.action = item

wcs/forms/common.py

@@ -541,7 +541,7 @@ class FormStatusPage(Directory, FormTemplateMixin):
         return False
 
     def should_fold_history(self):
-        return False
+        return bool(self.formdef.history_pane_default_mode == 'collapsed')
 
     def receipt(self, always_include_user=False, form_url='', mine=True):
         request_user = user = get_request().user

wcs/forms/root.py

@@ -1372,6 +1372,9 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
                 # a new ConditionsVars will get added to the substitution
                 # variables.
                 form_data = copy.copy(session.get_by_magictoken(magictoken, {}))
+                if form_data:
+                    # keep new copy in session
+                    session.add_magictoken(magictoken, form_data)
                 try:
                     with get_publisher().substitutions.temporary_feed(transient_formdata, force_mode='lazy'):
                         data = self.formdef.get_data(form, raise_on_error=True)

wcs/locale/fr/LC_MESSAGES/django.po

@@ -4,8 +4,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: wcs 0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-03-21 19:00+0100\n"
-"PO-Revision-Date: 2024-03-21 19:00+0100\n"
+"POT-Creation-Date: 2024-04-09 11:26+0200\n"
+"PO-Revision-Date: 2024-04-09 11:26+0200\n"
 "Last-Translator: Thomas Noël <tnoel@entrouvert.com>\n"
 "Language-Team: french\n"
 "Language: fr\n"
@@ -42,13 +42,13 @@ msgstr "Clé d’accès"
 msgid "Restrict to anonymised data"
 msgstr "Limiter aux données anonymisées"
 
-#: admin/api_access.py admin/roles.py admin/settings.py admin/users.py
-#: api_export_import.py backoffice/root.py
+#: admin/api_access.py admin/roles.py admin/settings.py admin/tests.py
+#: admin/users.py api_export_import.py backoffice/root.py
 msgid "Roles"
 msgstr "Rôles"
 
 #: admin/api_access.py admin/categories.py admin/data_sources.py admin/forms.py
-#: admin/users.py wf/resubmit.py
+#: admin/tests.py admin/users.py wf/resubmit.py
 msgid "Add Role"
 msgstr "Ajouter un rôle"
 
@@ -145,6 +145,7 @@ msgstr "Accès aux API"
 #: admin/api_access.py admin/categories.py admin/data_sources.py admin/forms.py
 #: admin/roles.py admin/tests.py admin/users.py admin/workflows.py
 #: admin/wscalls.py backoffice/cards.py qommon/ident/idp.py statistics/views.py
+#: templates/wcs/backoffice/test-users.html
 #: templates/wcs/backoffice/test-webservice-responses.html
 #: templates/wcs/backoffice/tests.html workflows.py
 msgid "New"
@@ -205,6 +206,7 @@ msgstr "Utilisation"
 #: templates/wcs/backoffice/formdef.html templates/wcs/backoffice/forms.html
 #: templates/wcs/backoffice/mail-templates.html
 #: templates/wcs/backoffice/snapshots.html
+#: templates/wcs/backoffice/test-users.html
 #: templates/wcs/backoffice/test-webservice-responses.html
 #: templates/wcs/backoffice/test_sidebar.html
 #: templates/wcs/backoffice/tests.html
@@ -232,9 +234,9 @@ msgstr "Dupliquer"
 msgid "Save snapshot"
 msgstr "Enregistrer une sauvegarde"
 
-#: admin/blocks.py admin/forms.py
-msgid "Overwrite"
-msgstr "Écraser"
+#: admin/blocks.py templates/wcs/backoffice/formdef.html
+msgid "Overwrite with new import"
+msgstr "Écraser avec un nouvel import"
 
 #: admin/blocks.py templates/wcs/backoffice/blocks.html
 #: templates/wcs/backoffice/cards.html templates/wcs/backoffice/category.html
@@ -305,6 +307,10 @@ msgstr ""
 msgid "File"
 msgstr "Fichier"
 
+#: admin/blocks.py admin/forms.py
+msgid "Overwrite"
+msgstr "Écraser"
+
 #: admin/blocks.py admin/forms.py
 msgid "Overwritten"
 msgstr "Écrasement"
@@ -1014,6 +1020,12 @@ msgstr ""
 "Il approche les limites du système et de nouveaux champs ne devraient pas "
 "être ajoutés."
 
+#: admin/fields.py
+#, python-format
+msgid "There are at least %d data fields, including fields in blocks."
+msgstr ""
+"Il y a au moins %d champs de données, en comptant les champs dans les blocs."
+
 #: admin/fields.py
 msgid "In a multipage form, the first field should be of type \"page\"."
 msgstr ""
@@ -1220,7 +1232,7 @@ msgstr "Commencer par un CAPTCHA pour les utilisateurs anonymes"
 msgid "CAPTCHA"
 msgstr "CAPTCHA"
 
-#: admin/forms.py
+#: admin/forms.py backoffice/cards.py
 msgid "Sidebar elements"
 msgstr "Contenu de la barre latérale"
 
@@ -1427,11 +1439,11 @@ msgctxt "confirmation page"
 msgid "Disabled"
 msgstr "Désactivée"
 
-#: admin/forms.py
+#: admin/forms.py backoffice/cards.py
 msgid "Custom"
 msgstr "Personnalisé"
 
-#: admin/forms.py workflows.py
+#: admin/forms.py backoffice/cards.py workflows.py
 msgid "Default"
 msgstr "Par défaut"
 
@@ -1714,14 +1726,6 @@ msgstr "Valeur par défaut : %s"
 msgid "Change in workflow variables"
 msgstr "Changement dans les variables de workflow"
 
-#: admin/forms.py
-msgid "Workflow Options"
-msgstr "Options du workflow"
-
-#: admin/forms.py
-msgid "Change in workflow options"
-msgstr "Changement dans les options de workflow"
-
 #: admin/forms.py
 msgid "Import Form"
 msgstr "Importer un formulaire"
@@ -1810,6 +1814,7 @@ msgid "Text"
 msgstr "Texte"
 
 #: admin/logged_errors.py backoffice/management.py formdata.py
+#: templates/wcs/backoffice/includes/inspect-draft-by-page.html
 #: wf/create_formdata.py workflows.py
 msgid "Unknown"
 msgstr "Inconnu"
@@ -2755,6 +2760,35 @@ msgstr "Réponses webservice"
 msgid "New webservice response"
 msgstr "Nouvelle réponse webservice"
 
+#: admin/tests.py
+msgid "Test user label"
+msgstr "Libellé de l’utilisateur de test"
+
+#: admin/tests.py
+msgid "A test user with this email already exists."
+msgstr "Il y a déjà un utilisateur de test avec ce courriel."
+
+#: admin/tests.py
+msgid "Edit test user"
+msgstr "Modifier l’utilisateur de test"
+
+#: admin/tests.py templates/wcs/backoffice/test-users.html
+#: templates/wcs/backoffice/tests.html
+msgid "Test users"
+msgstr "Utilisateurs de test"
+
+#: admin/tests.py
+msgid "Empty user"
+msgstr "Utilisateur vide"
+
+#: admin/tests.py
+msgid "Copy existing user"
+msgstr "Copier un utilisateur existant"
+
+#: admin/tests.py
+msgid "New test user"
+msgstr "Nouvel utilisateur de test"
+
 #: admin/users.py fields/base.py fields/email.py formdata.py formdef.py
 #: forms/root.py qommon/admin/emails.py qommon/ident/franceconnect.py
 #: qommon/ident/idp.py qommon/ident/password.py wf/profile.py wf/sendmail.py
@@ -3200,19 +3234,6 @@ msgstr "Nouveau statut « %s »"
 msgid "New Status"
 msgstr "Nouveau statut"
 
-#: admin/workflows.py
-msgid "or you can use this field to directly replace a workflow parameter:"
-msgstr ""
-"ou vous pouvez utiliser ce champ pour remplacer un paramètre du workflow :"
-
-#: admin/workflows.py
-msgid "This takes priority over a variable name"
-msgstr "Ce choix est prioritaire par rapport au nom de variable"
-
-#: admin/workflows.py qommon/substitution.py
-msgid "Variable"
-msgstr "Variable"
-
 #: admin/workflows.py
 msgid "Default Value"
 msgstr "Valeur par défaut"
@@ -3799,6 +3820,23 @@ msgstr "L’identifiant ne peut pas être modifié car il existe des fiches."
 msgid "Unique identifier template"
 msgstr "Gabarit pour un identifiant unique"
 
+#: backoffice/cards.py
+msgid "History pane default mode"
+msgstr "Affichage par défaut du volet « historique »"
+
+#: backoffice/cards.py
+msgid "Collapsed"
+msgstr "Plié"
+
+#: backoffice/cards.py
+msgid "Expanded"
+msgstr "Déplié"
+
+#: backoffice/cards.py
+msgctxt "cards"
+msgid "Management"
+msgstr "Gestion"
+
 #: backoffice/cards.py
 msgid ""
 "Warning: this field data will be permanently deleted from existing cards."
@@ -5700,6 +5738,10 @@ msgstr ""
 msgid "File storage system"
 msgstr "Système de stockage de fichier"
 
+#: fields/file.py
+msgid "file.bin"
+msgstr "fichier.bin"
+
 #: fields/item.py
 #, python-format
 msgid "unknown card value (%r)"
@@ -6512,6 +6554,10 @@ msgstr "Mise à jour des données pour les statistiques"
 msgid "Error"
 msgstr "Erreur"
 
+#: forms/actions.py
+msgid "This action link is no longer valid."
+msgstr "Ce lien d’action n’est plus valide."
+
 #: forms/actions.py
 msgid "This action link has already been used or has expired."
 msgstr "Ce lien d’action a déjà été utilisé ou est expiré."
@@ -7027,6 +7073,7 @@ msgstr ""
 "dessous :"
 
 #: qommon/admin/menu.py qommon/templates/qommon/forms/widgets/block_sub.html
+#: templates/wcs/backoffice/test-users.html
 #: templates/wcs/backoffice/test-webservice-responses.html
 #: templates/wcs/backoffice/workflow-global-action.html
 msgid "Remove"
@@ -7167,6 +7214,11 @@ msgstr "Unités de temps utilisables : %s."
 msgid "too many characters (limit is %d)"
 msgstr "trop de caractères (la limite est à %d)"
 
+#: qommon/form.py
+#, python-format
+msgid "Failed to convert value for field \"%s\""
+msgstr "Erreur à la conversion de la valeur pour le champ « %s «"
+
 #: qommon/form.py
 #, python-format
 msgid "Failed to set value on field \"%s\""
@@ -7180,11 +7232,6 @@ msgstr "erreur système à l’enregistrement du fichier"
 msgid "unknown storage system (system error)"
 msgstr "système de stockage inconnu (erreur système)"
 
-#: qommon/form.py
-#, python-format
-msgid "over file size limit (%s)"
-msgstr "dépasse la taille limite (%s)"
-
 #: qommon/form.py
 msgid "invalid file type"
 msgstr "type de fichier invalide"
@@ -7193,6 +7240,11 @@ msgstr "type de fichier invalide"
 msgid "forbidden file type"
 msgstr "type de fichier interdit"
 
+#: qommon/form.py
+#, python-format
+msgid "over file size limit (%s)"
+msgstr "dépasse la taille limite (%s)"
+
 #: qommon/form.py
 msgid "You should enter a valid email address, for example name@example.com."
 msgstr "Veuillez saisir une adresse électronique, par exemple nom@example.com."
@@ -7573,6 +7625,10 @@ msgstr "jour"
 msgid "days"
 msgstr "jours"
 
+#: qommon/humantime.py
+msgid "day(s)"
+msgstr "jour(s)"
+
 #: qommon/humantime.py
 msgid "hour"
 msgstr "heure"
@@ -7581,6 +7637,34 @@ msgstr "heure"
 msgid "hours"
 msgstr "heures"
 
+#: qommon/humantime.py
+msgid "hour(s)"
+msgstr "heure(s)"
+
+#: qommon/humantime.py
+msgid "minute"
+msgstr "minute"
+
+#: qommon/humantime.py
+msgid "minutes"
+msgstr "minutes"
+
+#: qommon/humantime.py
+msgid "minute(s)"
+msgstr "minute(s)"
+
+#: qommon/humantime.py
+msgid "second"
+msgstr "seconde"
+
+#: qommon/humantime.py
+msgid "seconds"
+msgstr "secondes"
+
+#: qommon/humantime.py
+msgid "second(s)"
+msgstr "seconde(s)"
+
 #: qommon/humantime.py
 msgid "month"
 msgstr "mois"
@@ -7589,6 +7673,10 @@ msgstr "mois"
 msgid "months"
 msgstr "mois"
 
+#: qommon/humantime.py
+msgid "month(s)"
+msgstr "mois"
+
 #: qommon/humantime.py
 msgid "year"
 msgstr "année"
@@ -7598,20 +7686,8 @@ msgid "years"
 msgstr "années"
 
 #: qommon/humantime.py
-msgid "minute"
-msgstr "minute"
-
-#: qommon/humantime.py
-msgid "minutes"
-msgstr "minutes"
-
-#: qommon/humantime.py
-msgid "second"
-msgstr "seconde"
-
-#: qommon/humantime.py
-msgid "seconds"
-msgstr "secondes"
+msgid "year(s)"
+msgstr "année(s)"
 
 #: qommon/humantime.py
 #, python-format
@@ -8931,13 +9007,11 @@ msgstr "Désolé"
 
 #: qommon/publisher.py
 msgid ""
-"Map data &copy; <a href='https://openstreetmap.org'>OpenStreetMap</a> "
-"contributors, <a href='http://creativecommons.org/licenses/by-sa/2.0/'>CC-BY-"
-"SA</a>"
+"Map data &copy; <a href=\"https://www.openstreetmap.org/"
+"copyright\">OpenStreetMap</a>"
 msgstr ""
-"Données &copy; contributeurs <a href='https://openstreetmap."
-"org'>OpenStreetMap</a>, <a href='http://creativecommons.org/licenses/by-"
-"sa/2.0/deed.fr'>CC-BY-SA</a>"
+"Données cartographiques &copy; <a href=\"https://www.openstreetmap.org/"
+"copyright\">OpenStreetMap</a>"
 
 #: qommon/publisher.py
 msgid "Belgian eID"
@@ -9026,6 +9100,10 @@ msgstr "Impossibilité de communiquer avec le fournisseur d’identités."
 msgid "Authentication error"
 msgstr "Erreur d’authentification"
 
+#: qommon/substitution.py
+msgid "Variable"
+msgstr "Variable"
+
 #: qommon/template.py
 msgid "the homepage"
 msgstr "la page d’accueil"
@@ -9205,8 +9283,8 @@ msgstr "{%% temporary_action_button %%} nécessite un paramètre « label »"
 
 #: qommon/templatetags/qommon.py
 #, python-format
-msgid "|%s used on invalid queryset (%r)"
-msgstr "|%s utilisé sur une requête invalide (%r)"
+msgid "|%s used on something else than a queryset (%r)"
+msgstr "|%s utilisé sur autre chose qu’une requête (%r)"
 
 #: qommon/templatetags/qommon.py
 #, python-format
@@ -9218,6 +9296,10 @@ msgstr "|objects appelé sur une source invalide (%r)"
 msgid "|objects with invalid reference (%r)"
 msgstr "|objects utilisé avec une référence invalide (%r)"
 
+#: qommon/templatetags/qommon.py
+msgid "|count used on uncountable value"
+msgstr "|count utilisé sur une valeur non dénombrable"
+
 #: qommon/templatetags/qommon.py
 #, python-format
 msgid "|convert_image_format: unknown format (must be one of %s)"
@@ -9422,6 +9504,10 @@ msgstr "Identifiant d’accès :"
 msgid "Access key:"
 msgstr "Clé d’accès :"
 
+#: templates/wcs/backoffice/api_access.html
+msgid "API client from identity provider, identifier:"
+msgstr "Client d’API du fournisseur d’identité, identifiant :"
+
 #: templates/wcs/backoffice/api_access.html
 msgid "Restricted to anonymised data"
 msgstr "Limité aux données anonymisées"
@@ -9732,6 +9818,10 @@ msgstr ""
 "tel qu’il existe actuellement, pas nécessairement tel qu’il était au moment "
 "de l’exécution."
 
+#: templates/wcs/backoffice/formdef-inspect.html
+msgid "Drafts"
+msgstr "Brouillons"
+
 #: templates/wcs/backoffice/formdef-inspect.html
 msgid "Custom views"
 msgstr "Vues personnalisées"
@@ -9779,6 +9869,35 @@ msgid_plural "%(page_count)s pages"
 msgstr[0] "%(page_count)s page"
 msgstr[1] "%(page_count)s pages"
 
+#: templates/wcs/backoffice/formdef-inspect.html
+msgid "Key indicators on existing drafts"
+msgstr "Indicateurs clés sur les brouillons existants"
+
+#: templates/wcs/backoffice/formdef-inspect.html
+#, python-format
+msgid "Covered period: last %(count)s days."
+msgstr "Période concernée : %(count)s derniers jours."
+
+#: templates/wcs/backoffice/formdef-inspect.html
+msgid "Rate for in-progress forms, by page"
+msgstr "Taux de demandes en cours de saisie, par page"
+
+#: templates/wcs/backoffice/formdef-inspect.html
+msgid "Completion rate: count of submitted forms, against count of drafts"
+msgstr ""
+"Taux d’achèvement : nombre de demandes enregistrées, sur le nombre total de "
+"brouillons"
+
+#: templates/wcs/backoffice/formdef-inspect.html
+#: templates/wcs/backoffice/includes/inspect-draft-by-page.html
+#, python-format
+msgid "%%"
+msgstr " %%"
+
+#: templates/wcs/backoffice/formdef-inspect.html
+msgid "There are currently no drafts for this form."
+msgstr "Il n’y a actuellement pas de brouillons pour cette démarche."
+
 #: templates/wcs/backoffice/formdef-inspect.html
 msgid "Default view"
 msgstr "Vue par défaut"
@@ -9799,10 +9918,6 @@ msgstr "Afficher le code QR"
 msgid "change title"
 msgstr "changer le titre"
 
-#: templates/wcs/backoffice/formdef.html
-msgid "Overwrite with new import"
-msgstr "Écraser avec un nouvel import"
-
 #: templates/wcs/backoffice/formdef.html
 msgid "Preview Online"
 msgstr "Aperçu en ligne"
@@ -9899,10 +10014,29 @@ msgstr "Publié à partir du %(date1)s"
 msgid "Published until %(date2)s"
 msgstr "Publié jusqu’au %(date2)s"
 
+#: templates/wcs/backoffice/includes/inspect-draft-by-page.html
+msgid "Only page"
+msgstr "Page unique"
+
+#: templates/wcs/backoffice/includes/inspect-draft-by-page.html
+msgid "Confirmation page"
+msgstr "Page de confirmation"
+
 #: templates/wcs/backoffice/includes/mail-templates.html
 msgid "There are no mail templates defined."
 msgstr "Il n’y a pas de modèle de courriel défini."
 
+#: templates/wcs/backoffice/includes/sql-fields-integrity.html
+msgid "There are integrity errors in the database column types."
+msgstr ""
+"Il y a des erreurs d’intégrité dans les types de colonne de la base de "
+"données."
+
+#: templates/wcs/backoffice/includes/sql-fields-integrity.html
+#, python-format
+msgid "expected: %(expected)s, got: %(got)s."
+msgstr "attendu : %(expected)s, récupéré : %(got)s."
+
 #: templates/wcs/backoffice/includes/test-result-fragment.html
 #: wf/display_message.py wf/register_comment.py
 msgid "Success"
@@ -10266,6 +10400,10 @@ msgstr "Démarré par"
 msgid "No test results yet."
 msgstr "Pas encore de résultats des tests."
 
+#: templates/wcs/backoffice/test-users.html
+msgid "There are no test users yet."
+msgstr "Il n’y a pas encore d’utilisateurs de tests."
+
 #: templates/wcs/backoffice/test-webservice-responses.html
 #: wf/assign_carddata.py wf/create_formdata.py wf/redirect_to_url.py
 #: wf/roles.py workflow_tests.py
@@ -10461,16 +10599,6 @@ msgstr "Plein écran"
 msgid "Unforce Terminal Status"
 msgstr "Ne plus forcer le caractère final"
 
-#: templates/wcs/backoffice/workflow-tests.html
-msgid "Backoffice user is not defined, workflow tests will not be executed."
-msgstr ""
-"L’utilisateur agent n’est pas défini, les tests de workflow ne seront pas "
-"exécutés."
-
-#: templates/wcs/backoffice/workflow-tests.html
-msgid "Open test options"
-msgstr "Accéder aux options"
-
 #: templates/wcs/backoffice/workflow-tests.html
 msgid "There are no workflow test actions yet."
 msgstr "Il n’y a pas encore d’actions de test."
@@ -10783,6 +10911,11 @@ msgstr "Valeur invalide (%r) pour le filtre « order_by »"
 msgid "Invalid operator \"%(operator)s\" for filter \"%(filter)s\""
 msgstr "Opérateur « %(operator)s » invalide pour le filtre « %(filter)s »"
 
+#: variables.py
+#, python-format
+msgid "Unknown custom view \"%(slug)s\""
+msgstr "Vue personnalisée « %(slug)s »"
+
 #: variables.py
 #, python-format
 msgid "invalid value for distance (%r)"
@@ -10892,6 +11025,10 @@ msgstr "Géolocalisation : position non disponible"
 msgid "Geolocation: timeout"
 msgstr "Géolocalisation : délai expiré"
 
+#: views.py
+msgid "Marker of selected position"
+msgstr "Marqueur pointant la position sélectionnée"
+
 #: views.py
 msgid "An error occured while fetching results"
 msgstr "Erreur à la récupération des résultats"
@@ -10912,6 +11049,10 @@ msgstr "Dézoomer"
 msgid "Display my position"
 msgstr "Afficher ma position"
 
+#: views.py
+msgid "Leaflet, a JavaScript library for interactive maps"
+msgstr "Leaflet, une bibliothèque JavaScript pour des cartes interactives"
+
 #: views.py
 msgid "The results could not be loaded"
 msgstr "Les résultats ne peuvent pas être chargés"
@@ -12246,6 +12387,16 @@ msgstr "Cassé, utilisateur manquant"
 msgid "Button \"%s\" is not displayed."
 msgstr "Le bouton « %s » n’est pas affiché."
 
+#: workflow_tests.py
+msgid "Selected user is \"Backoffice user\" but it is not defined."
+msgstr ""
+"L’utilisateur sélectionné est « Utilisateur agent » mais celui-ci n’est pas "
+"défini."
+
+#: workflow_tests.py
+msgid "Open test options"
+msgstr "Accéder aux options"
+
 #: workflow_tests.py
 msgid "not available"
 msgstr "pas disponible"
@@ -12293,6 +12444,16 @@ msgstr "Vérifier l’envoi d’un courriel"
 msgid "Email to \"%s\""
 msgstr "Courriel vers « %s »"
 
+#: workflow_tests.py
+#, python-format
+msgid "Subject must contain \"%s\""
+msgstr "Le sujet doit contenir « %s »"
+
+#: workflow_tests.py
+#, python-format
+msgid "Body must contain \"%s\""
+msgstr "Le corps doit contenir « %s »"
+
 #: workflow_tests.py
 msgid "No email was sent."
 msgstr "Aucun courriel envoyé."
@@ -12738,6 +12899,15 @@ msgid "Reindexing cards and forms after workflow change"
 msgstr ""
 "Ré-indexation des demandes et des fiches après modification du workflow"
 
+#: workflows.py
+#, python-format
+msgid "Workflow: %s"
+msgstr "Workflow : %s"
+
+#: workflows.py
+msgid "Change in workflow"
+msgstr "Modification dans le workflow"
+
 #: workflows.py
 msgid "Previously Marked Status"
 msgstr "Statut précédemment marqué"
@@ -12880,6 +13050,10 @@ msgstr ""
 msgid "Current Status"
 msgstr "Statut actuel"
 
+#: workflows.py
+msgid "invalid value, out of bounds"
+msgstr "valeur choisie invalide, hors des bornes"
+
 #: workflows.py
 msgid "Delay (in days)"
 msgstr "Délai (en jours)"

wcs/publisher.py

@@ -355,12 +355,14 @@ class WcsPublisher(QommonPublisher):
             for f in z.namelist():
                 if os.path.dirname(f) == 'datasources' and os.path.basename(f):
                     with z.open(f) as fd:
-                        data_source = NamedDataSource.import_from_xml(fd, include_id=True)
+                        data_source = NamedDataSource.import_from_xml(
+                            fd, include_id=True, check_deprecated=True
+                        )
                     data_source.store()
                     results['datasources'] += 1
                 if os.path.dirname(f) == 'wscalls' and os.path.basename(f):
                     with z.open(f) as fd:
-                        wscall = NamedWsCall.import_from_xml(fd, include_id=True)
+                        wscall = NamedWsCall.import_from_xml(fd, include_id=True, check_deprecated=True)
                     wscall.store()
                     results['wscalls'] += 1
 
@@ -370,7 +372,7 @@ class WcsPublisher(QommonPublisher):
             for f in z.namelist():
                 if os.path.dirname(f) == 'blockdefs_xml' and os.path.basename(f):
                     with z.open(f) as fd:
-                        blockdef = BlockDef.import_from_xml(fd, include_id=True)
+                        blockdef = BlockDef.import_from_xml(fd, include_id=True, check_deprecated=True)
                     blockdef.store()
                     results['blockdefs'] += 1
 
@@ -380,7 +382,9 @@ class WcsPublisher(QommonPublisher):
             for f in z.namelist():
                 if os.path.dirname(f) == 'workflows_xml' and os.path.basename(f):
                     with z.open(f) as fd:
-                        workflow = Workflow.import_from_xml(fd, include_id=True, check_datasources=False)
+                        workflow = Workflow.import_from_xml(
+                            fd, include_id=True, check_datasources=False, check_deprecated=True
+                        )
                     workflow.store()
                     results['workflows'] += 1
 
@@ -393,13 +397,17 @@ class WcsPublisher(QommonPublisher):
             for f in z.namelist():
                 if os.path.dirname(f) == 'formdefs_xml' and os.path.basename(f):
                     with z.open(f) as fd:
-                        formdef = FormDef.import_from_xml(fd, include_id=True, check_datasources=False)
+                        formdef = FormDef.import_from_xml(
+                            fd, include_id=True, check_datasources=False, check_deprecated=True
+                        )
                     formdef.store()
                     formdefs.append(formdef)
                     results['formdefs'] += 1
                 if os.path.dirname(f) == 'carddefs_xml' and os.path.basename(f):
                     with z.open(f) as fd:
-                        carddef = CardDef.import_from_xml(fd, include_id=True, check_datasources=False)
+                        carddef = CardDef.import_from_xml(
+                            fd, include_id=True, check_datasources=False, check_deprecated=True
+                        )
                     carddef.store()
                     carddefs.append(carddef)
                     results['carddefs'] += 1
@@ -485,6 +493,7 @@ class WcsPublisher(QommonPublisher):
             for _formdef in FormDef.select() + CardDef.select():
                 sql.do_formdef_tables(_formdef)
             sql.migrate_global_views(conn, cur)
+            sql.init_search_tokens()
             cur.close()
 
     def record_deprecated_usage(self, *args, **kwargs):

wcs/qommon/afterjobs.py

@@ -87,7 +87,7 @@ class AfterJob(StorableObject):
     def increment_count(self, amount=1):
         self.current_count = (self.current_count or 0) + amount
         # delay storage to avoid repeated writes on slow storage
-        if time.time() - self._last_store_time > 1:
+        if time.time() - self._last_store_time > 1 and self.id:
             self.store()
 
     def get_completion_status(self):

wcs/qommon/cron.py

@@ -16,9 +16,11 @@
 
 import datetime
 import os
+import sys
 import time
 from contextlib import contextmanager
 
+import psutil
 from django.conf import settings
 from django.utils.timezone import localtime
 from quixote import get_publisher
@@ -63,8 +65,8 @@ class CronJob:
                     'long job: %s (took %s minutes, %d CPU minutes)' % (self.name, minutes, process_minutes)
                 )
 
-    @staticmethod
-    def log(message, in_tenant=True):
+    @classmethod
+    def log(cls, message, in_tenant=True):
         now = localtime()
         if in_tenant:
             base_dir = get_publisher().tenant.directory
@@ -75,6 +77,12 @@ class CronJob:
         with open(os.path.join(log_dir, 'cron.log-%s' % now.strftime('%Y%m%d')), 'a+') as fd:
             fd.write('%s [%s] %s\n' % (now.isoformat(), os.getpid(), message))
 
+    def log_debug(self, message, in_tenant=True):
+        if get_publisher().get_site_option('cron-log-level') != 'debug':
+            return
+        memory = psutil.Process().memory_info().rss / (1024 * 1024)
+        self.log(f'(mem: {memory:.1f}M) {message}', in_tenant=in_tenant)
+
     def is_time(self, timetuple):
         minutes = self.minutes
         if minutes:
@@ -120,6 +128,7 @@ def cron_worker(publisher, since, job_name=None):
     if jobs:
         CronJob.log('running jobs: %r' % sorted([x.name or x for x in jobs]))
     for job in jobs:
+        publisher.current_cron_job = job
         publisher.install_lang()
         publisher.setup_timezone()
         publisher.reset_formdata_state()
@@ -128,4 +137,5 @@ def cron_worker(publisher, since, job_name=None):
             with job.log_long_job():
                 job.function(publisher, job=job)
         except Exception as e:
-            publisher.record_error(exception=e, context='[CRON]', notify=True)
+            job.log(f'exception running job {job.name}: {e}')
+            publisher.capture_exception(sys.exc_info())

wcs/qommon/form.py

@@ -29,6 +29,7 @@ import mimetypes
 import os
 import random
 import re
+import subprocess
 import sys
 import tempfile
 import time
@@ -924,6 +925,20 @@ class FileWithPreviewWidget(CompositeWidget):
         return False
 
     def set_value(self, value):
+        if isinstance(value, (str, dict)):
+            from wcs.fields.file import FileField
+
+            try:
+                value = FileField.convert_value_from_anything(value)
+            except ValueError as e:
+                value = None
+                if getattr(self, 'field', None):
+                    get_publisher().record_error(
+                        _('Failed to convert value for field "%s"') % self.field.label,
+                        formdef=getattr(self, 'formdef', None),
+                        exception=e,
+                    )
+
         try:
             self.value = value
             if self.value and self.get_value_from_token:
@@ -1047,12 +1062,6 @@ class FileWithPreviewWidget(CompositeWidget):
 
         self.value.content_type = filetype
 
-        if self.max_file_size and hasattr(self.value, 'file_size'):
-            # validate file size
-            if self.value.file_size > self.max_file_size_bytes:
-                self.set_error(_('over file size limit (%s)') % self.max_file_size)
-                return
-
         if self.file_type:
             # validate file type
             accepted_file_types = []
@@ -1098,6 +1107,40 @@ class FileWithPreviewWidget(CompositeWidget):
         ) or filetype in blacklisted_file_types:
             self.set_error(_('forbidden file type'))
 
+        if self.value.content_type in ('image/heic', 'image/heif') and not get_publisher().has_site_option(
+            'do-no-transform-heic-files'
+        ):
+            # convert HEIC files to JPEG
+            try:
+                with open(self.value.fp.name, 'rb') as fd:
+                    # libheic will automatically switch image orientation so we need to remove
+                    # EXIF profile to avoid it being applied a second time.
+                    # (graphicsmagick >= 1.3.41 have heif:ignore-transformations=false to avoid
+                    # that).
+                    rc = subprocess.run(
+                        ['gm', 'convert', '+profile', '"*"', 'HEIC:-', 'JPEG:-'],
+                        input=fd.read(),
+                        capture_output=True,
+                        check=True,
+                    )
+                    from wcs.fields.file import FileField
+
+                    self.value = FileField.convert_value_from_anything(
+                        {
+                            'content': rc.stdout,
+                            'filename': os.path.splitext(self.value.base_filename)[0] + '.jpeg',
+                            'content_type': 'image/jpeg',
+                        }
+                    )
+            except subprocess.CalledProcessError:
+                pass
+
+        if self.max_file_size and hasattr(self.value, 'file_size'):
+            # validate file size
+            if self.value.file_size > self.max_file_size_bytes:
+                self.set_error(_('over file size limit (%s)') % self.max_file_size)
+                return
+
 
 class EmailWidget(StringWidget):
     HTML_TYPE = 'email'

wcs/qommon/http_request.py

@@ -28,6 +28,8 @@ from quixote.errors import RequestError
 
 from .http_response import HTTPResponse
 
+user_agent_regex = re.compile(r'(?P<product>.*?)(?P<comment>\(.*?\))(?P<rest>.*)')
+
 
 class HTTPRequest(quixote.http_request.HTTPRequest):
     signed = False
@@ -222,6 +224,18 @@ class HTTPRequest(quixote.http_request.HTTPRequest):
             or user_agent.startswith('Wget')
         )
 
+    def is_from_mobile(self):
+        user_agent = self.get_environ('HTTP_USER_AGENT', '')
+        try:
+            dummy, comment, rest = user_agent_regex.match(user_agent).groups()
+        except AttributeError:
+            return False
+        # https://developer.mozilla.org/en-US/docs/Web/HTTP/Browser_detection_using_the_user_agent#mobile_tablet_or_desktop
+        # Mozilla (Gecko, Firefox) / Mobile or Tablet inside the comment
+        # WebKit-based (Android, Safari) / Mobile Safari token outside the comment
+        # Blink-based (Chromium, etc.) / Mobile Safari token outside the comment
+        return bool('Mobile' in comment or 'Tablet' in comment or 'Mobile Safari' in rest)
+
     def has_anonymised_data_api_restriction(self):
         from wcs.api_access import ApiAccess
 

wcs/qommon/humantime.py

@@ -34,21 +34,20 @@ def list2human(stringlist):
 
 
 _humandurations = (
-    ((_('day'), _('days')), _day),
-    ((_('hour'), _('hours')), _hour),
-    ((_('month'), _('months')), _month),
-    ((_('year'), _('years')), _year),
-    ((_('minute'), _('minutes')), _minute),
-    ((_('second'), _('seconds')), 1),
+    ((_('day'), _('days'), _('day(s)')), _day),
+    ((_('hour'), _('hours'), _('hour(s)')), _hour),
+    ((_('minute'), _('minutes'), _('minute(s)')), _minute),
+    ((_('second'), _('seconds'), _('second(s)')), 1),
+    ((_('month'), _('months'), _('month(s)')), _month),
+    ((_('year'), _('years'), _('year(s)')), _year),
 )
 
 
 def timewords():
     '''List of words one can use to specify durations'''
     result = []
-    for words, dummy in _humandurations:
-        for word in words:
-            result.append(str(word))  # str() to force translation
+    for (dummy, dummy, word), dummy in _humandurations:
+        result.append(str(word))  # str() to force translation
     return result
 
 
@@ -56,12 +55,11 @@ def humanduration2seconds(humanduration):
     if not humanduration:
         raise ValueError()
     seconds = 0
-    for words, quantity in _humandurations:
-        for word in words:
-            m = re.search(r'(\d+)\s*\b%s\b' % word, humanduration)
-            if m:
-                seconds = seconds + int(m.group(1)) * quantity
-                break
+    for (word1, word2, dummy), quantity in _humandurations:
+        # look for number then singular or plural forms of unit
+        m = re.search(r'(\d+)\s*\b(%s|%s)\b' % (word1, word2), humanduration)
+        if m:
+            seconds = seconds + int(m.group(1)) * quantity
     return seconds
 
 

wcs/qommon/publisher.py

@@ -58,6 +58,25 @@ except ImportError:
     sentry_sdk = None
 
 
+class MaxSizeDict(collections.OrderedDict):
+    # dictionary that will store at most 128 items, least recently used items are removed first.
+
+    def __setitem__(self, key, value):
+        super().__setitem__(key, value)
+        self.move_to_end(key, last=False)
+        if len(self) > 128:
+            self.popitem(last=True)
+
+    def __getitem__(self, key):
+        if key in self:
+            self.move_to_end(key, last=False)
+        return super().__getitem__(key)
+
+    def get(self, key, default=None):
+        # native get() doesn't use __getitem__
+        return self[key] if key in self else default
+
+
 class ImmediateRedirectException(Exception):
     def __init__(self, location):
         self.location = location
@@ -421,7 +440,7 @@ class QommonPublisher(Publisher):
         return string
 
     def load_site_options(self):
-        self.site_options = configparser.ConfigParser()
+        self.site_options = configparser.ConfigParser(interpolation=None)
         site_options_filename = os.path.join(self.app_dir, 'site-options.cfg')
         if not os.path.exists(site_options_filename):
             return
@@ -504,7 +523,7 @@ class QommonPublisher(Publisher):
 
     def reset_caches(self):
         self._cached_user_fields_formdef = None
-        self._cached_objects = collections.defaultdict(dict)
+        self._cached_objects = collections.defaultdict(MaxSizeDict)
 
     def set_app_dir(self, request):
         """
@@ -692,6 +711,11 @@ class QommonPublisher(Publisher):
         for error in self.loggederror_class.select(clause=clauses):
             self.loggederror_class.remove_object(error.id)
 
+    def clean_search_tokens(self, **kwargs):
+        from wcs import sql
+
+        sql.purge_obsolete_search_tokens()
+
     @classmethod
     def register_cronjobs(cls):
         cls.register_cronjob(CronJob(cls.clean_sessions, minutes=[0], name='clean_sessions'))
@@ -704,6 +728,9 @@ class QommonPublisher(Publisher):
         cls.register_cronjob(
             CronJob(cls.clean_loggederrors, hours=[3], minutes=[0], name='clean_loggederrors')
         )
+        cls.register_cronjob(
+            CronJob(cls.clean_search_tokens, weekdays=[0], hours=[1], minutes=[0], name='clean_search_tokens')
+        )
 
     _initialized = False
 
@@ -814,9 +841,7 @@ class QommonPublisher(Publisher):
                 'map-bounds-bottom-right'
             ).split(';')
         attrs['data-map-attribution'] = self.get_site_option('map-attribution') or _(
-            'Map data © '
-            "<a href='https://openstreetmap.org'>OpenStreetMap</a> contributors, "
-            "<a href='http://creativecommons.org/licenses/by-sa/2.0/'>CC-BY-SA</a>"
+            'Map data &copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a>'
         )
         attrs['data-tile-urltemplate'] = (
             self.get_site_option('map-tile-urltemplate') or 'https://tiles.entrouvert.org/hdm/{z}/{x}/{y}.png'

wcs/qommon/static/css/dc2/admin.scss

@@ -290,9 +290,23 @@ span.error-message {
 	font-weight: bold;
 }
 
+#inspect-drafts {
+	h2 {
+		margin-top: 0;
+	}
+	h3 {
+		margin-top: 2em;
+		margin-bottom: 0;
+		font-weight: normal;
+	}
+}
+
 table.stats {
 	margin: 1ex 0;
 	width: 100%;
+	&.completion-rate {
+		margin-top: 0;
+	}
 }
 
 table.stats thead th {
@@ -301,17 +315,27 @@ table.stats thead th {
 
 table.stats td {
 	padding-left: 1em;
+	&.percent,
+	&.total {
+		white-space: pre;
+	}
 }
 
 table.stats td.label {
+	padding-top: 1ex;
+	padding-left: 0;
 	width: 100%;
 }
 
+table.stats td.bar {
+	background: #eee;
+	padding-left: 0;
+}
+
 table.stats td.bar span {
 	background: #4BB2C5;
 	height: 1ex;
 	display: block;
-	margin-bottom: 1ex;
 	box-shadow: 2px 2px 2px #aaa;
 }
 
@@ -1162,6 +1186,7 @@ div.PrefillSelectionWidget div.content input[type=submit] {
 ul#field-filter,
 ul.columns-filter {
 	list-style: none;
+	padding-bottom: 1px;
 	padding-left: 0;
 	margin-left: 0;
 	max-height: calc(100vh - 14em);
@@ -2832,6 +2857,9 @@ div.file-upload-widget {
 		}
 		div.widget-message {
 			padding-top: 20px;
+			p {
+				margin: 0;
+			}
 			&::before {
 				pointer-events: none;
 				content: "\f016";  // file-o

wcs/qommon/static/js/qommon.forms.js

@@ -851,6 +851,8 @@ $(function() {
         data: form_data,
         headers: {'x-wcs-ajax-action': 'block-add-row'},
         success: function(result, text_status, jqXHR) {
+          var new_form_token = $(result).find('input[name="_form_id"]').val()
+          $('input[name="_form_id"]').val(new_form_token)
           const $new_block = $(result).find('[data-field-id="' + block_id + '"]');
           $block.replaceWith($new_block);
           const $new_blockrow = $new_block.find('.BlockSubWidget').last();

wcs/qommon/static/js/qommon.map.js

@@ -26,6 +26,8 @@ $(window).on('wcs:maps-init', function() {
      }
      map_options.gestureHandling = true;
      var map = L.map($(this).attr('id'), map_options);
+     map.attributionControl.setPrefix(
+       '<a href="https://leafletjs.com" title="' + WCS_I18N.map_leaflet_title_attribute + '">Leaflet</a>')
      var map_controls_position = $('body').data('map-controls-position') || 'topleft';
      if (! ($map_widget.parents('#sidebar').length || $map_widget.parents('td').length)) {
        new L.Control.Zoom({
@@ -204,7 +206,7 @@ $(window).on('wcs:maps-init', function() {
 
      $map_widget.on('set-geolocation', function(e, coords, options) {
        if (map.marker === null) {
-         map.marker = L.marker([0, 0]);
+         map.marker = L.marker([0, 0], {alt: WCS_I18N.map_position_marker_alt});
          map.marker.addTo(map);
        }
        map.marker.setLatLng(coords);

wcs/qommon/storage.py

@@ -528,7 +528,7 @@ class StorableObject:
         return cls.sort_results(objects, order_by)
 
     @classmethod
-    def get_on_index(cls, id, index, ignore_errors=False, ignore_migration=False):
+    def get_on_index(cls, id, index, ignore_errors=False, ignore_migration=False, use_cache=False):
         if not cls._indexes:
             raise KeyError()
         objects_dir = cls.get_objects_dir()
@@ -536,6 +536,14 @@ class StorableObject:
         if not os.path.exists(index_dir):
             cls.rebuild_indexes()
         filename = os.path.join(index_dir, str(fix_key(id)))
+        if use_cache:
+            try:
+                object_id = os.readlink(filename).split('/')[-1]
+            except FileNotFoundError:
+                if ignore_errors:
+                    return None
+                raise KeyError(id)
+            return cls.cached_get(object_id, ignore_errors=ignore_errors, ignore_migration=ignore_migration)
         return cls.get_filename(filename, ignore_errors=ignore_errors, ignore_migration=ignore_migration)
 
     @classmethod

wcs/qommon/templates/qommon/forms/widgets/block.html

@@ -1,3 +1,5 @@
 {% extends "qommon/forms/widget.html" %}
 
 {% block widget-css-classes %}{{ block.super }} {% if widget.had_add_clicked %}wcs-block-add-clicked{% endif %} {% if widget.remove_button %}wcs-block-with-remove-button{% endif %}{% endblock %}
+
+{% block widget-attrs %}id="form_{{ widget.field.id }}" {{ block.super }}{% endblock %}

wcs/qommon/templates/qommon/forms/widgets/file.html

@@ -9,10 +9,10 @@
       {{ w.render|safe }}
     {% endfor %}
     <div class="widget-message click-to-upload">
-      {% trans "Drop a file or click to select one" %}
+      <p>{% trans "Drop a file or click to select one" %}</p>
     </div>
     <div class="widget-message upload-done">
-      {% trans "Upload done" %}
+      <p>{% trans "Upload done" %}</p>
     </div>
     <div class="fileprogress" style="display: none;">
       <div class="bar"

wcs/qommon/templates/qommon/forms/widgets/map.html

@@ -2,7 +2,7 @@
 
 {% block widget-control %}
   <input type="hidden" name="{{widget.name}}$latlng" {% if widget.value %}value="{{widget.value}}"{% endif %}>
-  <div id="map-{{widget.get_name_for_id}}" class="qommon-map"
+  <div id="form_{{widget.get_name_for_id}}" class="qommon-map"
        {% if widget.readonly %}data-readonly="true"{% endif %}
        {% if widget.sync_map_and_address_fields %}data-address-sync="true"{% endif %}
        {% for key, value in widget.map_attributes.items %}{{key}}="{{value}}" {% endfor %}

wcs/qommon/templatetags/qommon.py

@@ -23,6 +23,7 @@ import json
 import math
 import os
 import random
+import re
 import string
 import subprocess
 import urllib.parse
@@ -53,7 +54,7 @@ from django.template import defaultfilters
 from django.utils import dateparse
 from django.utils.encoding import force_bytes, force_str
 from django.utils.safestring import mark_safe
-from django.utils.timezone import is_naive, make_aware
+from django.utils.timezone import is_naive, localtime, make_aware, make_naive
 
 from wcs.qommon import _, calendar, evalutils, upload_storage
 from wcs.qommon.admin.texts import TextsDirectory
@@ -355,7 +356,7 @@ def age_in_hours(value, now=None):
         if not now:
             return ''
     else:
-        now = datetime.datetime.now()
+        now = make_naive(localtime())
     return int((now - value).total_seconds() / 3600)
 
 
@@ -750,7 +751,9 @@ def decorate_queryset_filter(func, name, attr):
     @functools.wraps(func)
     def f(queryset, *args, **kwargs):
         if not hasattr(queryset, attr):
-            get_publisher().record_error(_('|%s used on invalid queryset (%r)') % (name, queryset))
+            get_publisher().record_error(
+                _('|%s used on something else than a queryset (%r)') % (name, queryset)
+            )
             return None
         return func(queryset, *args, **kwargs)
 
@@ -937,7 +940,11 @@ def count(queryset):
     queryset = unlazy(queryset)
     if queryset is None:
         return 0
-    return len(queryset)
+    try:
+        return len(queryset)
+    except TypeError:
+        get_publisher().record_error(_('|count used on uncountable value'))
+        return 0
 
 
 @register.filter
@@ -1377,3 +1384,25 @@ def details_format(value, format=None):
         get_publisher().record_error(_('|details_format called with unknown format (%s)') % format)
         return ''
     return evalutils.details_format(value, format=format)
+
+
+@register.filter
+def housenumber_number(housenumber):
+    housenumber = unlazy(housenumber)
+    if not housenumber:
+        return ''
+    match = re.match(r'^\s*([0-9]+)(.*)$', force_str(housenumber))
+    if not match:
+        return ''
+    return match.groups()[0]
+
+
+@register.filter
+def housenumber_btq(housenumber):
+    housenumber = unlazy(housenumber)
+    if not housenumber:
+        return ''
+    match = re.match(r'^\s*([0-9]+)(.*)$', force_str(housenumber))
+    if not match:
+        return ''
+    return match.groups()[1]

wcs/qommon/xml_storage.py

@@ -84,7 +84,7 @@ class XmlStorableObject(StorableObject):
             sub.text = role.name
 
     @classmethod
-    def import_from_xml(cls, fd, include_id=False, check_deprecated=True):
+    def import_from_xml(cls, fd, include_id=False, check_deprecated=False):
         try:
             tree = ET.parse(fd)
         except Exception:
@@ -92,7 +92,7 @@ class XmlStorableObject(StorableObject):
         return cls.import_from_xml_tree(tree, include_id=include_id, check_deprecated=check_deprecated)
 
     @classmethod
-    def import_from_xml_tree(cls, tree, include_id=False, check_deprecated=True, **kwargs):
+    def import_from_xml_tree(cls, tree, include_id=False, check_deprecated=False, **kwargs):
         obj = cls()
 
         # if the tree we get is actually a ElementTree for real, we get its

wcs/snapshots.py

@@ -223,7 +223,7 @@ class Snapshot:
             # else: keep serialization and ignore patch
             obj.store()
 
-            if get_response():
+            if get_response() and obj.object_type in ('formdef', 'carddef'):
                 from wcs.admin.tests import TestsAfterJob
 
                 get_response().add_after_job(
@@ -277,6 +277,7 @@ class Snapshot:
                 include_id=True,
                 snapshot=True,
                 check_datasources=getattr(self, '_check_datasources', True),
+                check_deprecated=False,
             )
             self._instance.readonly = True
             self._instance.snapshot_object = self

wcs/sql.py

@@ -96,6 +96,20 @@ SQL_TYPE_MAPPING = {
 }
 
 
+def _table_exists(cur, table_name):
+    cur.execute('SELECT 1 FROM pg_class WHERE relname = %s', (table_name,))
+    rows = cur.fetchall()
+    return len(rows) > 0
+
+
+def _trigger_exists(cur, table_name, trigger_name):
+    cur.execute(
+        'SELECT 1 FROM pg_trigger WHERE tgrelid = %s::regclass AND tgname = %s', (table_name, trigger_name)
+    )
+    rows = cur.fetchall()
+    return len(rows) > 0
+
+
 class WcsPgConnection(psycopg2.extensions.connection):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -544,6 +558,7 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
             cur.execute(f'ALTER TABLE {table_name} ALTER COLUMN last_update_time SET DATA TYPE timestamptz')
 
         # add new fields
+        field_integrity_errors = {}
         for field in formdef.get_all_fields():
             assert field.id is not None
             sql_type = SQL_TYPE_MAPPING.get(field.key, 'varchar')
@@ -554,6 +569,16 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
                 cur.execute(
                     '''ALTER TABLE %s ADD COLUMN %s %s''' % (table_name, get_field_id(field), sql_type)
                 )
+            else:
+                existing_type = existing_field_types.get(get_field_id(field))
+                # map to names returned in data_type column
+                expected_type = {
+                    'varchar': 'character varying',
+                    'text[]': 'ARRAY',
+                    'text[][]': 'ARRAY',
+                }.get(sql_type) or sql_type
+                if existing_type != expected_type:
+                    field_integrity_errors[str(field.id)] = {'got': existing_type, 'expected': expected_type}
             if field.store_display_value:
                 needed_fields.add('%s_display' % get_field_id(field))
                 if '%s_display' % get_field_id(field) not in existing_fields:
@@ -569,6 +594,10 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
                         % (table_name, '%s_structured' % get_field_id(field))
                     )
 
+        if (field_integrity_errors or None) != formdef.sql_integrity_errors:
+            formdef.sql_integrity_errors = field_integrity_errors
+            formdef.store(object_only=True)
+
         for field in (formdef.geolocations or {}).keys():
             column_name = 'geoloc_%s' % field
             needed_fields.add(column_name)
@@ -807,7 +836,8 @@ def do_user_table():
                                     lasso_dump text,
                                     last_seen timestamp,
                                     deleted_timestamp timestamp,
-                                    preferences jsonb
+                                    preferences jsonb,
+                                    test_uuid varchar
                                     )'''
             % table_name
         )
@@ -834,6 +864,7 @@ def do_user_table():
         'deleted_timestamp',
         'is_active',
         'preferences',
+        'test_uuid',
     }
 
     from wcs.admin.settings import UserFieldsFormDef
@@ -883,6 +914,9 @@ def do_user_table():
     if 'preferences' not in existing_fields:
         cur.execute('ALTER TABLE %s ADD COLUMN preferences jsonb' % table_name)
 
+    if 'test_uuid' not in existing_fields:
+        cur.execute('ALTER TABLE %s ADD COLUMN test_uuid varchar' % table_name)
+
     # delete obsolete fields
     for field in existing_fields - needed_fields:
         cur.execute('''ALTER TABLE %s DROP COLUMN %s''' % (table_name, field))
@@ -1508,6 +1542,15 @@ def drop_global_views(conn, cur):
         cur.execute('''DROP VIEW IF EXISTS %s''' % view_name)
 
 
+def update_global_view_formdef_category(formdef):
+    _, cur = get_connection_and_cursor()
+    with cur:
+        cur.execute(
+            '''UPDATE wcs_all_forms set category_id = %s WHERE formdef_id = %s''',
+            (formdef.category_id, formdef.id),
+        )
+
+
 def do_global_views(conn, cur):
     # recreate global views
     # XXX TODO: make me dynamic, please ?
@@ -1582,6 +1625,8 @@ def do_global_views(conn, cur):
                 % (name, category.id)
             )
 
+    init_search_tokens_triggers(cur)
+
 
 def clean_global_views(conn, cur):
     # Purge of any dead data
@@ -1674,11 +1719,182 @@ def init_global_table(conn=None, cur=None):
                 endpoint_status=endpoint_status_filter,
             )
         )
+        init_search_tokens_data(cur)
 
     if own_conn:
         cur.close()
 
 
+def init_search_tokens(conn=None, cur=None):
+    """Initialize the search_tokens mechanism.
+
+    It's based on three parts:
+    - a token table
+    - triggers to feed this table from the tsvectors used in the database
+    - a search function that will leverage these tokens to extend the search query.
+
+    So far, the sources used are wcs_all_forms and searchable_formdefs.
+
+    Example: let's say the sources texts are "Tarif d'école" and "La cantine".
+    This gives the following tsvectors: ('tarif', 'écol') and ('cantin')
+    Our tokens table will have these three words.
+    When the search function is launched, it splits the search query and will
+    replace unavailable tokens by those close, if available.
+    The search query 'tari' will be expanded to 'tarif'.
+    The search query 'collège' will remain unchanged (and return nothing)
+    If several tokens match or are close enough, the query will be expanded to
+    an OR.
+    """
+
+    own_cur = False
+    if cur is None:
+        own_cur = True
+        conn, cur = get_connection_and_cursor()
+
+    # Create table
+    cur.execute('CREATE TABLE IF NOT EXISTS wcs_search_tokens(token TEXT PRIMARY KEY);')
+
+    # Create triggers
+    init_search_tokens_triggers(cur)
+
+    # Fill table
+    init_search_tokens_data(cur)
+
+    # Index at the end, small performance trick... not that useful, but it's free...
+    cur.execute('CREATE EXTENSION IF NOT EXISTS pg_trgm;')
+    cur.execute(
+        'CREATE INDEX IF NOT EXISTS wcs_search_tokens_trgm ON wcs_search_tokens USING gin(token gin_trgm_ops);'
+    )
+
+    # And last: functions to use this brand new table
+    # These two aggregates make the search query far simpler to write, allowing writing an OR/AND of search terms
+    # directly as an SQL aggregation.
+    # They use the tsquery_or and tsquery_and functions that are included in PostgreSQL since 8.3, but documented
+    # under their operator names || and &&.
+    cur.execute('CREATE OR REPLACE AGGREGATE tsquery_agg_or  (tsquery) (sfunc=tsquery_or,  stype=tsquery);')
+    cur.execute('CREATE OR REPLACE AGGREGATE tsquery_agg_and (tsquery) (sfunc=tsquery_and, stype=tsquery);')
+    cur.execute(
+        r"""CREATE OR REPLACE FUNCTION public.wcs_tsquery(text)
+ RETURNS tsquery
+ LANGUAGE sql
+ STABLE
+AS $function$
+WITH
+        tokenized AS (SELECT unnest(regexp_split_to_array($1, '\s+')) word),
+        super_tokenized AS (
+            -- perfect: tokens that are found as is in table, thus no OR required
+            -- partial: tokens found using distance search on tokens table (note: numbers are excluded here)
+            --          distance search is done using pg_trgm, https://www.postgresql.org/docs/current/pgtrgm.html
+            -- otherwise: token as is and likely no search result later
+            SELECT word,
+                coalesce((select plainto_tsquery(perfect.token) FROM wcs_search_tokens AS perfect WHERE perfect.token = plainto_tsquery(word)::text),
+                         tsquery_agg_or(plainto_tsquery(partial.token)),
+                         plainto_tsquery(word)) AS tokens
+            FROM tokenized
+            LEFT JOIN wcs_search_tokens AS partial ON partial.token % plainto_tsquery(word)::text AND word not similar to '%[0-9]{2,}%'
+            GROUP BY word)
+SELECT tsquery_agg_and(tokens) FROM super_tokenized;
+$function$;"""
+    )
+
+    if own_cur:
+        cur.close()
+
+
+def init_search_tokens_triggers(cur):
+    # We define only appending triggers, ie on INSERT and UPDATE.
+    # It would be far heavier to maintain deletions here, and having extra data has
+    # no or marginal side effect on search performances, and absolutely no impact
+    # on search results.
+    # Instead, a weekly cron job will delete obsolete entries, thus making it sure no
+    # personal data is kept uselessly.
+
+    # First part: the appending function
+    cur.execute(
+        """CREATE OR REPLACE FUNCTION wcs_search_tokens_trigger_fn ()
+ RETURNS trigger
+ LANGUAGE plpgsql
+AS $function$
+BEGIN
+        INSERT INTO wcs_search_tokens SELECT unnest(tsvector_to_array(NEW.fts)) ON CONFLICT(token) DO NOTHING;
+        RETURN NEW;
+END;
+$function$;"""
+    )
+
+    if not (_table_exists(cur, 'wcs_search_tokens')):
+        # abort trigger creation if tokens table doesn't exist yet
+        return
+
+    if _table_exists(cur, 'wcs_all_forms') and not _trigger_exists(
+        cur, 'wcs_all_forms', 'wcs_all_forms_fts_trg_upd'
+    ):
+        # Second part: insert and update triggers for wcs_all_forms
+        cur.execute(
+            """CREATE TRIGGER wcs_all_forms_fts_trg_ins
+            AFTER INSERT ON wcs_all_forms
+            FOR EACH ROW WHEN (NEW.fts IS NOT NULL)
+            EXECUTE PROCEDURE wcs_search_tokens_trigger_fn();"""
+        )
+        cur.execute(
+            """CREATE TRIGGER wcs_all_forms_fts_trg_upd
+            AFTER UPDATE OF fts ON wcs_all_forms
+            FOR EACH ROW WHEN (NEW.fts IS NOT NULL)
+            EXECUTE PROCEDURE wcs_search_tokens_trigger_fn();"""
+        )
+
+    if _table_exists(cur, 'searchable_formdefs') and not _trigger_exists(
+        cur, 'searchable_formdefs', 'searchable_formdefs_fts_trg_upd'
+    ):
+        # Third part: insert and update triggers for searchable_formdefs
+        cur.execute(
+            """CREATE TRIGGER searchable_formdefs_fts_trg_ins
+            AFTER INSERT ON searchable_formdefs
+            FOR EACH ROW WHEN (NEW.fts IS NOT NULL)
+            EXECUTE PROCEDURE wcs_search_tokens_trigger_fn();"""
+        )
+        cur.execute(
+            """CREATE TRIGGER searchable_formdefs_fts_trg_upd
+            AFTER UPDATE OF fts ON searchable_formdefs
+            FOR EACH ROW WHEN (NEW.fts IS NOT NULL)
+            EXECUTE PROCEDURE wcs_search_tokens_trigger_fn();"""
+        )
+
+
+def init_search_tokens_data(cur):
+    if not (_table_exists(cur, 'wcs_search_tokens')):
+        # abort table data initialization if tokens table doesn't exist yet
+        return
+
+    if _table_exists(cur, 'wcs_all_forms'):
+        cur.execute(
+            """INSERT INTO wcs_search_tokens
+        SELECT unnest(tsvector_to_array(fts)) FROM wcs_all_forms
+        ON CONFLICT(token) DO NOTHING;"""
+        )
+    if _table_exists(cur, 'searchable_formdefs'):
+        cur.execute(
+            """INSERT INTO wcs_search_tokens
+        SELECT unnest(tsvector_to_array(fts)) FROM searchable_formdefs
+        ON CONFLICT(token) DO NOTHING;"""
+        )
+
+
+def purge_obsolete_search_tokens(cur=None):
+    own_cur = False
+    if cur is None:
+        own_cur = True
+        _, cur = get_connection_and_cursor()
+
+    cur.execute(
+        """DELETE FROM wcs_search_tokens
+    WHERE token NOT IN (SELECT unnest(tsvector_to_array(fts)) FROM wcs_all_forms)
+    AND token NOT IN (SELECT unnest(tsvector_to_array(fts)) FROM wcs_all_forms);"""
+    )
+    if own_cur:
+        cur.close()
+
+
 class SqlMixin:
     _table_name = None
     _numerical_id = True
@@ -2406,8 +2622,9 @@ class SqlDataMixin(SqlMixin):
         return [row[1].total_seconds() for row in results if row[1].total_seconds() >= 0]
 
     def _set_auto_fields(self, cur):
-        if self.set_auto_fields():
-            self._has_changed_digest = True
+        changed_auto_fields = self.set_auto_fields()
+        if changed_auto_fields:
+            self._has_changed_digest = bool('digests' in changed_auto_fields)
             sql_statement = (
                 '''UPDATE %s
                                   SET id_display = %%(id_display)s,
@@ -2858,8 +3075,9 @@ class SqlCardData(SqlDataMixin, wcs.carddata.CardData):
     def store(self, *args, **kwargs):
         if self.uuid is None:
             self.uuid = str(uuid.uuid4())
+        is_new_card = bool(not self.id)
         super().store(*args, **kwargs)
-        if self._has_changed_digest:
+        if self._has_changed_digest and not is_new_card:
             self.update_related()
 
 
@@ -2879,6 +3097,7 @@ class SqlUser(SqlMixin, wcs.users.User):
         ('deleted_timestamp', 'timestamp'),
         ('is_active', 'bool'),
         ('preferences', 'jsonb'),
+        ('test_uuid', 'varchar'),
     ]
     _sql_indexes = [
         'users_name_idx ON users (name)',
@@ -2895,6 +3114,21 @@ class SqlUser(SqlMixin, wcs.users.User):
         self.verified_fields = []
         self.roles = []
 
+    @classmethod
+    def select(cls, clause=None, **kwargs):
+        has_explicit_test_user_filter = bool(
+            isinstance(clause, list)
+            and any(x.attribute == 'test_uuid' for x in clause if hasattr(x, 'attribute'))
+        )
+        if not has_explicit_test_user_filter:
+            clause = clause or []
+            if callable(clause):
+                clause = [clause]
+
+            clause.append(Null('test_uuid'))
+
+        return super().select(clause=clause, **kwargs)
+
     @invalidate_substitution_cache
     def store(self):
         sql_dict = {
@@ -2910,6 +3144,7 @@ class SqlUser(SqlMixin, wcs.users.User):
             'deleted_timestamp': self.deleted_timestamp,
             'is_active': self.is_active,
             'preferences': self.preferences,
+            'test_uuid': self.test_uuid,
         }
         if self.last_seen:
             sql_dict['last_seen'] = (datetime.datetime.fromtimestamp(self.last_seen),)
@@ -3006,6 +3241,7 @@ class SqlUser(SqlMixin, wcs.users.User):
             o.deleted_timestamp,
             o.is_active,
             o.preferences,
+            o.test_uuid,
         ) = row[: len(cls._table_static_fields)]
         if o.last_seen:
             o.last_seen = time.mktime(o.last_seen.timetuple())
@@ -3995,6 +4231,7 @@ class TestDef(SqlMixin):
         ('data', 'jsonb'),
         ('is_in_backoffice', 'boolean'),
         ('expected_error', 'varchar'),
+        ('user_uuid', 'varchar'),
         ('agent_id', 'varchar'),
     ]
 
@@ -4020,6 +4257,7 @@ class TestDef(SqlMixin):
                                             data jsonb,
                                             is_in_backoffice boolean NOT NULL DEFAULT FALSE,
                                             expected_error varchar,
+                                            user_uuid varchar,
                                             agent_id varchar
                                             )'''
                 % table_name
@@ -4043,6 +4281,9 @@ class TestDef(SqlMixin):
         if 'agent_id' not in existing_fields:
             cur.execute('''ALTER TABLE %s ADD COLUMN agent_id varchar''' % table_name)
 
+        if 'user_uuid' not in existing_fields:
+            cur.execute('''ALTER TABLE %s ADD COLUMN user_uuid varchar''' % table_name)
+
         # delete obsolete fields
         needed_fields = {x[0] for x in TestDef._table_static_fields}
         for field in existing_fields - needed_fields:
@@ -4093,6 +4334,22 @@ class TestDef(SqlMixin):
                 testdef.expected_error = testdef.data['expected_error']
                 del testdef.data['expected_error']
                 testdef.store()
+            if testdef.data.get('user'):
+                cls.create_and_link_test_users(testdef)
+
+    @staticmethod
+    def create_and_link_test_users(testdef):
+        from wcs.testdef import TestDef
+
+        try:
+            user = get_publisher().user_class.get(testdef.data['user']['id'])
+        except KeyError:
+            return
+
+        user, _ = TestDef.get_or_create_test_user(user)
+        testdef.user_uuid = user.test_uuid
+        del testdef.data['user']
+        testdef.store()
 
 
 class TestResult(SqlMixin):
@@ -4811,7 +5068,6 @@ class SearchableFormDef(SqlMixin):
                 % (cls._table_name, cls._table_name)
             )
         cls.do_indexes(cur)
-        cur.close()
 
         from wcs.carddef import CardDef
         from wcs.formdef import FormDef
@@ -4820,6 +5076,8 @@ class SearchableFormDef(SqlMixin):
             CardDef.select(ignore_errors=True), FormDef.select(ignore_errors=True)
         ):
             cls.update(obj=objectdef)
+        init_search_tokens(cur)
+        cur.close()
 
     @classmethod
     def update(cls, obj=None, removed_obj_type=None, removed_obj_id=None):
@@ -4857,7 +5115,7 @@ class SearchableFormDef(SqlMixin):
     def search(cls, obj_type, string):
         _, cur = get_connection_and_cursor()
         cur.execute(
-            'SELECT object_id FROM searchable_formdefs WHERE fts @@ plainto_tsquery(%s)',
+            'SELECT object_id FROM searchable_formdefs WHERE fts @@ wcs_tsquery(%s)',
             (FtsMatch.get_fts_value(string),),
         )
         ids = [x[0] for x in cur.fetchall()]
@@ -5122,7 +5380,7 @@ def get_period_total(
 # latest migration, number + description (description is not used
 # programmaticaly but will make sure git conflicts if two migrations are
 # separately added with the same number)
-SQL_LEVEL = (106, 'add context column to logged_errors table')
+SQL_LEVEL = (108, 'new fts mechanism with tokens table')
 
 
 def migrate_global_views(conn, cur):
@@ -5256,7 +5514,7 @@ def migrate():
         # 53: add kind column to logged_errors table
         # 106: add context column to logged_errors table
         do_loggederrors_table()
-    if sql_level < 94:
+    if sql_level < 107:
         # 3: introduction of _structured for user fields
         # 4: removal of identification_token
         # 12: (first part) add fts to users
@@ -5267,6 +5525,7 @@ def migrate():
         # 65: index users(name_identifiers)
         # 85: remove anonymous column
         # 94: add preferences column to users table
+        # 107: add test_uuid column to users table
         do_user_table()
     if sql_level < 32:
         # 25: create session_table
@@ -5292,18 +5551,20 @@ def migrate():
         # 79: add translatable column to TranslatableMessage table
         # 100: always create translation messages table
         TranslatableMessage.do_table()
-    if sql_level < 104:
+    if sql_level < 107:
         # 72: add testdef table
         # 87: add testdef is_in_backoffice column
         # 88: add testdef expected_error column
         # 103: drop testdef slug column
         # 104: add testdef agent_id column
+        # 107: add test_uuid column to users table
         TestDef.do_table()
     if sql_level < 95:
         # 95: add a searchable_formdefs table
         SearchableFormDef.do_table()
-    if sql_level < 87:
+    if sql_level < 107:
         # 88: add testdef expected_error column
+        # 107: add test_uuid column to users table
         set_reindex('testdef', 'needed', conn=conn, cur=cur)
     if sql_level < 76:
         # 75: migrate to dedicated workflow traces table
@@ -5456,6 +5717,10 @@ def migrate():
         for formdef in FormDef.select() + CardDef.select():
             do_formdef_tables(formdef, rebuild_views=False, rebuild_global_views=False)
 
+    if sql_level < 108:
+        # 108: new fts mechanism with tokens table
+        init_search_tokens()
+
     if sql_level != SQL_LEVEL[0]:
         cur.execute(
             '''UPDATE wcs_meta SET value = %s, updated_at=NOW() WHERE key = %s''',

wcs/sql_criterias.py

@@ -379,6 +379,11 @@ class FtsMatch(Criteria):
         return 'fts @@ plainto_tsquery(%%(c%s)s)' % id(self.value)
 
 
+class WcsFtsMatch(FtsMatch):
+    def as_sql(self):
+        return 'fts @@ wcs_tsquery(%%(c%s)s)' % id(self.value)
+
+
 class ElementEqual(Criteria):
     def __init__(self, attribute, key, value, **kwargs):
         super().__init__(attribute, value)

wcs/templates/wcs/backoffice/api_access.html

@@ -3,10 +3,12 @@
 {% block body %}
   <div id="appbar">
     <h2>{% trans "API access" %} - {{ api_access.name }}</h2>
-    <span class="actions">
-      <a href="delete" rel="popup">{% trans "Delete" %}</a>
-      <a href="edit">{% trans "Edit" %}</a>
-    </span>
+    {% if not api_access.idp_api_client %}
+      <span class="actions">
+        <a href="delete" rel="popup">{% trans "Delete" %}</a>
+        <a href="edit">{% trans "Edit" %}</a>
+      </span>
+    {% endif %}
   </div>
 
   {% if api_access.description %}
@@ -16,8 +18,12 @@
   <div class="bo-block">
     <h3>{% trans "Parameters" %}</h3>
     <ul>
-      <li>{% trans "Access identifier:" %} {{ api_access.access_identifier }}</li>
-      <li>{% trans "Access key:" %} {{ api_access.access_key }}</li>
+      {% if not api_access.idp_api_client %}
+        <li>{% trans "Access identifier:" %} {{ api_access.access_identifier }}</li>
+        <li>{% trans "Access key:" %} {{ api_access.access_key }}</li>
+      {% else %}
+        <li>{% trans "API client from identity provider, identifier:" %} {{ api_access.access_identifier|removeprefix:"_idp_" }}</li>
+      {% endif %}
       {% if api_access.restrict_to_anonymised_data %}<li>{% trans "Restricted to anonymised data" %}</li>{% endif %}
       {% if api_access.get_roles %}
         <li>{% trans "Roles:" %}

wcs/templates/wcs/backoffice/carddef.html

@@ -14,7 +14,9 @@
       </p>
     {% endif %}
   </div>
+
   {{ publisher.get_request.session.display_message|safe }}
+  {% include "wcs/backoffice/includes/sql-fields-integrity.html" %}
 
   <div class="bo-block">
     <h3>{% trans "Information" %}</h3>
@@ -41,6 +43,7 @@
       <ul class="biglist optionslist">
         {{ options.templates|safe }}
         {{ options.user_support|safe }}
+        {{ options.management|safe }}
       </ul>
     </div>
   </div>

wcs/templates/wcs/backoffice/formdef-inspect.html

@@ -11,6 +11,9 @@
       <button role="tab" aria-selected="false" aria-controls="inspect-workflow" id="tab-workflow" tabindex="-1">{% trans "Workflow" %}</button>
       <button role="tab" aria-selected="false" aria-controls="inspect-options" id="tab-options" tabindex="-1">{% trans "Options" %}</button>
       <button role="tab" aria-selected="false" aria-controls="inspect-fields" id="tab-fields" tabindex="-1">{% trans "Fields" %}</button>
+      {% if not snapshots_diff and not is_carddef %}
+        <button role="tab" aria-selected="false" aria-controls="inspect-drafts" id="tab-drafts" tabindex="-1">{% trans "Drafts" %}</button>
+      {% endif %}
       {% if custom_views %}
         <button role="tab" aria-selected="false" aria-controls="inspect-customviews" id="tab-customviews" tabindex="-1">{% trans "Custom views" %}</button>
       {% endif %}
@@ -93,6 +96,46 @@
         {% endfor %}
       </div>
 
+      {% if not snapshots_diff and not is_carddef %}
+        <div id="inspect-drafts" role="tabpanel" tabindex="0" aria-labelledby="tab-drafts" hidden>
+          {% if total_drafts %}
+            <h2>{% trans "Key indicators on existing drafts" %}</h2>
+            <div class="infonotice">
+              <p>
+                {% blocktrans trimmed with count=formdef.get_drafts_lifespan %}
+                  Covered period: last {{ count }} days.
+                {% endblocktrans %}
+              </p>
+            </div>
+            <h3>{% trans "Rate for in-progress forms, by page" %}</h3>
+            <table class="stats" data-table-id="rate-among-drafts">
+              <tbody>
+                {% for page_drafts in drafts %}
+                  {% include "wcs/backoffice/includes/inspect-draft-by-page.html" with page_id=page_drafts.0 field=page_drafts.1.field percent=page_drafts.1.percent num=page_drafts.1.total den=total_drafts %}
+                {% endfor %}
+              </tbody>
+            </table>
+            <h3>{% trans "Completion rate: count of submitted forms, against count of drafts" %}</h2>
+            <table class="stats completion-rate">
+              <tbody>
+                <tr>
+                  <td class="label"></td>
+                  <td class="percent">{{ percent_submitted_formdata|floatformat }}{% trans "%" %}</td>
+                  <td class="total">({{ total_formdata|subtract:total_drafts }}/{{ total_formdata }})</td>
+                </tr>
+                <tr>
+                  <td class="bar" colspan="3">
+                    <span style="width: {{ percent_submitted_formdata|floatformat:"3u" }}%"></span>
+                  </td>
+                </tr>
+              </tbody>
+            </table>
+          {% else %}
+            <p>{% trans "There are currently no drafts for this form." %}</p>
+          {% endif %}
+        </div>
+      {% endif %}
+
       <div id="inspect-customviews" role="tabpanel" tabindex="0" aria-labelledby="tab-customviews" hidden>
         <div>
           {% for custom_view in custom_views %}

wcs/templates/wcs/backoffice/formdef.html

@@ -35,6 +35,7 @@
   </div>
 
   {{ publisher.get_request.session.display_message|safe }}
+  {% include "wcs/backoffice/includes/sql-fields-integrity.html" %}
 
   <div class="bo-block">
     <h3>{% trans "Information" %}</h3>

wcs/templates/wcs/backoffice/includes/inspect-draft-by-page.html

@@ -0,0 +1,23 @@
+{% load i18n %}
+{% if num %}
+  <tr data-page-id="{{ page_id }}">
+    <td class="label">
+      {% if page_id == "_unknown" %}
+        {% trans "Unknown" %}
+      {% elif page_id == "_first_page" %}
+        {% trans "Only page" %}
+      {% elif page_id == "_confirmation_page" %}
+        {% trans "Confirmation page" %}
+      {% else %}
+        {{ field.ellipsized_label }}
+      {% endif %}
+    </td>
+    <td class="percent">{{ percent|floatformat }}{% trans "%" %}</td>
+    <td class="total">({{ num }}/{{ den }})</td>
+  </tr>
+  <tr>
+    <td class="bar" colspan="3">
+      <span style="width: {{ percent|floatformat:"3u" }}%"></span>
+    </td>
+  </tr>
+{% endif %}