sql: test purge of search tokens (#86527)

introduce a new mechanism to implement FTS with fuzzy-match.
This is made possible by adding and maintaining a table of the
FTS tokens, wcs_search_tokens, fed with searchable_formdefs
and wcs_all_forms.
When a query is issued, its tokens are matched against the
tokens with a fuzzy match when no direct match is found, and
the query is then rebuilt.

tests/admin_pages/test_block.py

@@ -257,6 +257,55 @@ def test_block_delete(pub):
     assert 'This block is still used' in resp
 
 
+def test_block_export_overwrite(pub):
+    create_superuser(pub)
+    BlockDef.wipe()
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [fields.StringField(id='123', required=True, label='Test')]
+    block.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/blocks/%s/' % block.id)
+    resp = resp.click(href=re.compile('^export$'))
+    xml_export = resp.text
+
+    block.slug = 'new-slug'
+    block.name = 'New foobar'
+    block.fields = [fields.StringField(id='123', required=True, label='Test bebore overwrite')]
+    block.store()
+
+    resp = app.get('/backoffice/forms/blocks/%s/' % block.id)
+    resp = resp.click('Overwrite')
+    resp = resp.form.submit('cancel').follow()
+    resp = resp.click('Overwrite')
+    resp = resp.form.submit()
+    assert 'There were errors processing your form.' in resp
+
+    resp.form['file'] = Upload('block', xml_export.encode('utf-8'))
+    resp = resp.form.submit()
+    resp = resp.follow()
+    assert BlockDef.count() == 1
+
+    block.refresh_from_storage()
+    assert block.fields[0].label == 'Test'
+    assert block.name == 'foobar'
+    assert block.slug == 'new-slug'  # not overwritten
+
+    # unknown reference
+    block.fields = [fields.StringField(id='1', data_source={'type': 'foobar'})]
+    block.store()
+    resp = app.get('/backoffice/forms/blocks/%s/' % block.id)
+    resp = resp.click(href=re.compile('^export$'))
+    xml_export = resp.text
+    resp = app.get('/backoffice/forms/blocks/%s/' % block.id)
+    resp = resp.click('Overwrite')
+    resp.form['file'] = Upload('block', xml_export.encode('utf-8'))
+    resp = resp.form.submit()
+    assert 'Invalid File (Unknown referenced objects)' in resp
+    assert '<ul><li>Unknown datasources: foobar</li></ul>' in resp
+
+
 def test_block_edit_duplicate_delete_field(pub):
     create_superuser(pub)
     BlockDef.wipe()
@@ -453,14 +502,14 @@ def test_block_edit_field_warnings(pub):
     blockdef.store()
     resp = app.get('/backoffice/forms/blocks/%s/' % blockdef.id)
     assert 'more than 30 fields' not in resp.text
-    assert '<div id="new-field"><h3>New Field</h3>' in resp.text
+    assert resp.pyquery('#new-field')
     assert resp.pyquery('#fields-list a[title="Duplicate"]').length
 
     blockdef.fields.extend([fields.StringField(id='%d' % i, label='field %d' % i) for i in range(21, 51)])
     blockdef.store()
     resp = app.get('/backoffice/forms/blocks/%s/' % blockdef.id)
     assert 'This block of fields contains 60 fields.' in resp.text
-    assert '<div id="new-field"><h3>New Field</h3>' not in resp.text
+    assert not resp.pyquery('#new-field')
     assert not resp.pyquery('#fields-list a[title="Duplicate"]').length
 
 

tests/admin_pages/test_card.py

@@ -404,16 +404,6 @@ def test_card_id_template(pub):
     carddata.store()
 
     app = login(get_app(pub))
-    resp = app.get('/backoffice/cards/1/')
-    resp = resp.click('Templates')
-    assert 'id_template' not in resp.text
-
-    if not pub.site_options.has_section('options'):
-        pub.site_options.add_section('options')
-    pub.site_options.set('options', 'enable-card-identifier-template', 'true')
-    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
-        pub.site_options.write(fd)
-
     resp = app.get('/backoffice/cards/1/')
     resp = resp.click('Templates')
     assert 'Identifier cannot be modified if there are existing cards.' in resp.text
@@ -437,6 +427,16 @@ def test_card_id_template(pub):
     carddata.store()
     assert carddata.id_display == 'XbarY'
 
+    # check option is not advertised if disabled
+    if not pub.site_options.has_section('options'):
+        pub.site_options.add_section('options')
+    pub.site_options.set('options', 'enable-card-identifier-template', 'false')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+    resp = app.get('/backoffice/cards/1/')
+    resp = resp.click('Templates')
+    assert 'id_template' not in resp.text
+
 
 def test_card_digest_template(pub):
     create_superuser(pub)
@@ -1035,7 +1035,7 @@ def test_card_edit_field_warnings(pub):
     resp = app.get('/backoffice/cards/%s/fields/' % carddef.id)
     assert 'more than 200 fields' not in resp.text
     assert 'first field should be of type "page"' in resp.text
-    assert '<div id="new-field"><h3>New Field</h3>' in resp.text
+    assert resp.pyquery('#new-field')
 
     carddef.fields.extend([fields.StringField(id='%d' % i, label='field %d' % i) for i in range(10, 210)])
     carddef.store()
@@ -1049,7 +1049,7 @@ def test_card_edit_field_warnings(pub):
     resp = app.get('/backoffice/cards/%s/fields/' % carddef.id)
     assert 'This card model contains 410 fields.' in resp.text
     assert 'first field should be of type "page"' in resp.text
-    assert '<div id="new-field"><h3>New Field</h3>' not in resp.text
+    assert not resp.pyquery('#new-field')
     assert '>Duplicate<' not in resp.text
 
 

tests/admin_pages/test_form.py

@@ -242,13 +242,26 @@ def test_forms_edit_management(pub, formdef):
     # Misc management
     assert_option_display(resp, 'Management', 'Default')
     resp = resp.click('Management', href='options/management')
-    assert resp.forms[0]['include_download_all_button'].checked is False
-    resp.forms[0]['include_download_all_button'].checked = True
+    assert resp.forms[0]['management_sidebar_items$elementgeneral'].checked is True
+    assert resp.forms[0]['management_sidebar_items$elementdownload-files'].checked is False
+    resp.forms[0]['management_sidebar_items$elementdownload-files'].checked = True
     resp = resp.forms[0].submit()
     assert resp.location == 'http://example.net/backoffice/forms/1/'
     resp = resp.follow()
     assert_option_display(resp, 'Management', 'Custom')
-    assert FormDef.get(1).include_download_all_button is True
+    assert 'general' in FormDef.get(1).management_sidebar_items
+    assert 'download-files' in FormDef.get(1).management_sidebar_items
+
+    resp = resp.click('Management', href='options/management')
+    resp.forms[0]['management_sidebar_items$elementgeneral'].checked = False
+    resp = resp.forms[0].submit().follow()
+    assert 'general' not in FormDef.get(1).management_sidebar_items
+
+    resp = resp.click('Management', href='options/management')
+    resp.forms[0]['management_sidebar_items$elementgeneral'].checked = True
+    resp.forms[0]['management_sidebar_items$elementdownload-files'].checked = False
+    resp = resp.forms[0].submit().follow()
+    assert FormDef.get(1).management_sidebar_items == {'__default__'}
 
 
 def test_forms_edit_tracking_code(pub, formdef):
@@ -510,6 +523,34 @@ def test_forms_edit_publication_date(pub):
     assert 'invalid value' in resp
 
 
+def test_forms_list_publication_date(pub):
+    create_superuser(pub)
+    create_role(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = []
+    formdef.publication_date = '2024-03-06 00:00'
+    formdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/')
+    assert resp.pyquery('.publication-dates').text() == 'Published from 2024-03-06 00:00'
+
+    formdef.expiration_date = '2024-03-10 00:00'
+    formdef.store()
+    resp = app.get('/backoffice/forms/')
+    assert (
+        resp.pyquery('.publication-dates').text() == 'Published from 2024-03-06 00:00 until 2024-03-10 00:00'
+    )
+
+    formdef.publication_date = None
+    formdef.store()
+    resp = app.get('/backoffice/forms/')
+    assert resp.pyquery('.publication-dates').text() == 'Published until 2024-03-10 00:00'
+
+
 def test_form_category(pub):
     create_superuser(pub)
     create_role(pub)
@@ -3602,7 +3643,7 @@ def test_form_edit_field_warnings(pub):
     resp = app.get('/backoffice/forms/%s/fields/' % formdef.id)
     assert 'more than 200 fields' not in resp.text
     assert 'first field should be of type "page"' in resp.text
-    assert '<div id="new-field"><h3>New Field</h3>' in resp.text
+    assert resp.pyquery('#new-field')
 
     formdef.fields.extend([fields.StringField(id='%d' % i, label='field %d' % i) for i in range(10, 210)])
     formdef.store()
@@ -3617,7 +3658,7 @@ def test_form_edit_field_warnings(pub):
     assert 'This form contains 410 fields.' in resp.text
     assert 'no new fields can be added.' in resp.text
     assert 'first field should be of type "page"' in resp.text
-    assert '<div id="new-field"><h3>New Field</h3>' not in resp.text
+    assert not resp.pyquery('#new-field')
     assert '>Duplicate<' not in resp.text
     assert resp.pyquery('aside .errornotice')
     assert not resp.pyquery('aside form[action=new]')
@@ -3630,7 +3671,7 @@ def test_form_edit_field_warnings(pub):
 
     resp = app.get('/backoffice/forms/%s/fields/' % formdef.id)
     assert 'no new fields should be added.' in resp.text
-    assert '<div id="new-field"><h3>New Field</h3>' in resp.text
+    assert resp.pyquery('#new-field')
     assert '>Duplicate<' in resp.text
     assert not resp.pyquery('aside .errornotice')
     assert resp.pyquery('aside form[action=new]')
@@ -4661,6 +4702,21 @@ def test_admin_form_inspect(pub):
     # check field links targets per-page URL
     assert '/pages/' not in resp.pyquery('.inspect-field h4 a')[0].attrib['href']
 
+    # check drafts lifespan value
+    assert [
+        PyQuery(x).parent().text()
+        for x in resp.pyquery('.parameter')
+        if x.text == 'Lifespan of drafts (in days):'
+    ] == ['Lifespan of drafts (in days): 100']
+    formdef.drafts_lifespan = '40'
+    formdef.store()
+    resp = app.get('/backoffice/forms/%s/inspect' % formdef.id)
+    assert [
+        PyQuery(x).parent().text()
+        for x in resp.pyquery('.parameter')
+        if x.text == 'Lifespan of drafts (in days):'
+    ] == ['Lifespan of drafts (in days): 40']
+
 
 def test_admin_form_inspect_validation(pub):
     create_superuser(pub)

tests/admin_pages/test_i18n.py

@@ -7,7 +7,7 @@ from webtest import Upload
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.categories import Category
-from wcs.fields import ItemField, StringField
+from wcs.fields import ItemField, PageField, StringField
 from wcs.formdef import FormDef
 from wcs.i18n import TranslatableMessage
 from wcs.mail_templates import MailTemplate
@@ -81,6 +81,13 @@ def test_i18n_page(pub):
     formdef = FormDef()
     formdef.name = 'test title'
     formdef.fields = [
+        PageField(
+            id='0',
+            label='page field',
+            post_conditions=[
+                {'condition': {'type': 'django', 'value': 'blah'}, 'error_message': 'page error message'},
+            ],
+        ),
         StringField(id='1', label='text field'),
         StringField(
             id='2',
@@ -125,6 +132,9 @@ def test_i18n_page(pub):
     # check 'text field' only appears one
     assert TranslatableMessage.count([Equal('string', 'text field')]) == 1
 
+    # check page post condition
+    assert TranslatableMessage.count([Equal('string', 'page error message')]) == 1
+
     # check global action name appears only if there's a manual trigger
     assert TranslatableMessage.count([Equal('string', 'Global Manual')]) == 1
     assert TranslatableMessage.count([Equal('string', 'Global No Trigger')]) == 0
@@ -153,7 +163,7 @@ def test_i18n_page(pub):
     # check filtering on a formdef/carddef outputs related workflow strings
     resp.form['formdef'] = 'forms/1'
     resp = resp.form.submit()
-    assert resp.pyquery('tr').length == 12
+    assert resp.pyquery('tr').length == 14
     assert 'test title' in {x.text for x in resp.pyquery('tr td:first-child')}
     assert 'Global Manual' in {x.text for x in resp.pyquery('tr td:first-child')}
     assert 'second workflow' not in {x.text for x in resp.pyquery('tr td:first-child')}

tests/api/test_formdata.py

@@ -1366,6 +1366,9 @@ def test_api_list_formdata_string_filter(pub, local_user):
         ('existing', 'on', 3),
         ('between', 'FOO 1|FOO 2', 1),
         ('between', 'FOO 2|FOO 1', 1),
+        ('icontains', 'FOO', 3),
+        ('icontains', 'foo', 3),
+        ('icontains', '2', 1),
     ]
     for operator, value, result in params:
         resp = get_app(pub).get(
@@ -1542,6 +1545,9 @@ def test_api_list_formdata_text_filter(pub, local_user):
         ('existing', 'on', 3),
         ('between', 'FOO 1|FOO 2', 1),
         ('between', 'FOO 2|FOO 1', 1),
+        ('icontains', 'FOO', 3),
+        ('icontains', 'foo', 3),
+        ('icontains', '2', 1),
     ]
     for operator, value, result in params:
         resp = get_app(pub).get(
@@ -2011,6 +2017,8 @@ def test_api_list_formdata_email_filter(pub, local_user):
         ('not_in', 'a@localhost|b@localhost', 1),
         ('absent', 'on', 2),
         ('existing', 'on', 3),
+        ('icontains', 'A@LOCAL', 1),
+        ('icontains', 'C@LOCAL', 0),
     ]
     for operator, value, result in params:
         resp = get_app(pub).get(
@@ -2307,6 +2315,8 @@ def test_api_list_formdata_block_field_filter(pub, local_user):
         ('existing', 'on', 12),
         ('between', 'plop1|plop5', 7),
         ('between', 'plop5|plop1', 7),
+        ('icontains', 'PLOP', 12),
+        ('icontains', 'LOP1', 4),  # plop1 (twice), plop10, plop11
     ]
     for operator, value, result in params:
         resp = get_app(pub).get(

tests/api/test_formdef.py

@@ -429,6 +429,9 @@ def test_backoffice_submission_formdef_list_search(pub, local_user, access, auth
     resp = get_url('/api/formdefs/?backoffice-submission=on&q=test')
     assert len(resp.json['data']) == 2
 
+    resp = get_url('/api/formdefs/?backoffice-submission=on&q=tes')
+    assert len(resp.json['data']) == 2
+
     resp = get_url('/api/formdefs/?backoffice-submission=on&q=xyz')
     assert len(resp.json['data']) == 0
 

tests/backoffice_pages/test_all.py

@@ -2184,7 +2184,7 @@ def test_backoffice_download_as_zip(pub):
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
     assert 'Download all files as .zip' not in resp
-    formdef.include_download_all_button = True
+    formdef.management_sidebar_items.add('download-files')
     formdef.store()
     resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
     resp = resp.click('Download all files as .zip')
@@ -2264,6 +2264,48 @@ def test_backoffice_geolocation_info(pub):
     assert 'data-init-lat="48.83' in resp.text
 
 
+def test_backoffice_sidebar_elements(pub):
+    user = create_user(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.geolocations = {'base': 'Geolocation'}
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.geolocations = {'base': {'lat': 48.83, 'lon': 2.32}}
+    formdata.user_id = user.id
+    formdata.just_created()
+    formdata.store()
+
+    other_formdata = formdef.data_class()()
+    other_formdata.just_created()
+    other_formdata.just_created()
+    other_formdata.store()
+
+    app = login(get_app(pub))
+    resp = app.get(formdata.get_backoffice_url())
+    assert [x.text for x in resp.pyquery('#sidebar .extra-context h3')] == [
+        'General Information',
+        'Associated User',
+        'Geolocation',
+    ]
+    assert len(resp.pyquery('[data-async-url$="/user-pending-forms"]')) == 1
+
+    formdef.management_sidebar_items = ['general', 'pending-forms']
+    formdef.store()
+    resp = app.get(formdata.get_backoffice_url())
+    assert [x.text for x in resp.pyquery('#sidebar .extra-context h3')] == ['General Information']
+    assert len(resp.pyquery('[data-async-url$="/user-pending-forms"]')) == 1
+
+    formdef.management_sidebar_items = ['geolocation']
+    formdef.store()
+    resp = app.get(formdata.get_backoffice_url())
+    assert [x.text for x in resp.pyquery('#sidebar .extra-context h3')] == ['Geolocation']
+    assert len(resp.pyquery('[data-async-url$="/user-pending-forms"]')) == 0
+
+
 def test_backoffice_info_text(pub):
     create_user(pub)
     create_environment(pub)
@@ -5081,7 +5123,8 @@ def test_backoffice_logged_errors(pub):
     assert 'ZeroDivisionError' in resp2.text
     resp = resp2.click('Failed to evaluate condition')
     assert 'ZeroDivisionError: integer division or modulo by zero' in resp.text
-    assert 'Python Expression: <code>1//0</code>' in resp.text
+    assert 'Condition: <code>1//0</code>' in resp.text
+    assert 'Condition type: <code>python</code>' in resp.text
     resp = resp.click('Delete').follow()
     assert pub.loggederror_class.count() == 0
 
@@ -5459,322 +5502,6 @@ def test_lazy_eval_with_conditional_workflow_form(pub):
     assert context['form_var_foo_bar'] == 'go'
 
 
-@pytest.fixture(params=[{'attach_to_history': True}, {}])
-def create_formdata(request, pub):
-    admin = create_user(pub, is_admin=True)
-
-    FormDef.wipe()
-
-    source_formdef = FormDef()
-    source_formdef.name = 'source form'
-    source_formdef.workflow_roles = {'_receiver': 1}
-    source_formdef.fields = [
-        fields.StringField(id='0', label='string', varname='toto_string'),
-        fields.FileField(id='1', label='file', varname='toto_file'),
-    ]
-    source_formdef.store()
-
-    target_formdef = FormDef()
-    target_formdef.name = 'target form'
-    target_formdef.workflow_roles = {'_receiver': 1}
-    target_formdef.backoffice_submission_roles = admin.roles[:]
-    target_formdef.fields = [
-        fields.StringField(id='0', label='string', varname='foo_string'),
-        fields.FileField(id='1', label='file', varname='foo_file'),
-    ]
-    target_formdef.store()
-    wf = Workflow(name='create-formdata')
-
-    st1 = wf.add_status('New')
-    st2 = wf.add_status('Resubmit')
-
-    jump = st1.add_action('choice', id='_resubmit')
-    jump.label = 'Resubmit'
-    jump.by = ['_receiver']
-    jump.status = st2.id
-
-    create_formdata = st2.add_action('create_formdata', id='_create_formdata')
-    create_formdata.varname = 'resubmitted'
-    create_formdata.draft = True
-    create_formdata.formdef_slug = target_formdef.url_name
-    create_formdata.user_association_mode = 'keep-user'
-    create_formdata.backoffice_submission = True
-    create_formdata.attach_to_history = request.param.get('attach_to_history', False)
-    create_formdata.mappings = [
-        Mapping(field_id='0', expression='=form_var_toto_string'),
-        Mapping(field_id='1', expression='=form_var_toto_file_raw'),
-    ]
-
-    redirect = st2.add_action('redirect_to_url', id='_redirect')
-    redirect.url = '{{ form_links_resubmitted.form_backoffice_url }}'
-
-    jump = st2.add_action('jumponsubmit', id='_jump')
-    jump.status = st1.id
-
-    wf.store()
-    source_formdef.workflow_id = wf.id
-    source_formdef.store()
-    source_formdef.data_class().wipe()
-    target_formdef.data_class().wipe()
-    return locals()
-
-
-def test_backoffice_create_formdata_backoffice_submission(pub, create_formdata):
-    # create submitting user
-    user = create_formdata['pub'].user_class()
-    user.name = 'Jean Darmette'
-    user.email = 'jean.darmette@triffouilis.fr'
-    user.store()
-
-    # create source formdata
-    formdata = create_formdata['source_formdef'].data_class()()
-    upload = PicklableUpload('/foo/bar', content_type='text/plain')
-    upload.receive([b'hello world'])
-    formdata.data = {
-        '0': 'coucou',
-        '1': upload,
-    }
-    formdata.user = user
-    formdata.just_created()
-    formdata.store()
-    formdata.perform_workflow()
-
-    # agent login and go to backoffice management pages
-    app = get_app(create_formdata['pub'])
-    app = login(app)
-    resp = app.get(create_formdata['source_formdef'].get_url(backoffice=True))
-
-    # click on first available formdata
-    resp = resp.click('%s-%s' % (create_formdata['source_formdef'].id, formdata.id))
-    target_data_class = create_formdata['target_formdef'].data_class()
-    assert target_data_class.count() == 0
-    # resubmit it through backoffice submission
-    resp = resp.form.submit(name='button_resubmit')
-    assert pub.loggederror_class.count() == 0
-    assert target_data_class.count() == 1
-    target_formdata = target_data_class.select()[0]
-
-    assert target_formdata.submission_context == {
-        'orig_object_type': 'formdef',
-        'orig_formdata_id': '1',
-        'orig_formdef_id': '1',
-    }
-    assert target_formdata.submission_agent_id == str(create_formdata['admin'].id)
-    assert target_formdata.user.id == user.id
-    assert target_formdata.status == 'draft'
-    assert target_formdata.receipt_time
-    assert resp.location == 'http://example.net/backoffice/management/target-form/%s/' % target_formdata.id
-    resp = resp.follow()
-    assert resp.location == 'http://example.net/backoffice/submission/target-form/%s/' % target_formdata.id
-    resp = resp.follow()
-    # second redirect with magic-token
-    resp = resp.follow()
-    resp = resp.form.submit(name='submit')  # -> validation
-    resp = resp.form.submit(name='submit')  # -> submission
-    target_formdata = target_data_class.get(id=target_formdata.id)
-    assert target_formdata.user.id == user.id
-    assert target_formdata.status == 'wf-new'
-    resp = resp.follow()
-    pq = resp.pyquery.remove_namespaces()
-    assert pq('.field-type-string .value').text() == 'coucou'
-    assert pq('.field-type-file .value').text() == 'bar'
-
-
-def test_linked_forms_variables(pub, create_formdata):
-    # create source formdata
-    formdata = create_formdata['source_formdef'].data_class()()
-    upload = PicklableUpload('/foo/bar', content_type='text/plain')
-    upload.receive([b'hello world'])
-    formdata.data = {
-        '0': 'coucou',
-        '1': upload,
-    }
-    formdata.just_created()
-    formdata.store()
-    formdata.perform_workflow()
-    formdata.store()
-    formdata.jump_status('2')
-    formdata.perform_workflow()
-    formdata.store()
-
-    pub.substitutions.reset()
-    pub.substitutions.feed(formdata)
-    substvars = pub.substitutions.get_context_variables(mode='lazy')
-    assert str(substvars['form_links_resubmitted_form_var_foo_string']) == 'coucou'
-    assert 'form_links_resubmitted_form_var_foo_string' in substvars.get_flat_keys()
-
-    source_formdata = create_formdata['source_formdef'].data_class().select()[0]
-
-    app = get_app(create_formdata['pub'])
-    app = login(app)
-    resp = app.get(source_formdata.get_url(backoffice=True) + 'inspect')
-    assert '?expand=form_links_resubmitted' in resp
-    resp = app.get(source_formdata.get_url(backoffice=True) + 'inspect?expand=form_links_resubmitted')
-    assert 'form_links_resubmitted_form_var_foo_string' in resp
-
-    # delete target formdata
-    create_formdata['target_formdef'].data_class().wipe()
-    resp = app.get(source_formdata.get_url(backoffice=True) + 'inspect')
-    assert '?expand=form_links_resubmitted' not in resp
-    assert 'form_links_resubmitted_form_var_foo_string' not in resp
-
-    # delete target formdef
-    create_formdata['target_formdef'].remove_self()
-    resp = app.get(source_formdata.get_url(backoffice=True) + 'inspect')
-
-
-def test_backoffice_create_formdata_map_fields_by_varname(pub, create_formdata):
-    create_formdata['create_formdata'].map_fields_by_varname = True
-    create_formdata['create_formdata'].mappings = []
-    create_formdata['wf'].store()
-    create_formdata['source_formdef'].fields = [
-        fields.StringField(id='0', label='string', varname='string0'),
-        fields.FileField(id='1', label='file', varname='file1'),
-        fields.StringField(id='2', label='string', varname='string2', required=False),
-        fields.FileField(id='3', label='file', varname='file3', required=False),
-    ]
-    create_formdata['source_formdef'].store()
-    create_formdata['target_formdef'].fields = [
-        fields.StringField(id='0', label='string', varname='string0'),
-        fields.FileField(id='1', label='file', varname='file1'),
-        fields.StringField(id='2', label='string', varname='string2', required=False),
-        fields.FileField(id='3', label='file', varname='file3', required=False),
-    ]
-    create_formdata['target_formdef'].store()
-
-    # create submitting user
-    user = create_formdata['pub'].user_class()
-    user.name = 'Jean Darmette'
-    user.email = 'jean.darmette@triffouilis.fr'
-    user.store()
-
-    # create source formdata
-    create_formdata['source_formdef'].digest_templates = {'default': 'blah'}
-    create_formdata['source_formdef'].store()
-    formdata = create_formdata['source_formdef'].data_class()()
-    create_formdata['formdata'] = formdata
-    upload = PicklableUpload('/foo/bar', content_type='text/plain')
-    upload.receive([b'hello world'])
-    formdata.data = {
-        '0': 'coucou',
-        '1': upload,
-    }
-    formdata.user = user
-    formdata.just_created()
-    formdata.store()
-    formdata.perform_workflow()
-
-    # agent login and go to backoffice management pages
-    app = get_app(create_formdata['pub'])
-    app = login(app)
-    resp = app.get(create_formdata['source_formdef'].get_url(backoffice=True))
-
-    # click on first available formdata
-    resp = resp.click('%s-%s' % (create_formdata['source_formdef'].id, formdata.id))
-    target_data_class = create_formdata['target_formdef'].data_class()
-    assert target_data_class.count() == 0
-    # resubmit it through backoffice submission
-    resp = resp.form.submit(name='button_resubmit')
-    assert pub.loggederror_class.count() == 0
-    assert target_data_class.count() == 1
-    target_formdata = target_data_class.select()[0]
-
-    assert target_formdata.submission_context == {
-        'orig_object_type': 'formdef',
-        'orig_formdata_id': '1',
-        'orig_formdef_id': '1',
-    }
-    assert target_formdata.submission_agent_id == str(create_formdata['admin'].id)
-    assert target_formdata.user.id == user.id
-    assert target_formdata.status == 'draft'
-    assert resp.location == 'http://example.net/backoffice/management/target-form/%s/' % target_formdata.id
-    resp = resp.follow()
-    assert resp.location == 'http://example.net/backoffice/submission/target-form/%s/' % target_formdata.id
-    resp = resp.follow()
-    # second redirect with magic-token
-    resp = resp.follow()
-    # check parent form is displayed in sidebar
-    assert resp.pyquery('.extra-context--orig-data').attr.href == formdata.get_backoffice_url()
-    assert resp.pyquery('.extra-context--orig-data').text() == 'source form #1-1 (blah)'
-    resp = resp.form.submit(name='submit')  # -> validation
-    resp = resp.form.submit(name='submit')  # -> submission
-    target_formdata = target_data_class.get(id=target_formdata.id)
-    assert target_formdata.user.id == user.id
-    assert target_formdata.status == 'wf-new'
-    resp = resp.follow()
-    pq = resp.pyquery.remove_namespaces()
-    assert pq('.field-type-string .value').text() == 'coucou'
-    assert pq('.field-type-file .value').text() == 'bar'
-
-    resp = app.get(create_formdata['formdata'].get_url(backoffice=True))
-    pq = resp.pyquery.remove_namespaces()
-    assert pq('.field-type-string .value').text() == 'coucou'
-    if create_formdata['create_formdata'].attach_to_history:
-        assert pq('.wf-links')
-    else:
-        assert not pq('.wf-links')
-
-
-def test_backoffice_create_formdata_map_fields_by_varname_plus_empty(pub, create_formdata):
-    create_formdata['create_formdata'].map_fields_by_varname = True
-    create_formdata['create_formdata'].mappings = [
-        Mapping(field_id='0', expression=None),
-    ]
-    create_formdata['wf'].store()
-    create_formdata['source_formdef'].fields = [
-        fields.StringField(id='0', label='string', varname='string0'),
-        fields.StringField(id='2', label='string', varname='string2', required=False),
-    ]
-    create_formdata['source_formdef'].store()
-    create_formdata['target_formdef'].fields = [
-        fields.StringField(id='0', label='string', varname='string0'),
-        fields.StringField(id='2', label='string', varname='string2', required=False),
-    ]
-    create_formdata['target_formdef'].store()
-
-    # create submitting user
-    user = create_formdata['pub'].user_class()
-    user.name = 'Jean Darmette'
-    user.email = 'jean.darmette@triffouilis.fr'
-    user.store()
-
-    # create source formdata
-    formdata = create_formdata['source_formdef'].data_class()()
-    create_formdata['formdata'] = formdata
-    formdata.data = {
-        '0': 'foo',
-        '2': 'bar',
-    }
-    formdata.user = user
-    formdata.just_created()
-    formdata.store()
-    formdata.perform_workflow()
-
-    # agent login and go to backoffice management pages
-    app = get_app(create_formdata['pub'])
-    app = login(app)
-    resp = app.get(create_formdata['source_formdef'].get_url(backoffice=True))
-
-    # click on first available formdata
-    resp = resp.click('%s-%s' % (create_formdata['source_formdef'].id, formdata.id))
-    target_data_class = create_formdata['target_formdef'].data_class()
-    assert target_data_class.count() == 0
-    # resubmit it through backoffice submission
-    resp = resp.form.submit(name='button_resubmit')
-    assert target_data_class.count() == 1
-    target_formdata = target_data_class.select()[0]
-
-    assert target_formdata.submission_context == {
-        'orig_object_type': 'formdef',
-        'orig_formdata_id': '1',
-        'orig_formdef_id': '1',
-    }
-    assert target_formdata.submission_agent_id == str(create_formdata['admin'].id)
-    assert target_formdata.user.id == user.id
-    assert target_formdata.status == 'draft'
-    assert target_formdata.data == {'0': None, '2': 'bar'}
-
-
 def test_backoffice_create_carddata_from_formdata(pub):
     CardDef.wipe()
     FormDef.wipe()

tests/backoffice_pages/test_filters.py

@@ -12,7 +12,8 @@ from wcs.carddef import CardDef
 from wcs.data_sources import NamedDataSource
 from wcs.formdef import FormDef
 from wcs.qommon.http_request import HTTPRequest
-from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef
+from wcs.wf.criticality import MODE_INC
+from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef, WorkflowCriticalityLevel
 
 from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
 from .test_all import create_environment, create_superuser
@@ -1048,6 +1049,7 @@ def test_backoffice_string_filter(pub):
         'not_in',
         'absent',
         'existing',
+        'icontains',
     ]
 
     resp.forms['listing-settings']['filter-4-value'].value = 'a'
@@ -1167,6 +1169,7 @@ def test_backoffice_text_filter(pub):
         'not_in',
         'absent',
         'existing',
+        'icontains',
     ]
 
     resp.forms['listing-settings']['filter-4-value'].value = 'a'
@@ -1227,6 +1230,7 @@ def test_backoffice_email_filter(pub):
         'not_in',
         'absent',
         'existing',
+        'icontains',
     ]
 
     resp.forms['listing-settings']['filter-4-value'].value = 'a@localhost'
@@ -1245,6 +1249,12 @@ def test_backoffice_email_filter(pub):
     assert resp.text.count('>a@localhost</') > 0
     assert resp.text.count('>b@localhost</') == 0
 
+    resp.forms['listing-settings']['filter-4-value'].value = 'a@local'
+    resp.forms['listing-settings']['filter-4-operator'].value = 'icontains'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('>a@localhost</') > 0
+    assert resp.text.count('>b@localhost</') == 0
+
 
 def test_backoffice_date_filter(pub):
     pub.user_class.wipe()
@@ -1761,6 +1771,7 @@ def test_backoffice_block_field_filter(pub):
         'not_in',
         'absent',
         'existing',
+        'icontains',
     ]
     resp.forms['listing-settings']['filter-0-1-value'].value = 'plop0'
     resp = resp.forms['listing-settings'].submit()
@@ -1879,6 +1890,7 @@ def test_backoffice_block_field_filter(pub):
         'not_in',
         'absent',
         'existing',
+        'icontains',
     ]
     resp.forms['listing-settings']['filter-0-5-value'].value = 'a@localhost'
     resp = resp.forms['listing-settings'].submit()
@@ -1893,6 +1905,10 @@ def test_backoffice_block_field_filter(pub):
     resp.forms['listing-settings']['filter-0-5-operator'].value = 'ne'
     resp = resp.forms['listing-settings'].submit()
     assert resp.text.count('<tr') == 1 + 4
+    resp.forms['listing-settings']['filter-0-5-value'].value = '@localhost'
+    resp.forms['listing-settings']['filter-0-5-operator'].value = 'icontains'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<tr') == 1 + 10
 
     # mix
     resp = app.get('/backoffice/management/form-title/')
@@ -1975,3 +1991,76 @@ def test_backoffice_numeric_filter(pub):
     resp = resp.forms['listing-settings'].submit()
     assert resp.text.count('<td>123.4</td>') == 0
     assert resp.text.count('<td>315</td>') > 0
+
+
+def test_backoffice_criticality_filter(pub):
+    pub.user_class.wipe()
+    create_superuser(pub)
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    Workflow.wipe()
+    workflow = Workflow(name='test')
+    workflow.criticality_levels = [
+        WorkflowCriticalityLevel(name='green'),
+        WorkflowCriticalityLevel(name='yellow'),
+        WorkflowCriticalityLevel(name='red'),
+        WorkflowCriticalityLevel(name='black'),
+    ]
+    workflow.add_status('st1')
+    st2 = workflow.add_status('st2')
+    action = st2.add_action('modify_criticality')
+    action.mode = MODE_INC
+    st3 = workflow.add_status('st3')
+    action = st3.add_action('modify_criticality')
+    action.mode = MODE_INC
+    action = st3.add_action('modify_criticality')
+    action.mode = MODE_INC
+    workflow.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.fields = [
+        fields.StringField(id='1', label='Test', type='string', display_locations=['listings']),
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.workflow = workflow
+    formdef.store()
+
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    for i in range(3):
+        formdata = data_class()
+        formdata.data = {'1': f'baz{i}'}
+        formdata.just_created()
+        formdata.store()
+        if i == 0:
+            formdata.jump_status(st2.id)
+        else:
+            formdata.jump_status(st3.id)
+        formdata.perform_workflow()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/form-title/')
+    resp.forms['listing-settings']['filter'] = 'all'
+    resp.forms['listing-settings']['filter-criticality-level'].checked = True
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<td>baz') == 3
+    resp.forms['listing-settings']['filter-criticality-level-value'] = '0'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<td>baz') == 0
+    resp.forms['listing-settings']['filter-criticality-level-value'] = '1'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<td>baz') == 1
+    resp.forms['listing-settings']['filter-criticality-level-value'] = '2'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<td>baz') == 2
+    resp.forms['listing-settings']['filter-criticality-level-value'] = '3'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<td>baz') == 0
+    resp.forms['listing-settings']['filter-criticality-level-value'] = ''
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<td>baz') == 3

tests/backoffice_pages/test_submission.py

@@ -10,6 +10,7 @@ from wcs import fields
 from wcs.carddef import CardDef
 from wcs.formdef import FormDef
 from wcs.qommon.http_request import HTTPRequest
+from wcs.qommon.ident.password_accounts import PasswordAccount
 from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef
 from wcs.wscalls import NamedWsCall
 
@@ -2245,3 +2246,50 @@ def test_backoffice_submission_no_roles(pub):
     assert formdef.data_class().count() == 1
     formdata = formdef.data_class().select()[0]
     assert formdata.data == {'1': 'xxx'}
+
+
+def test_backoffice_submission_then_front(pub):
+    user = create_user(pub)
+
+    front_user = pub.user_class()
+    front_user.name = 'front user'
+    front_user.email = 'test@invalid'
+    front_user.store()
+    account = PasswordAccount(id='front')
+    account.set_password('front')
+    account.user_id = front_user.id
+    account.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [
+        fields.PageField(id='0', label='1st page'),
+        fields.StringField(id='1', label='Field on 1st page'),
+        fields.PageField(id='2', label='2nd page'),
+        fields.StringField(id='3', label='Field on 2nd page'),
+    ]
+    formdef.backoffice_submission_roles = user.roles[:]
+    formdef.workflow_roles = {'_receiver': 1}
+    formdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/submission/')
+    resp = resp.click(formdef.name)
+
+    resp.form['user_id'] = str(front_user.id)  # happens via javascript
+    resp.form['submission_channel'] = 'phone'
+    resp.form['f1'] = 'test submission'
+    resp = resp.form.submit('submit')  # -> 2nd page
+    resp.form['f3'] = 'baz'
+    resp = resp.form.submit('submit')  # -> validation page
+    resp = resp.form.submit('submit')  # final submit
+
+    formdata = formdef.data_class().get(resp.location.split('/')[-2])
+    resp = login(get_app(pub), username='front', password='front').get(formdata.get_url())
+    assert (
+        resp.pyquery('.text-form-recorded').text()
+        == f'The form has been recorded on {formdata.receipt_time.strftime("%Y-%m-%d %H:%M")} '
+        f'with the number {formdata.get_display_id()}. It has been submitted for you by '
+        f'admin after a phone call.'
+    )

tests/form_pages/test_all.py

@@ -6,6 +6,7 @@ import time
 import urllib.parse
 import xml.etree.ElementTree as ET
 import zipfile
+import zoneinfo
 from unittest import mock
 
 import pytest
@@ -1282,7 +1283,7 @@ def test_form_multi_page_page_name_as_title(pub):
     next_page = next_page.forms[0].submit('submit')
     assert_current_page(next_page, 'Validating')
     assert 'Check values then click submit.' in next_page.text
-    assert next_page.text.count('1st page') == 2  # in steps and in main body
+    assert next_page.text.count('1st page') == 3  # in steps (twice) and in main body
 
     # add a comment that will not be displayed and should therefore not be
     # considered.
@@ -1305,7 +1306,48 @@ def test_form_multi_page_page_name_as_title(pub):
     next_page = next_page.forms[0].submit('submit')
     assert_current_page(next_page, 'Validating')
     assert 'Check values then click submit.' in next_page.text
-    assert next_page.text.count('1st page') == 2  # in steps and in main body
+    assert next_page.text.count('1st page') == 3  # in steps (twice) and in main body
+
+
+def test_form_multi_page_go_back(pub):
+    formdef = create_formdef()
+    formdef.fields = [
+        fields.PageField(id='0', label='1st page'),
+        fields.StringField(id='1', label='string'),
+        fields.PageField(id='2', label='2nd page'),
+        fields.PageField(id='3', label='3rd page'),
+        fields.StringField(id='4', label='string 2'),
+    ]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    resp = get_app(pub).get('/test/')
+    resp.forms[0]['f1'] = 'foo'
+    resp = resp.forms[0].submit('submit')  # -> 2nd page
+    assert_current_page(resp, '2nd page')
+    resp = resp.forms[0].submit('submit')  # -> 3rd page
+    assert_current_page(resp, '3rd page')
+    resp.forms[0]['f4'] = 'foo'
+    resp = resp.forms[0].submit('submit')  # -> validation page
+    assert_current_page(resp, 'Validating')
+
+    # go back to second page (javascript would set this)
+    resp.forms[0]['previous-page-id'] = '2'
+    resp = resp.forms[0].submit('previous')
+    assert_current_page(resp, '2nd page')
+    resp = resp.forms[0].submit('submit')  # -> 3rd page
+
+    # go back to first page (javascript would set this)
+    resp.forms[0]['previous-page-id'] = '0'
+    resp = resp.forms[0].submit('previous')
+    assert_current_page(resp, '1st page')
+    resp = resp.forms[0].submit('submit')  # -> 2nd page
+    resp = resp.forms[0].submit('submit')  # -> 3rd page
+
+    # go back to invalid page (javascript would not set this)
+    resp.forms[0]['previous-page-id'] = '10'
+    resp = resp.forms[0].submit('previous')
+    assert_current_page(resp, '1st page')  # fallback to first page
 
 
 def test_form_submit_with_user(pub, emails):
@@ -3366,7 +3408,16 @@ def test_logged_errors(pub):
         )
     )[0]
     assert error.occurences_count == 2
-    assert error.expression == '2//0'
+    assert error.context == {
+        'stack': [
+            {
+                'condition': '2//0',
+                'condition_type': 'python',
+                'source_label': 'Automatic Jump',
+                'source_url': 'http://example.net/backoffice/workflows/12/status/just_submitted/items/_jump/',
+            }
+        ]
+    }
 
     assert pub.loggederror_class.count([Equal('formdef_id', '34')]) == 1
     assert pub.loggederror_class.count([Equal('formdef_id', 'X')]) == 0
@@ -6095,3 +6146,45 @@ def test_form_submit_no_csrf_suddenly_single_page(pub):
     resp = app.get(formdef.get_url())
     resp = resp.form.submit('submit').follow()
     assert formdef.data_class().select()[0].status == 'wf-new'
+
+
+def test_form_submit_timezone(pub):
+    pub.load_site_options()
+    if not pub.site_options.has_section('options'):
+        pub.site_options.add_section('options')
+    pub.site_options.set('options', 'timezone', 'Brazil/East')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+
+    formdef = create_formdef()
+    formdef.data_class().wipe()
+    app = get_app(pub)
+    resp = app.get('/test/')
+    resp = resp.forms[0].submit('submit')  # -> validation
+    resp = resp.forms[0].submit('submit').follow()  # -> done
+    formdata = formdef.data_class().select()[0]
+    assert formdata.receipt_time.astimezone(zoneinfo.ZoneInfo('Brazil/East')).strftime('%H:%M') in resp.text
+    assert (
+        formdata.receipt_time.astimezone(zoneinfo.ZoneInfo('Europe/Paris')).strftime('%H:%M') not in resp.text
+    )
+
+    pub.site_options.set('options', 'timezone', 'Europe/Paris')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+
+    resp = app.get(formdata.get_url())
+    assert (
+        formdata.receipt_time.astimezone(zoneinfo.ZoneInfo('Brazil/East')).strftime('%H:%M') not in resp.text
+    )
+    assert formdata.receipt_time.astimezone(zoneinfo.ZoneInfo('Europe/Paris')).strftime('%H:%M') in resp.text
+
+    # do not crash on invalid timezone
+    pub.site_options.set('options', 'timezone', 'invalid')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+
+    resp = app.get(formdata.get_url())
+    assert (
+        formdata.receipt_time.astimezone(zoneinfo.ZoneInfo('Brazil/East')).strftime('%H:%M') not in resp.text
+    )
+    assert formdata.receipt_time.astimezone(zoneinfo.ZoneInfo('Europe/Paris')).strftime('%H:%M') in resp.text

tests/form_pages/test_file_field.py

@@ -573,3 +573,22 @@ def test_form_file_field_in_block_aria_description(pub):
         resp.pyquery.find('#' + resp.pyquery('[aria-describedby]').attr['aria-describedby']).text()
         == 'field label'
     )
+
+
+def test_file_download_url_on_wrong_field(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [fields.StringField(id='1', label='str1')]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    create_user(pub)
+    app = get_app(pub)
+    login(app, username='foo', password='foo')
+    resp = app.get(formdef.get_url())
+    resp.form['f1'] = 'test'
+    resp = resp.form.submit('submit')  # -> validation
+    resp = resp.form.submit('submit').follow()  # -> submit
+    formdata = formdef.data_class().select()[0]
+    app.get(formdata.get_url() + 'files/1/', status=404)

tests/form_pages/test_formdata.py

@@ -18,10 +18,14 @@ from wcs.data_sources import NamedDataSource
 from wcs.formdef import FormDef
 from wcs.qommon.form import UploadedFile
 from wcs.qommon.misc import ConnectionError
-from wcs.wf.create_formdata import Mapping
 from wcs.wf.export_to_model import transform_to_pdf
 from wcs.wf.form import WorkflowFormFieldsFormDef
-from wcs.workflows import ContentSnapshotPart, Workflow, WorkflowBackofficeFieldsFormDef
+from wcs.workflows import (
+    AttachmentEvolutionPart,
+    ContentSnapshotPart,
+    Workflow,
+    WorkflowBackofficeFieldsFormDef,
+)
 from wcs.wscalls import NamedWsCall
 
 from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
@@ -258,7 +262,7 @@ def test_formdata_attachment_download_to_backoffice_file_field_only(pub):
     attach = st1.add_action('addattachment', id='_attach')
     attach.by = ['_submitter']
     attach.backoffice_filefield_id = 'bo1'
-    attach.attach_to_history = False  # store only in backoffice field
+    attach.attach_to_history = False  # do not display in history
     wf.store()
 
     assert attach.get_backoffice_filefield_options() == [('bo1', 'bo field 1', 'bo1')]
@@ -291,11 +295,56 @@ def test_formdata_attachment_download_to_backoffice_file_field_only(pub):
     assert bo1.content_type == 'text/plain'
     assert bo1.get_content() == b'foobar'
 
-    # but nothing in history
+    # nothing displayed in history
+    resp = resp.follow()
+    assert 'resp.text' not in resp.text
     assert len(formdata.evolution) == 2
     assert len(formdata.evolution[0].parts) == 1
     assert isinstance(formdata.evolution[0].parts[0], ContentSnapshotPart)
-    assert formdata.evolution[1].parts is None
+
+    # but attachment stored
+    assert isinstance(formdata.evolution[1].parts[0], AttachmentEvolutionPart)
+
+
+def test_formdata_attachment_stored(pub):
+    create_user(pub)
+    wf = Workflow(name='status')
+    st1 = wf.add_status('Status1', 'st1')
+    attach = st1.add_action('addattachment', id='_attach')
+    attach.by = ['_submitter']
+    attach.backoffice_filefield_id = None  # do not store as backoffice field
+    attach.attach_to_history = False  # do not display in history
+    wf.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.workflow_id = wf.id
+    formdef.fields = []
+    formdef.store()
+    formdef.data_class().wipe()
+
+    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
+    resp = resp.forms[0].submit('submit')
+    assert 'Check values then click submit.' in resp.text
+    resp = resp.forms[0].submit('submit')
+    assert resp.status_int == 302
+    resp = resp.follow()
+    assert 'The form has been recorded' in resp.text
+
+    resp.forms[0]['attachment_attach$file'] = Upload('test.txt', b'foobar', 'text/plain')
+    resp = resp.forms[0].submit('button_attach')
+
+    # nothing displayed in history
+    resp = resp.follow()
+    assert 'resp.text' not in resp.text
+    formdata = formdef.data_class().select()[0]
+    assert len(formdata.evolution) == 2
+    assert len(formdata.evolution[0].parts) == 1
+    assert isinstance(formdata.evolution[0].parts[0], ContentSnapshotPart)
+
+    # but attachment stored
+    assert isinstance(formdata.evolution[1].parts[0], AttachmentEvolutionPart)
 
 
 def test_formdata_attachment_file_options(pub):
@@ -1540,6 +1589,57 @@ def test_formdata_named_wscall_in_conditions(http_requests, pub):
     assert http_requests.count() == 1
 
 
+def test_formdata_error_with_wscall_in_conditions(http_requests, pub):
+    create_user(pub)
+    NamedWsCall.wipe()
+
+    wscall = NamedWsCall()
+    wscall.name = 'Hello world'
+    wscall.request = {'url': 'http://remote.example.net/404', 'method': 'GET'}
+    wscall.record_on_errors = True
+    wscall.store()
+    assert wscall.slug == 'hello_world'
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.PageField(id='0', label='1st page'),
+        fields.PageField(
+            id='1',
+            label='2nd page',
+            condition={'type': 'python', 'value': 'webservice.hello_world["foo"] == "bar"'},
+        ),
+    ]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    pub.loggederror_class.wipe()
+    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
+    assert '>1st page<' in resp.text
+    assert '>2nd page<' in resp.text
+
+    # condition error and wscall error
+    assert pub.loggederror_class.count() == 2
+    wscall_error, condition_error = pub.loggederror_class.select(order_by='id')
+    assert (
+        wscall_error.context
+        == condition_error.context
+        == {
+            'stack': [
+                {
+                    'condition': 'webservice.hello_world["foo"] == "bar"',
+                    'condition_type': 'python',
+                    'source_label': 'Field: 2nd page',
+                    'source_url': 'http://example.net/backoffice/forms/1/fields/1/',
+                }
+            ]
+        }
+    )
+    assert wscall_error.summary == '[WSCALL] 404 Not Found'
+    assert condition_error.summary == 'Failed to evaluate condition'
+
+
 def test_formdata_named_wscall_in_comment(pub):
     create_user(pub)
     NamedWsCall.wipe()
@@ -1730,131 +1830,6 @@ def test_formdata_evolution_register_comment_to_with_attachment(pub):
     ]
 
 
-def test_create_formdata_show_link_in_history(pub):
-    FormDef.wipe()
-    pub.tracking_code_class.wipe()
-
-    target_formdef = FormDef()
-    target_formdef.name = 'target-form'
-    target_formdef.fields = [
-        fields.StringField(id='0', label='string', varname='foo_string'),
-    ]
-    target_formdef.store()
-
-    wf = Workflow(name='create-formdata')
-    wf.possible_status = Workflow.get_default_workflow().possible_status[:]
-    create = wf.possible_status[1].add_action('create_formdata', id='_create', prepend=True)
-    create.label = 'create a new linked form'
-    create.varname = 'resubmitted'
-    create.mappings = [
-        Mapping(field_id='0', expression='="coincoin"'),
-    ]
-    wf.store()
-
-    source_formdef = FormDef()
-    source_formdef.name = 'source-form'
-    source_formdef.fields = []
-    source_formdef.workflow_id = wf.id
-    source_formdef.enable_tracking_codes = True
-    source_formdef.store()
-
-    create.formdef_slug = target_formdef.url_name
-    create.attach_to_history = True
-    wf.store()
-
-    source_formdef.data_class().wipe()
-    target_formdef.data_class().wipe()
-
-    create_user(pub)
-    app = login(get_app(pub), username='foo', password='foo')
-    resp = app.get('/source-form/')
-    resp = resp.forms[0].submit('submit')
-    assert 'Check values then click submit.' in resp.text
-    resp = resp.forms[0].submit('submit')
-    assert resp.status_int == 302
-    resp = resp.follow()
-    assert 'The form has been recorded' in resp.text
-
-    formdata = source_formdef.data_class().select()[0]
-
-    # logged access: show link to created formdata
-    resp = app.get('/source-form/%s/' % formdata.id)
-    assert 'The form has been recorded on' in resp.text
-    assert 'New form "target-form" created' in resp.text
-    assert resp.pyquery('.wf-links a')
-
-    # anonymous access via tracking code: no link
-    app = get_app(pub)
-    resp = app.get('/code/%s/load' % formdata.tracking_code)
-    resp = resp.follow()
-    assert 'The form has been recorded on' in resp.text
-    assert 'New form "target-form" created' not in resp.text
-    assert not resp.pyquery('.wf-links a')
-
-
-def test_create_formdata_multiple(pub):
-    FormDef.wipe()
-    pub.tracking_code_class.wipe()
-
-    target_formdef = FormDef()
-    target_formdef.name = 'target-form'
-    target_formdef.fields = [
-        fields.StringField(id='0', label='string', varname='foo_string'),
-    ]
-    target_formdef.store()
-
-    wf = Workflow(name='create-formdata')
-    wf.possible_status = Workflow.get_default_workflow().possible_status[:]
-    global_action = wf.add_global_action('create formdata')
-    trigger = global_action.triggers[0]
-    trigger.roles = ['_submitter']
-    create = global_action.add_action('create_formdata')
-    create.label = 'create a new linked form'
-    create.varname = 'resubmitted'
-    create.mappings = [Mapping(field_id='0', expression='plop')]
-    wf.store()
-
-    source_formdef = FormDef()
-    source_formdef.name = 'source-form'
-    source_formdef.fields = []
-    source_formdef.workflow_id = wf.id
-    source_formdef.enable_tracking_codes = True
-    source_formdef.store()
-
-    create.formdef_slug = target_formdef.url_name
-    wf.store()
-
-    source_formdef.data_class().wipe()
-    target_formdef.data_class().wipe()
-
-    user = create_user(pub)
-
-    formdata = source_formdef.data_class()()
-    formdata.user_id = user.id
-    formdata.just_created()
-    formdata.store()
-
-    formdata2 = source_formdef.data_class()()
-    formdata2.user_id = user.id
-    formdata2.just_created()
-    formdata2.store()
-
-    app = login(get_app(pub), username='foo', password='foo')
-    resp = app.get(formdata.get_url())
-
-    resp = resp.form.submit('button-action-1')
-    assert target_formdef.data_class().count() == 1
-
-    resp = app.get(formdata.get_url())
-    resp = resp.form.submit('button-action-1')
-    assert target_formdef.data_class().count() == 2
-
-    # do it from another formdata (should not trigger recursive call detection)
-    resp = app.get(formdata2.get_url())
-    resp = resp.form.submit('button-action-1')
-    assert target_formdef.data_class().count() == 3
-
-
 def test_include_authors_in_form_history(pub):
     user, admin = create_user_and_admin(pub)
     pub.role_class.wipe()

tests/form_pages/test_i18n.py

@@ -86,9 +86,18 @@ def test_i18n_form(pub, user, emails, http_requests):
     formdef = FormDef()
     formdef.name = 'test form'
     formdef.fields = [
-        PageField(id='0', label='page field'),
+        PageField(
+            id='0',
+            label='page field',
+            post_conditions=[
+                {
+                    'condition': {'type': 'django', 'value': 'form_var_text == "test"'},
+                    'error_message': 'page error message',
+                },
+            ],
+        ),
         # label has a trailing white space to check for strip()
-        StringField(id='1', label='text field ', hint='an hint text'),
+        StringField(id='1', label='text field ', hint='an hint text', varname='text'),
         ItemField(
             id='2',
             label='list field',
@@ -126,6 +135,7 @@ def test_i18n_form(pub, user, emails, http_requests):
         ('Notification Body', 'Contenu de notification'),
         ('an hint text', 'un texte d’aide'),
         ('a second hint text', 'un deuxième texte d’aide'),
+        ('page error message', 'message d’erreur de page'),
     ):
         msg = TranslatableMessage()
         msg.string = en
@@ -152,6 +162,10 @@ def test_i18n_form(pub, user, emails, http_requests):
     assert resp.pyquery('#form_label_f1').text() == 'champ texte*'
     assert resp.pyquery('option:nth-child(3)').text() == 'deuxième'
 
+    resp.form['f1'] = 'xxx'
+    resp = resp.form.submit('submit', headers={'Accept-Language': 'fr'})
+    assert 'message d’erreur de page' in resp.pyquery('.global-errors').text()
+
     resp.form['f1'] = 'test'
     resp.form['f2'] = 'second'
     resp.form['f3$element0'] = True

tests/form_pages/test_item_field.py

@@ -604,10 +604,10 @@ def test_item_field_from_custom_view_on_cards(pub):
     resp = app.get(formdef.data_class().select()[0].get_url())
     assert resp.pyquery('.field-type-item .value').text() == 'Yattr%sZ' % baz_id
 
-    # remove card (back to value stored at first)
+    # remove card, the value is still displayed
     carddef.data_class().wipe()
     resp = app.get(formdef.data_class().select()[0].get_url())
-    assert resp.pyquery('.field-type-item .value').text() == 'Xattr%sY' % baz_id
+    assert resp.pyquery('.field-type-item .value').text() == 'Yattr%sZ' % baz_id
 
 
 def test_item_field_from_custom_view_on_cards_filter_status(pub):

tests/form_pages/test_live.py

@@ -777,6 +777,7 @@ def test_field_live_select_autocomplete_jsonvalue_prefill(pub, http_requests):
     assert resp.pyquery('[data-field-id="1"][data-live-source]')
 
 
+@pytest.mark.skipif('JOB_NAME' in os.environ, reason='jenkins python segfault')
 def test_field_live_select(pub, http_requests):
     FormDef.wipe()
     formdef = FormDef()
@@ -1777,6 +1778,102 @@ def test_dynamic_item_field_from_custom_view_on_cards(pub, field_type):
     assert logged_error.summary == '[DATASOURCE] Unknown custom view "as-data-source" for CardDef "items"'
 
 
+def test_dynamic_date_field_from_custom_view_on_cards(pub):
+    pub.role_class.wipe()
+    pub.custom_view_class.wipe()
+
+    user = create_user(pub)
+    role = pub.role_class(name='xxx')
+    role.store()
+    user.roles = [role.id]
+    user.is_admin = True
+    user.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = []
+    formdef.store()
+    formdef.data_class().wipe()
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'items'
+    carddef.digest_templates = {'default': '{{form_var_attr}}'}
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.fields = [
+        fields.StringField(id='1', label='string', varname='attr'),
+        fields.DateField(id='2', label='date'),
+    ]
+    carddef.store()
+    carddef.data_class().wipe()
+    for i, value in enumerate(['foo', 'bar', 'baz']):
+        carddata = carddef.data_class()()
+        carddata.data = {
+            '1': value,
+            '2': datetime.date(2024, 1, 1 + i).timetuple(),
+        }
+        carddata.just_created()
+        carddata.store()
+
+    # create custom view
+    app = login(get_app(pub), username='foo', password='foo')
+
+    resp = app.get('/backoffice/data/items/')
+    resp.forms['listing-settings']['filter-2'].checked = True
+    resp.forms['listing-settings']['filter-status'].checked = True
+    resp = resp.forms['listing-settings'].submit()
+
+    resp.forms['listing-settings']['filter'].value = 'recorded'
+    resp = resp.forms['listing-settings'].submit()
+
+    resp.forms['save-custom-view']['title'] = 'as data source'
+    resp.forms['save-custom-view']['visibility'] = 'datasource'
+    resp = resp.forms['save-custom-view'].submit().follow()
+
+    # make sure <input type=date> is not used, so a template can be entered
+    assert resp.pyquery('[name="filter-2-value"]')[0].attrib['type'] == 'text'
+    resp.forms['listing-settings']['filter-2-value'] = '{{ form_var_blah }}'
+    resp.forms['listing-settings']['filter-2-operator'].value = 'gte'
+
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.forms['listing-settings']['filter-2-value'].value == '{{ form_var_blah }}'
+    assert resp.text.count('<tr') == 1  # thead only
+
+    # save custom view with filter
+    resp = resp.forms['save-custom-view'].submit().follow()
+
+    custom_view = pub.custom_view_class.select()[0]
+
+    # use custom view as source
+    ds = {'type': 'carddef:%s:%s' % (carddef.url_name, custom_view.slug)}
+    formdef.fields = [
+        fields.PageField(id='2', label='1st page'),
+        fields.ItemField(
+            id='0', label='item', varname='blah', items=['2023-01-02', '2024-01-02', '2025-01-02']
+        ),
+        fields.ItemField(id='1', label='string', data_source=ds, display_disabled_items=True),
+    ]
+    formdef.store()
+
+    resp = get_app(pub).get('/test/')
+    assert resp.form['f1'].options == [('', False, '---')]
+    resp.form['f0'] = '2024-01-02'
+    live_resp = app.post('/test/live?modified_field_id[]=0', params=resp.form.submit_fields())
+    assert live_resp.json['result']['1']['items'] == [
+        {'attr': 'bar', 'id': 2, 'text': 'bar'},
+        {'attr': 'baz', 'id': 3, 'text': 'baz'},
+    ]
+
+    resp.form['f0'] = '2023-01-02'
+    live_resp = app.post('/test/live?modified_field_id[]=0', params=resp.form.submit_fields())
+    assert len(live_resp.json['result']['1']['items']) == 3
+
+    resp.form['f0'] = '2025-01-02'
+    live_resp = app.post('/test/live?modified_field_id[]=0', params=resp.form.submit_fields())
+    assert len(live_resp.json['result']['1']['items']) == 0
+
+
 def test_dynamic_item_fields_from_custom_view_on_cards(pub):
     pub.role_class.wipe()
     pub.custom_view_class.wipe()

tests/form_pages/test_prefill.py

@@ -13,6 +13,7 @@ from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.categories import Category
 from wcs.data_sources import NamedDataSource
+from wcs.formdef import FormDef
 from wcs.wf.create_formdata import Mapping
 from wcs.workflows import Workflow
 
@@ -634,6 +635,7 @@ def test_form_page_item_with_variable_data_source_prefill(pub):
 def test_form_page_item_with_card_with_custom_id_prefill(pub):
     create_user(pub)
     CardDef.wipe()
+    FormDef.wipe()
 
     carddef = CardDef()
     carddef.name = 'Test'
@@ -678,6 +680,7 @@ def test_form_page_item_with_card_with_custom_id_prefill(pub):
 def test_form_page_block_with_item_with_card_with_custom_id_prefill(pub):
     create_user(pub)
     CardDef.wipe()
+    FormDef.wipe()
 
     carddef = CardDef()
     carddef.name = 'Test'

tests/test_carddef.py

@@ -5,6 +5,7 @@ import xml.etree.ElementTree as ET
 import pytest
 
 from wcs.blocks import BlockDef
+from wcs.carddata import UpdateRelationsAfterJob
 from wcs.carddef import CardDef
 from wcs.categories import CardDefCategory
 from wcs.data_sources import NamedDataSource
@@ -1328,3 +1329,346 @@ def test_card_custom_id_format(pub):
     assert data_class.force_valid_id_characters('_Fôô  bar-') == '_Foo-bar-'
     assert data_class.force_valid_id_characters('_Fôô  bar☭-') == '_Foo-bar-'
     assert data_class.force_valid_id_characters('_Fôô  bar❗') == '_Foo-bar'
+
+
+def test_card_update_related(pub):
+    BlockDef.wipe()
+    CardDef.wipe()
+    FormDef.wipe()
+
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = [
+        StringField(id='1', label='Test', varname='foo'),
+    ]
+    carddef.digest_templates = {'default': '{{ form_var_foo }}'}
+    carddef.store()
+    carddef.data_class().wipe()
+
+    carddata1 = carddef.data_class()()
+    carddata1.data = {'1': 'card1'}
+    carddata1.just_created()
+    carddata1.store()
+
+    carddata2 = carddef.data_class()()
+    carddata2.data = {'1': 'card2'}
+    carddata2.just_created()
+    carddata2.store()
+
+    # check update against item field
+    formdef = FormDef()
+    formdef.name = 'foo'
+    formdef.fields = [
+        ItemField(id='1', label='Test', data_source={'type': 'carddef:foo'}),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {'1': '1'}
+    formdata.data['1_display'] = formdef.fields[0].store_display_value(formdata.data, formdef.fields[0].id)
+    assert formdata.data['1_display'] == 'card1'
+    formdata.just_created()
+    formdata.store()
+
+    pub.cleanup()
+    carddef = carddef.get(carddef.id)
+    carddata1 = carddef.data_class().get(carddata1.id)
+    carddata1.data = {'1': 'card1-change1'}
+    carddata1.store()
+
+    formdata.refresh_from_storage()
+    assert formdata.data['1_display'] == 'card1-change1'
+
+    # check update against items field
+    formdef = FormDef()
+    formdef.name = 'foo'
+    formdef.fields = [
+        ItemsField(id='1', label='Test', data_source={'type': 'carddef:foo'}),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {'1': ['1', '2']}
+    formdata.data['1_display'] = formdef.fields[0].store_display_value(formdata.data, formdef.fields[0].id)
+    assert formdata.data['1_display'] == 'card1-change1, card2'
+    formdata.just_created()
+    formdata.store()
+
+    pub.cleanup()
+    carddef = carddef.get(carddef.id)
+    carddata1 = carddef.data_class().get(carddata1.id)
+    carddata1.data = {'1': 'card1-change2'}
+    carddata1.store()
+
+    formdata.refresh_from_storage()
+    assert formdata.data['1_display'] == 'card1-change2, card2'
+
+    # check update against block field
+    blockdef = BlockDef()
+    blockdef.name = 'foo'
+    blockdef.fields = [
+        ItemField(id='1', label='Test', varname='bar', data_source={'type': 'carddef:foo'}),
+    ]
+    blockdef.digest_template = 'bloc:{{ block_var_bar }}'
+    blockdef.store()
+
+    formdef = FormDef()
+    formdef.name = 'foo2'
+    formdef.fields = [
+        BlockField(id='1', label='Test', block_slug=blockdef.slug),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {
+        '1': {
+            'data': [
+                {
+                    '1': '1',
+                    '1_display': 'card1-change2',
+                },
+                {
+                    '1': '2',
+                    '1_display': 'card2',
+                },
+            ],
+            'schema': {},
+        }
+    }
+
+    formdata.data['1_display'] = formdef.fields[0].store_display_value(formdata.data, formdef.fields[0].id)
+    assert formdata.data['1_display'] == 'bloc:card1-change2, bloc:card2'
+    formdata.just_created()
+    formdata.store()
+
+    pub.cleanup()
+    carddef = carddef.get(carddef.id)
+    carddata1 = carddef.data_class().get(carddata1.id)
+    carddata1.data = {'1': 'card1-change3'}
+    carddata1.store()
+
+    formdata.refresh_from_storage()
+    assert formdata.data['1']['data'][0]['1'] == '1'
+    assert formdata.data['1']['data'][0]['1_display'] == 'card1-change3'
+    assert formdata.data['1']['data'][1]['1'] == '2'
+    assert formdata.data['1']['data'][1]['1_display'] == 'card2'
+    assert formdata.data['1_display'] == 'bloc:card1-change3, bloc:card2'
+
+
+def test_card_update_related_with_custom_view(pub):
+    CardDef.wipe()
+    FormDef.wipe()
+    pub.custom_view_class.wipe()
+
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = [
+        StringField(id='1', label='Test', varname='foo'),
+    ]
+    carddef.digest_templates = {
+        'default': '{{ form_var_foo }}',
+        'custom-view:view': 'view-{{ form_var_foo }}',
+    }
+    carddef.store()
+    carddef.data_class().wipe()
+
+    carddata1 = carddef.data_class()()
+    carddata1.data = {'1': 'card1'}
+    carddata1.just_created()
+    carddata1.store()
+
+    carddata2 = carddef.data_class()()
+    carddata2.data = {'1': 'card2'}
+    carddata2.just_created()
+    carddata2.store()
+
+    custom_view = pub.custom_view_class()
+    custom_view.title = 'view'
+    custom_view.formdef = carddef
+    custom_view.columns = {'list': [{'id': 'id'}]}
+    custom_view.filters = {}
+    custom_view.visibility = 'datasource'
+    custom_view.store()
+
+    formdef = FormDef()
+    formdef.name = 'foo'
+    formdef.fields = [
+        ItemField(id='1', label='Test', data_source={'type': 'carddef:foo:view'}),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {'1': '1'}
+    formdata.data['1_display'] = formdef.fields[0].store_display_value(formdata.data, formdef.fields[0].id)
+    assert formdata.data['1_display'] == 'view-card1'
+    formdata.just_created()
+    formdata.store()
+
+    pub.cleanup()
+    carddef = carddef.get(carddef.id)
+    carddata1 = carddef.data_class().get(carddata1.id)
+    carddata1.data = {'1': 'card1-change1'}
+    carddata1.store()
+
+    formdata.refresh_from_storage()
+    assert formdata.data['1_display'] == 'view-card1-change1'
+
+
+def test_card_update_related_cascading(pub):
+    BlockDef.wipe()
+    CardDef.wipe()
+    FormDef.wipe()
+
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = [
+        StringField(id='1', label='Test', varname='foo'),
+    ]
+    carddef.digest_templates = {'default': '{{ form_var_foo }}'}
+    carddef.store()
+    carddef.data_class().wipe()
+
+    carddata1 = carddef.data_class()()
+    carddata1.data = {'1': 'card1'}
+    carddata1.just_created()
+    carddata1.store()
+
+    carddef2 = CardDef()
+    carddef2.name = 'bar'
+    carddef2.fields = [
+        ItemField(id='1', label='Test', varname='foo', data_source={'type': 'carddef:foo'}),
+    ]
+    carddef2.digest_templates = {'default': 'bar-{{ form_var_foo }}'}
+    carddef2.store()
+    carddef2.data_class().wipe()
+
+    carddata2 = carddef2.data_class()()
+    carddata2.data = {'1': '1'}
+    carddata2.data['1_display'] = carddef2.fields[0].store_display_value(
+        carddata2.data, carddef2.fields[0].id
+    )
+    carddata2.just_created()
+    carddata2.store()
+
+    formdef = FormDef()
+    formdef.name = 'foo'
+    formdef.fields = [
+        ItemField(id='1', label='Test', data_source={'type': 'carddef:bar'}),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {'1': '1'}
+    formdata.data['1_display'] = formdef.fields[0].store_display_value(formdata.data, formdef.fields[0].id)
+    assert formdata.data['1_display'] == 'bar-card1'
+    formdata.just_created()
+    formdata.store()
+
+    pub.cleanup()
+    carddef = carddef.get(carddef.id)
+    carddata1 = carddef.data_class().get(carddata1.id)
+    carddata1.data = {'1': 'card1-change1'}
+    carddata1.store()
+
+    formdata.refresh_from_storage()
+    assert formdata.data['1_display'] == 'bar-card1-change1'
+
+
+def test_card_update_related_cascading_loop(pub):
+    BlockDef.wipe()
+    CardDef.wipe()
+    FormDef.wipe()
+
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = [
+        StringField(id='1', label='Test', varname='foo'),
+        ItemField(id='2', label='Test', varname='x', data_source={'type': 'carddef:bar'}),
+    ]
+    carddef.digest_templates = {'default': '{{ form_var_foo }} {{ form_var_x }}'}
+    carddef.store()
+    carddef.data_class().wipe()
+
+    carddef2 = CardDef()
+    carddef2.name = 'bar'
+    carddef2.fields = [
+        StringField(id='1', label='Test', varname='foo'),
+        ItemField(id='2', label='Test', varname='x', data_source={'type': 'carddef:foo'}),
+    ]
+    carddef2.digest_templates = {'default': '{{ form_var_foo }} {{ form_var_x }}'}
+    carddef2.store()
+    carddef2.data_class().wipe()
+
+    carddata1 = carddef.data_class()()
+    carddata1.data = {'1': 'card1'}
+    carddata1.just_created()
+    carddata1.store()
+
+    carddata2 = carddef2.data_class()()
+    carddata2.data = {'1': 'card2', '2': '1'}
+    carddata2.data['2_display'] = carddef2.fields[1].store_display_value(
+        carddata2.data, carddef2.fields[1].id
+    )
+    assert carddata2.data['2_display'] == 'card1 None'
+    carddata2.just_created()
+    carddata2.store()
+
+    pub.cleanup()
+    carddef = carddef.get(carddef.id)
+    carddata1 = carddef.data_class().get(carddata1.id)
+    carddata1.data['2'] = str(carddata2.id)
+    carddata1.data['2_display'] = carddef.fields[1].store_display_value(carddata1.data, carddef.fields[1].id)
+    carddata1.store()
+
+    # check it will have stopped once getting back to carddata2
+    carddata2.refresh_from_storage()
+    assert carddata2.data['2_display'] == 'card1 card2 card1 None'
+
+
+def test_card_update_related_deleted(pub):
+    BlockDef.wipe()
+    CardDef.wipe()
+    FormDef.wipe()
+
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = [
+        StringField(id='1', label='Test', varname='foo'),
+    ]
+    carddef.digest_templates = {'default': 'card-{{ form_var_foo }}'}
+    carddef.store()
+    carddef.data_class().wipe()
+
+    carddata1 = carddef.data_class()()
+    carddata1.data = {'1': 'card1'}
+    carddata1.just_created()
+    carddata1.store()
+
+    formdef = FormDef()
+    formdef.name = 'foo'
+    formdef.fields = [
+        ItemField(id='1', label='Test', data_source={'type': 'carddef:foo'}),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {'1': '1'}
+    formdata.data['1_display'] = formdef.fields[0].store_display_value(formdata.data, formdef.fields[0].id)
+    assert formdata.data['1_display'] == 'card-card1'
+    formdata.just_created()
+    formdata.store()
+    formdef.remove_self()
+
+    pub.cleanup()
+    carddef = carddef.get(carddef.id)
+    carddata1 = carddef.data_class().get(carddata1.id)
+    carddata1.data = {'1': 'card1-change1'}
+    carddata1.store()  # do not crash looking for related formdef that has been deleted
+
+    # check the job doesn't fail if the carddef or carddata have been removed
+    job = UpdateRelationsAfterJob(carddata=carddata1)
+    carddata1.remove_self()
+    job.execute()
+
+    carddef.remove_self()
+    job.execute()

tests/test_formdata.py

@@ -3742,6 +3742,16 @@ def test_formdata_user_field(pub, variable_test_data):
     assert condition.evaluate() is False
 
 
+def test_formdata_user_has_deleted_account(pub, variable_test_data):
+    condition = Condition({'type': 'django', 'value': 'form_user_has_deleted_account'})
+    assert condition.evaluate() is False
+
+    local_user = variable_test_data._formdata.user
+    local_user.set_deleted()
+    condition = Condition({'type': 'django', 'value': 'form_user_has_deleted_account'})
+    assert condition.evaluate() is True
+
+
 def test_string_filters(pub, variable_test_data):
     tmpl = Template('{% with form_var_foo_foo|split:"a" as x %}{{x.0}}{% endwith %}', raises=True)
     for mode in (None, 'lazy'):
@@ -4236,6 +4246,8 @@ def test_formdata_filtering_on_fields(pub):
         ('between', 'plop5|plop1', '4'),
         ('between', ['plop1', 'plop5'], '4'),
         ('between', ['plop5', 'plop1'], '4'),
+        ('icontains', 'plop', '10'),
+        ('icontains', 'PLOP', '10'),
     ]
     for operator, value, result in params:
         context['value'] = None
@@ -4494,6 +4506,8 @@ def test_formdata_filtering_on_fields(pub):
         ('not_in', 'a@localhost|b@localhost', '1'),
         ('absent', '', '2'),
         ('existing', '', '10'),
+        ('icontains', 'A@local', '5'),
+        ('icontains', '@LOCAL', '10'),
     ]
     for operator, value, result in params:
         if value:
@@ -4543,6 +4557,8 @@ def test_formdata_filtering_on_fields(pub):
         ('between', 'plop5|plop1', '4'),
         ('between', ['plop1', 'plop5'], '4'),
         ('between', ['plop5', 'plop1'], '4'),
+        ('icontains', 'plop', '10'),
+        ('icontains', 'PLOP', '10'),
     ]
     for operator, value, result in params:
         context['value'] = None
@@ -5312,10 +5328,15 @@ def test_fts_phone(pub):
     formdata.just_created()
     formdata.store()
 
-    assert formdef.data_class().count([FtsMatch('01 23 45 67 89')]) == 1
-    assert formdef.data_class().count([FtsMatch('0123456789')]) == 1
-    assert formdef.data_class().count([FtsMatch('+33123456789')]) == 1
-    assert formdef.data_class().count([FtsMatch('+33(0)123456789')]) == 1
+    formdata = formdef.data_class()()
+    formdata.data = {'1': None, '2': '0123456789'}
+    formdata.just_created()
+    formdata.store()
+
+    assert formdef.data_class().count([FtsMatch('01 23 45 67 89')]) == 2
+    assert formdef.data_class().count([FtsMatch('0123456789')]) == 2
+    assert formdef.data_class().count([FtsMatch('+33123456789')]) == 2
+    assert formdef.data_class().count([FtsMatch('+33(0)123456789')]) == 2
     assert formdef.data_class().count([FtsMatch('+33(0)123456789 foo')]) == 1
     assert formdef.data_class().count([FtsMatch('+33(0)123456789 bar')]) == 0
     assert formdef.data_class().count([FtsMatch('foo +33(0)123456789')]) == 1

tests/test_formdef_import.py

@@ -1077,3 +1077,12 @@ def test_tracking_code_attributes(pub):
         assert f2.enable_tracking_codes == formdef.enable_tracking_codes
         assert f2.tracking_code_verify_fields == formdef.tracking_code_verify_fields
         assert f2.confirmation == formdef.confirmation
+
+
+def test_management_sidebar_items(pub):
+    formdef = FormDef()
+    formdef.name = 'Foo'
+    formdef.url_name = 'foo'
+    formdef.management_sidebar_items = {'general', 'pending-forms'}
+    f2 = assert_xml_import_export_works(formdef)
+    assert f2.management_sidebar_items == {'general', 'pending-forms'}

tests/test_hobo_notify.py

@@ -474,6 +474,7 @@ PROFILE = {
 
 def test_process_notification_user_provision(pub):
     User = pub.user_class
+    User.wipe()
 
     # create some roles
     from wcs.ctl.management.commands.hobo_deploy import Command

tests/test_sql.py

@@ -1183,6 +1183,45 @@ def test_sql_criteria_fts(pub):
     assert data_class.select([st.FtsMatch(formdata1.id_display)])[0].id_display == formdata1.id_display
 
 
+def test_search_tokens_purge(pub):
+    _, cur = sql.get_connection_and_cursor()
+
+    # purge garbage from other tests
+    sql.purge_obsolete_search_tokens()
+
+    cur.execute('SELECT count(*) FROM wcs_search_tokens;')
+    start = cur.fetchone()[0]
+
+    # define a new table
+    test_formdef = FormDef()
+    test_formdef.name = 'tableSelectFTStokens'
+    test_formdef.fields = [fields.StringField(id='3', label='string')]
+    test_formdef.store()
+    data_class = test_formdef.data_class(mode='sql')
+
+    cur.execute('SELECT count(*) FROM wcs_search_tokens;')
+    assert cur.fetchone()[0] == start + 1
+
+    t = data_class()
+    t.data = {'3': 'foofortokensofcourse'}
+    t.just_created()
+    t.store()
+
+    cur.execute('SELECT count(*) FROM wcs_search_tokens;')
+    assert cur.fetchone()[0] == start + 2
+
+    t.data = {'3': 'chaussettefortokensofcourse'}
+    t.store()
+
+    cur.execute('SELECT count(*) FROM wcs_search_tokens;')
+    assert cur.fetchone()[0] == start + 3
+
+    sql.purge_obsolete_search_tokens()
+
+    cur.execute('SELECT count(*) FROM wcs_search_tokens;')
+    assert cur.fetchone()[0] == start + 2
+
+
 def table_exists(cur, table_name):
     cur.execute(
         '''SELECT COUNT(*) FROM information_schema.tables

tests/workflow/test_all.py

@@ -1964,6 +1964,7 @@ def test_redirect_to_url(pub):
 
 
 def test_workflow_action_condition(pub):
+    Workflow.wipe()
     workflow = Workflow(name='jump condition migration')
     st1 = workflow.add_status('Status1', 'st1')
     workflow.store()
@@ -2046,8 +2047,16 @@ def test_workflow_action_condition(pub):
     assert logged_error.summary == 'Failed to evaluate condition'
     assert logged_error.exception_class == 'NameError'
     assert logged_error.exception_message == "name 'foobar' is not defined"
-    assert logged_error.expression == 'foobar == barfoo'
-    assert logged_error.expression_type == 'python'
+    assert logged_error.context == {
+        'stack': [
+            {
+                'condition': 'foobar == barfoo',
+                'condition_type': 'python',
+                'source_label': 'Manual Jump',
+                'source_url': 'http://example.net/backoffice/workflows/1/status/st1/items/_x/',
+            }
+        ]
+    }
 
 
 def test_workflow_field_migration(pub):
@@ -2344,3 +2353,15 @@ def test_visibility_migration(pub):
     assert workflow.possible_status[0].visibility == ['__restricted__']
     assert workflow.possible_status[1].visibility == ['__hidden__']
     assert not workflow.possible_status[2].visibility
+
+
+def test_variables_formdef_clean_prefill(pub):
+    workflow = Workflow(name='variables')
+    workflow.variables_formdef = WorkflowVariablesFieldsFormDef(workflow=workflow)
+    workflow.variables_formdef.fields.append(
+        StringField(label='Test', default_value='123', prefill={'type': 'string', 'value': 'plop'})
+    )
+    workflow.store()
+
+    workflow = Workflow.get(id=workflow.id)
+    assert not workflow.variables_formdef.fields[0].prefill

tests/workflow/test_create_formdata.py

@@ -3,16 +3,20 @@ import datetime
 import pytest
 from quixote import cleanup, get_publisher, get_response
 
-from wcs import sessions
+from wcs import fields, sessions
 from wcs.carddef import CardDef
 from wcs.fields import EmailField, ItemField, StringField
 from wcs.formdef import FormDef
 from wcs.qommon.http_request import HTTPRequest
+from wcs.qommon.upload_storage import PicklableUpload
 from wcs.wf.create_formdata import Mapping
 from wcs.workflow_traces import WorkflowTrace
 from wcs.workflows import Workflow
 
-from ..utilities import clean_temporary_pub, create_temporary_pub
+from ..backoffice_pages.test_all import create_user as create_backoffice_user
+from ..backoffice_pages.test_all import login
+from ..form_pages.test_all import create_user
+from ..utilities import clean_temporary_pub, create_temporary_pub, get_app
 
 
 def setup_module(module):
@@ -27,6 +31,7 @@ def teardown_module(module):
 def pub(request):
     pub = create_temporary_pub()
     pub.cfg['language'] = {'language': 'en'}
+    pub.cfg['identification'] = {'methods': ['password']}
     pub.write_cfg()
     req = HTTPRequest(None, {'SERVER_NAME': 'example.net', 'SCRIPT_NAME': ''})
     req.response.filter = {}
@@ -647,3 +652,514 @@ def test_global_timeouts_create_formdata(pub):
     pub.apply_global_action_timeouts()
 
     assert subformdef.data_class().count() == 1
+
+
+@pytest.fixture(params=[{'attach_to_history': True}, {}])
+def create_formdata(request, pub):
+    admin = create_backoffice_user(pub, is_admin=True)
+
+    FormDef.wipe()
+
+    source_formdef = FormDef()
+    source_formdef.name = 'source form'
+    source_formdef.workflow_roles = {'_receiver': 1}
+    source_formdef.fields = [
+        fields.StringField(id='0', label='string', varname='toto_string'),
+        fields.FileField(id='1', label='file', varname='toto_file'),
+    ]
+    source_formdef.store()
+
+    target_formdef = FormDef()
+    target_formdef.name = 'target form'
+    target_formdef.workflow_roles = {'_receiver': 1}
+    target_formdef.backoffice_submission_roles = admin.roles[:]
+    target_formdef.fields = [
+        fields.StringField(id='0', label='string', varname='foo_string'),
+        fields.FileField(id='1', label='file', varname='foo_file'),
+    ]
+    target_formdef.store()
+    wf = Workflow(name='create-formdata')
+
+    st1 = wf.add_status('New')
+    st2 = wf.add_status('Resubmit')
+
+    jump = st1.add_action('choice', id='_resubmit')
+    jump.label = 'Resubmit'
+    jump.by = ['_receiver']
+    jump.status = st2.id
+
+    create_formdata = st2.add_action('create_formdata', id='_create_formdata')
+    create_formdata.varname = 'resubmitted'
+    create_formdata.draft = True
+    create_formdata.formdef_slug = target_formdef.url_name
+    create_formdata.user_association_mode = 'keep-user'
+    create_formdata.backoffice_submission = True
+    create_formdata.attach_to_history = request.param.get('attach_to_history', False)
+    create_formdata.mappings = [
+        Mapping(field_id='0', expression='=form_var_toto_string'),
+        Mapping(field_id='1', expression='=form_var_toto_file_raw'),
+    ]
+
+    redirect = st2.add_action('redirect_to_url', id='_redirect')
+    redirect.url = '{{ form_links_resubmitted.form_backoffice_url }}'
+
+    jump = st2.add_action('jumponsubmit', id='_jump')
+    jump.status = st1.id
+
+    wf.store()
+    source_formdef.workflow_id = wf.id
+    source_formdef.store()
+    source_formdef.data_class().wipe()
+    target_formdef.data_class().wipe()
+    return locals()
+
+
+def test_backoffice_create_formdata_backoffice_submission(pub, create_formdata):
+    # create submitting user
+    user = create_formdata['pub'].user_class()
+    user.name = 'Jean Darmette'
+    user.email = 'jean.darmette@triffouilis.fr'
+    user.store()
+
+    # create source formdata
+    formdata = create_formdata['source_formdef'].data_class()()
+    upload = PicklableUpload('/foo/bar', content_type='text/plain')
+    upload.receive([b'hello world'])
+    formdata.data = {
+        '0': 'coucou',
+        '1': upload,
+    }
+    formdata.user = user
+    formdata.just_created()
+    formdata.store()
+    formdata.perform_workflow()
+
+    # agent login and go to backoffice management pages
+    app = get_app(create_formdata['pub'])
+    app = login(app)
+    resp = app.get(create_formdata['source_formdef'].get_url(backoffice=True))
+
+    # click on first available formdata
+    resp = resp.click('%s-%s' % (create_formdata['source_formdef'].id, formdata.id))
+    target_data_class = create_formdata['target_formdef'].data_class()
+    assert target_data_class.count() == 0
+    # resubmit it through backoffice submission
+    resp = resp.form.submit(name='button_resubmit')
+    assert pub.loggederror_class.count() == 0
+    assert target_data_class.count() == 1
+    target_formdata = target_data_class.select()[0]
+
+    assert target_formdata.submission_context == {
+        'orig_object_type': 'formdef',
+        'orig_formdata_id': '1',
+        'orig_formdef_id': '1',
+    }
+    assert target_formdata.submission_agent_id == str(create_formdata['admin'].id)
+    assert target_formdata.user.id == user.id
+    assert target_formdata.status == 'draft'
+    assert target_formdata.receipt_time
+    assert resp.location == 'http://example.net/backoffice/management/target-form/%s/' % target_formdata.id
+    resp = resp.follow()
+    assert resp.location == 'http://example.net/backoffice/submission/target-form/%s/' % target_formdata.id
+    resp = resp.follow()
+    # second redirect with magic-token
+    resp = resp.follow()
+    resp = resp.form.submit(name='submit')  # -> validation
+    resp = resp.form.submit(name='submit')  # -> submission
+    target_formdata = target_data_class.get(id=target_formdata.id)
+    assert target_formdata.user.id == user.id
+    assert target_formdata.status == 'wf-new'
+    resp = resp.follow()
+    pq = resp.pyquery.remove_namespaces()
+    assert pq('.field-type-string .value').text() == 'coucou'
+    assert pq('.field-type-file .value').text() == 'bar'
+
+
+def test_linked_forms_variables(pub, create_formdata):
+    # create source formdata
+    formdata = create_formdata['source_formdef'].data_class()()
+    upload = PicklableUpload('/foo/bar', content_type='text/plain')
+    upload.receive([b'hello world'])
+    formdata.data = {
+        '0': 'coucou',
+        '1': upload,
+    }
+    formdata.just_created()
+    formdata.store()
+    formdata.perform_workflow()
+    formdata.store()
+    formdata.jump_status('2')
+    formdata.perform_workflow()
+    formdata.store()
+
+    pub.substitutions.reset()
+    pub.substitutions.feed(formdata)
+    substvars = pub.substitutions.get_context_variables(mode='lazy')
+    assert str(substvars['form_links_resubmitted_form_var_foo_string']) == 'coucou'
+    assert 'form_links_resubmitted_form_var_foo_string' in substvars.get_flat_keys()
+
+    source_formdata = create_formdata['source_formdef'].data_class().select()[0]
+
+    app = get_app(create_formdata['pub'])
+    app = login(app)
+    resp = app.get(source_formdata.get_url(backoffice=True) + 'inspect')
+    assert '?expand=form_links_resubmitted' in resp
+    resp = app.get(source_formdata.get_url(backoffice=True) + 'inspect?expand=form_links_resubmitted')
+    assert 'form_links_resubmitted_form_var_foo_string' in resp
+
+    # delete target formdata
+    create_formdata['target_formdef'].data_class().wipe()
+    resp = app.get(source_formdata.get_url(backoffice=True) + 'inspect')
+    assert '?expand=form_links_resubmitted' not in resp
+    assert 'form_links_resubmitted_form_var_foo_string' not in resp
+
+    # delete target formdef
+    create_formdata['target_formdef'].remove_self()
+    resp = app.get(source_formdata.get_url(backoffice=True) + 'inspect')
+
+
+def test_backoffice_create_formdata_map_fields_by_varname(pub, create_formdata):
+    create_formdata['create_formdata'].map_fields_by_varname = True
+    create_formdata['create_formdata'].mappings = []
+    create_formdata['wf'].store()
+    create_formdata['source_formdef'].fields = [
+        fields.StringField(id='0', label='string', varname='string0'),
+        fields.FileField(id='1', label='file', varname='file1'),
+        fields.StringField(id='2', label='string', varname='string2', required=False),
+        fields.FileField(id='3', label='file', varname='file3', required=False),
+    ]
+    create_formdata['source_formdef'].store()
+    create_formdata['target_formdef'].fields = [
+        fields.StringField(id='0', label='string', varname='string0'),
+        fields.FileField(id='1', label='file', varname='file1'),
+        fields.StringField(id='2', label='string', varname='string2', required=False),
+        fields.FileField(id='3', label='file', varname='file3', required=False),
+    ]
+    create_formdata['target_formdef'].store()
+
+    # create submitting user
+    user = create_formdata['pub'].user_class()
+    user.name = 'Jean Darmette'
+    user.email = 'jean.darmette@triffouilis.fr'
+    user.store()
+
+    # create source formdata
+    create_formdata['source_formdef'].digest_templates = {'default': 'blah'}
+    create_formdata['source_formdef'].store()
+    formdata = create_formdata['source_formdef'].data_class()()
+    create_formdata['formdata'] = formdata
+    upload = PicklableUpload('/foo/bar', content_type='text/plain')
+    upload.receive([b'hello world'])
+    formdata.data = {
+        '0': 'coucou',
+        '1': upload,
+    }
+    formdata.user = user
+    formdata.just_created()
+    formdata.store()
+    formdata.perform_workflow()
+
+    # agent login and go to backoffice management pages
+    app = get_app(create_formdata['pub'])
+    app = login(app)
+    resp = app.get(create_formdata['source_formdef'].get_url(backoffice=True))
+
+    # click on first available formdata
+    resp = resp.click('%s-%s' % (create_formdata['source_formdef'].id, formdata.id))
+    target_data_class = create_formdata['target_formdef'].data_class()
+    assert target_data_class.count() == 0
+    # resubmit it through backoffice submission
+    resp = resp.form.submit(name='button_resubmit')
+    assert pub.loggederror_class.count() == 0
+    assert target_data_class.count() == 1
+    target_formdata = target_data_class.select()[0]
+
+    assert target_formdata.submission_context == {
+        'orig_object_type': 'formdef',
+        'orig_formdata_id': '1',
+        'orig_formdef_id': '1',
+    }
+    assert target_formdata.submission_agent_id == str(create_formdata['admin'].id)
+    assert target_formdata.user.id == user.id
+    assert target_formdata.status == 'draft'
+    assert resp.location == 'http://example.net/backoffice/management/target-form/%s/' % target_formdata.id
+    resp = resp.follow()
+    assert resp.location == 'http://example.net/backoffice/submission/target-form/%s/' % target_formdata.id
+    resp = resp.follow()
+    # second redirect with magic-token
+    resp = resp.follow()
+    # check parent form is displayed in sidebar
+    assert resp.pyquery('.extra-context--orig-data').attr.href == formdata.get_backoffice_url()
+    assert resp.pyquery('.extra-context--orig-data').text() == 'source form #1-1 (blah)'
+    resp = resp.form.submit(name='submit')  # -> validation
+    resp = resp.form.submit(name='submit')  # -> submission
+    target_formdata = target_data_class.get(id=target_formdata.id)
+    assert target_formdata.user.id == user.id
+    assert target_formdata.status == 'wf-new'
+    resp = resp.follow()
+    pq = resp.pyquery.remove_namespaces()
+    assert pq('.field-type-string .value').text() == 'coucou'
+    assert pq('.field-type-file .value').text() == 'bar'
+
+    resp = app.get(create_formdata['formdata'].get_url(backoffice=True))
+    pq = resp.pyquery.remove_namespaces()
+    assert pq('.field-type-string .value').text() == 'coucou'
+    if create_formdata['create_formdata'].attach_to_history:
+        assert pq('.wf-links')
+    else:
+        assert not pq('.wf-links')
+
+
+def test_backoffice_create_formdata_map_fields_by_varname_plus_empty(pub, create_formdata):
+    create_formdata['create_formdata'].map_fields_by_varname = True
+    create_formdata['create_formdata'].mappings = [
+        Mapping(field_id='0', expression=None),
+    ]
+    create_formdata['wf'].store()
+    create_formdata['source_formdef'].fields = [
+        fields.StringField(id='0', label='string', varname='string0'),
+        fields.StringField(id='2', label='string', varname='string2', required=False),
+    ]
+    create_formdata['source_formdef'].store()
+    create_formdata['target_formdef'].fields = [
+        fields.StringField(id='0', label='string', varname='string0'),
+        fields.StringField(id='2', label='string', varname='string2', required=False),
+    ]
+    create_formdata['target_formdef'].store()
+
+    # create submitting user
+    user = create_formdata['pub'].user_class()
+    user.name = 'Jean Darmette'
+    user.email = 'jean.darmette@triffouilis.fr'
+    user.store()
+
+    # create source formdata
+    formdata = create_formdata['source_formdef'].data_class()()
+    create_formdata['formdata'] = formdata
+    formdata.data = {
+        '0': 'foo',
+        '2': 'bar',
+    }
+    formdata.user = user
+    formdata.just_created()
+    formdata.store()
+    formdata.perform_workflow()
+
+    # agent login and go to backoffice management pages
+    app = get_app(create_formdata['pub'])
+    app = login(app)
+    resp = app.get(create_formdata['source_formdef'].get_url(backoffice=True))
+
+    # click on first available formdata
+    resp = resp.click('%s-%s' % (create_formdata['source_formdef'].id, formdata.id))
+    target_data_class = create_formdata['target_formdef'].data_class()
+    assert target_data_class.count() == 0
+    # resubmit it through backoffice submission
+    resp = resp.form.submit(name='button_resubmit')
+    assert target_data_class.count() == 1
+    target_formdata = target_data_class.select()[0]
+
+    assert target_formdata.submission_context == {
+        'orig_object_type': 'formdef',
+        'orig_formdata_id': '1',
+        'orig_formdef_id': '1',
+    }
+    assert target_formdata.submission_agent_id == str(create_formdata['admin'].id)
+    assert target_formdata.user.id == user.id
+    assert target_formdata.status == 'draft'
+    assert target_formdata.data == {'0': None, '2': 'bar'}
+
+
+def test_create_formdata_show_link_in_history(pub):
+    FormDef.wipe()
+    pub.tracking_code_class.wipe()
+
+    target_formdef = FormDef()
+    target_formdef.name = 'target-form'
+    target_formdef.fields = [
+        fields.StringField(id='0', label='string', varname='foo_string'),
+    ]
+    target_formdef.store()
+
+    wf = Workflow(name='create-formdata')
+    wf.possible_status = Workflow.get_default_workflow().possible_status[:]
+    create = wf.possible_status[1].add_action('create_formdata', id='_create', prepend=True)
+    create.label = 'create a new linked form'
+    create.varname = 'resubmitted'
+    create.mappings = [
+        Mapping(field_id='0', expression='="coincoin"'),
+    ]
+    wf.store()
+
+    source_formdef = FormDef()
+    source_formdef.name = 'source-form'
+    source_formdef.fields = []
+    source_formdef.workflow_id = wf.id
+    source_formdef.enable_tracking_codes = True
+    source_formdef.store()
+
+    create.formdef_slug = target_formdef.url_name
+    create.attach_to_history = True
+    wf.store()
+
+    source_formdef.data_class().wipe()
+    target_formdef.data_class().wipe()
+
+    create_user(pub)
+    app = login(get_app(pub), username='foo', password='foo')
+    resp = app.get('/source-form/')
+    resp = resp.forms[0].submit('submit')
+    assert 'Check values then click submit.' in resp.text
+    resp = resp.forms[0].submit('submit')
+    assert resp.status_int == 302
+    resp = resp.follow()
+    assert 'The form has been recorded' in resp.text
+
+    formdata = source_formdef.data_class().select()[0]
+
+    # logged access: show link to created formdata
+    resp = app.get('/source-form/%s/' % formdata.id)
+    assert 'The form has been recorded on' in resp.text
+    assert 'New form "target-form" created' in resp.text
+    assert resp.pyquery('.wf-links a')
+
+    # anonymous access via tracking code: no link
+    app = get_app(pub)
+    resp = app.get('/code/%s/load' % formdata.tracking_code)
+    resp = resp.follow()
+    assert 'The form has been recorded on' in resp.text
+    assert 'New form "target-form" created' not in resp.text
+    assert not resp.pyquery('.wf-links a')
+
+
+def test_create_formdata_multiple(pub):
+    FormDef.wipe()
+    pub.tracking_code_class.wipe()
+
+    target_formdef = FormDef()
+    target_formdef.name = 'target-form'
+    target_formdef.fields = [
+        fields.StringField(id='0', label='string', varname='foo_string'),
+    ]
+    target_formdef.store()
+
+    wf = Workflow(name='create-formdata')
+    wf.possible_status = Workflow.get_default_workflow().possible_status[:]
+    global_action = wf.add_global_action('create formdata')
+    trigger = global_action.triggers[0]
+    trigger.roles = ['_submitter']
+    create = global_action.add_action('create_formdata')
+    create.label = 'create a new linked form'
+    create.varname = 'resubmitted'
+    create.mappings = [Mapping(field_id='0', expression='plop')]
+    wf.store()
+
+    source_formdef = FormDef()
+    source_formdef.name = 'source-form'
+    source_formdef.fields = []
+    source_formdef.workflow_id = wf.id
+    source_formdef.enable_tracking_codes = True
+    source_formdef.store()
+
+    create.formdef_slug = target_formdef.url_name
+    wf.store()
+
+    source_formdef.data_class().wipe()
+    target_formdef.data_class().wipe()
+
+    user = create_user(pub)
+
+    formdata = source_formdef.data_class()()
+    formdata.user_id = user.id
+    formdata.just_created()
+    formdata.store()
+
+    formdata2 = source_formdef.data_class()()
+    formdata2.user_id = user.id
+    formdata2.just_created()
+    formdata2.store()
+
+    app = login(get_app(pub), username='foo', password='foo')
+    resp = app.get(formdata.get_url())
+
+    resp = resp.form.submit('button-action-1')
+    assert target_formdef.data_class().count() == 1
+
+    resp = app.get(formdata.get_url())
+    resp = resp.form.submit('button-action-1')
+    assert target_formdef.data_class().count() == 2
+
+    # do it from another formdata (should not trigger recursive call detection)
+    resp = app.get(formdata2.get_url())
+    resp = resp.form.submit('button-action-1')
+    assert target_formdef.data_class().count() == 3
+
+
+@pytest.mark.parametrize('mode', ['single', 'partial'])
+def test_create_formdata_edit_single_or_partial_pages(pub, mode):
+    FormDef.wipe()
+    pub.tracking_code_class.wipe()
+
+    target_formdef = FormDef()
+    target_formdef.name = 'target-form'
+    target_formdef.fields = [
+        fields.PageField(id='1', label='page1'),
+        fields.StringField(id='2', label='string', varname='foo_string'),
+        fields.PageField(id='3', label='page2', varname='page2'),
+        fields.StringField(id='4', label='string2', varname='bar_string'),
+        fields.PageField(id='4', label='page3'),
+    ]
+    target_formdef.store()
+
+    wf = Workflow(name='create-formdata')
+    wf.possible_status = Workflow.get_default_workflow().possible_status[:]
+    create = wf.possible_status[1].add_action('create_formdata', id='_create', prepend=True)
+    create.label = 'create a new linked form'
+    create.varname = 'resubmitted'
+    create.draft = True
+    create.formdef_slug = target_formdef.url_name
+    create.attach_to_history = True
+    create.draft_edit_operation_mode = mode
+    create.page_identifier = 'page2'
+    create.mappings = [
+        Mapping(field_id='2', expression='blah1'),
+        Mapping(field_id='4', expression='blah2'),
+    ]
+    wf.store()
+
+    source_formdef = FormDef()
+    source_formdef.name = 'source-form'
+    source_formdef.fields = []
+    source_formdef.workflow_id = wf.id
+    source_formdef.enable_tracking_codes = True
+    source_formdef.store()
+
+    source_formdef.data_class().wipe()
+    target_formdef.data_class().wipe()
+
+    create_user(pub)
+    app = login(get_app(pub), username='foo', password='foo')
+    resp = app.get('/source-form/')
+    resp = resp.forms[0].submit('submit')  # -> validation
+    resp = resp.forms[0].submit('submit').follow()  # -> submit
+    assert 'The form has been recorded' in resp.text
+
+    created_url = resp.pyquery('.wf-links a')[0].attrib['href']
+    resp = app.get(created_url).follow()
+
+    if mode == 'single':
+        assert resp.pyquery('.wcs-step').length == 2
+    else:
+        assert resp.pyquery('.wcs-step').length == 3
+    assert resp.pyquery('.wcs-step.current .label').text() == 'page2 (current step)'
+    assert resp.forms[1]['f4'].value == 'blah2'
+
+    if mode == 'partial':
+        resp = resp.forms[1].submit('submit')  # -> page 3
+        assert resp.pyquery('.wcs-step.current .label').text() == 'page3 (current step)'
+
+    resp = resp.forms[1].submit('submit')  # -> validation
+    resp = resp.forms[1].submit('submit')  # -> submit
+    assert target_formdef.data_class().count() == 1
+    formdata = target_formdef.data_class().select()[0]
+    assert formdata.data == {'2': 'blah1', '4': 'blah2'}

tests/workflow/test_email.py

@@ -18,7 +18,8 @@ from wcs.wf.backoffice_fields import SetBackofficeFieldsWorkflowStatusItem
 from wcs.wf.sendmail import SendmailWorkflowStatusItem
 from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef
 
-from ..utilities import MockSubstitutionVariables, clean_temporary_pub, create_temporary_pub
+from ..admin_pages.test_all import create_superuser
+from ..utilities import MockSubstitutionVariables, clean_temporary_pub, create_temporary_pub, get_app, login
 
 
 def setup_module(module):
@@ -33,6 +34,7 @@ def teardown_module(module):
 def pub(request):
     pub = create_temporary_pub()
     pub.cfg['language'] = {'language': 'en'}
+    pub.cfg['identification'] = {'methods': ['password']}
     pub.write_cfg()
     req = HTTPRequest(None, {'SERVER_NAME': 'example.net', 'SCRIPT_NAME': ''})
     req.response.filter = {}
@@ -748,3 +750,26 @@ def test_email_invalid_recipients(pub, req):
         if req:
             get_response().process_after_jobs()
         assert send_email_job.call_count == 0
+
+
+def test_workflows_edit_sendmail_action(pub):
+    create_superuser(pub)
+    Workflow.wipe()
+    workflow = Workflow(name='foo')
+    st1 = workflow.add_status(name='baz')
+    workflow.store()
+
+    app = login(get_app(pub))
+    resp = app.get(st1.get_admin_url())
+
+    resp.forms[0]['action-interaction'] = 'Email'
+    resp = resp.forms[0].submit()
+    resp = resp.follow()
+
+    resp = resp.click('Email')
+    resp.form['to$element0$choice'] = '__other'
+    resp.form['to$element0$other$value_template'] = '{{ test }}'
+    resp.form.submit('submit')
+
+    workflow.refresh_from_storage()
+    assert workflow.possible_status[0].items[0].to == ['{{ test }}']

tests/workflow/test_jump.py

@@ -158,8 +158,16 @@ def test_jump_bad_python_condition(pub):
     assert logged_error.summary == 'Failed to evaluate condition'
     assert logged_error.exception_class == 'NameError'
     assert logged_error.exception_message == "name 'form_var_foobar' is not defined"
-    assert logged_error.expression == 'form_var_foobar == 0'
-    assert logged_error.expression_type == 'python'
+    assert logged_error.context == {
+        'stack': [
+            {
+                'condition': 'form_var_foobar == 0',
+                'condition_type': 'python',
+                'source_label': 'Automatic Jump',
+                'source_url': '',
+            }
+        ]
+    }
 
     pub.loggederror_class.wipe()
     item.condition = {'type': 'python', 'value': '~ invalid ~'}
@@ -169,8 +177,16 @@ def test_jump_bad_python_condition(pub):
     assert logged_error.summary == 'Failed to evaluate condition'
     assert logged_error.exception_class == 'SyntaxError'
     assert logged_error.exception_message == 'invalid syntax (<string>, line 1)'
-    assert logged_error.expression == '~ invalid ~'
-    assert logged_error.expression_type == 'python'
+    assert logged_error.context == {
+        'stack': [
+            {
+                'condition': '~ invalid ~',
+                'source_url': '',
+                'source_label': 'Automatic Jump',
+                'condition_type': 'python',
+            }
+        ]
+    }
 
 
 def test_jump_django_conditions(pub):
@@ -207,8 +223,16 @@ def test_jump_django_conditions(pub):
     assert logged_error.summary == 'Failed to evaluate condition'
     assert logged_error.exception_class == 'TemplateSyntaxError'
     assert logged_error.exception_message == "Could not parse the remainder: '~' from '~'"
-    assert logged_error.expression == '~ invalid ~'
-    assert logged_error.expression_type == 'django'
+    assert logged_error.context == {
+        'stack': [
+            {
+                'condition': '~ invalid ~',
+                'source_url': '',
+                'source_label': 'Automatic Jump',
+                'condition_type': 'django',
+            }
+        ]
+    }
 
 
 def test_timeout(pub):

tox.ini

@@ -12,6 +12,7 @@ setenv =
     LC_ALL=C
     LC_TIME=C
     LANG=C
+    JOB_NAME={env:JOB_NAME:}
     coverage: COVERAGE=--cov-report xml --cov-report html --cov=wcs/ --cov-config .coveragerc -v
 passenv =
     USER

wcs/admin/blocks.py

@@ -54,6 +54,7 @@ class BlockDirectory(FieldsDirectory):
         'inspect',
         'duplicate',
         ('history', 'snapshots_dir'),
+        'overwrite',
     ]
     field_def_page_class = BlockFieldDefPage
     blacklisted_types = ['page', 'table', 'table-select', 'tablerows', 'ranked-items', 'blocks', 'computed']
@@ -106,6 +107,14 @@ class BlockDirectory(FieldsDirectory):
         r = TemplateIO(html=True)
         r += htmltext('<div id="appbar">')
         r += htmltext('<h2>%s</h2>') % self.objectdef.name
+        r += htmltext('<span class="actions">')
+        r += htmltext('<a class="extra-actions-menu-opener"></a>')
+        r += htmltext('<ul class="extra-actions-menu">')
+        r += htmltext('<li><a href="export">%s</a></li>') % _('Export')
+        r += htmltext('<li><a href="delete" rel="popup">%s</a></li>') % _('Delete')
+        r += htmltext('</ul>')
+        r += htmltext('<a href="settings" rel="popup" role="button">%s</a>') % _('Settings')
+        r += htmltext('</span>')
         r += htmltext('</div>')
         r += utils.last_modification_block(obj=self.objectdef)
         r += get_session().display_message()
@@ -132,17 +141,25 @@ class BlockDirectory(FieldsDirectory):
 
     def get_new_field_form_sidebar(self, page_id):
         r = TemplateIO(html=True)
-        r += htmltext('<ul id="sidebar-actions">')
-        r += htmltext('<li><a href="delete" rel="popup">%s</a></li>') % _('Delete')
-        r += htmltext('<li><a href="duplicate" rel="popup">%s</a></li>') % _('Duplicate')
-        r += htmltext('<li><a href="export">%s</a></li>') % _('Export')
-        if get_publisher().snapshot_class:
-            r += htmltext('<li><a rel="popup" href="history/save">%s</a></li>') % _('Save snapshot')
-            r += htmltext('<li><a href="history/">%s</a></li>') % _('History')
-        r += htmltext('<li><a href="inspect">%s</a></li>') % _('Inspector')
-        r += htmltext('<li><a href="settings" rel="popup">%s</a></li>') % _('Settings')
-        r += htmltext('</ul>')
         r += super().get_new_field_form_sidebar(page_id=page_id)
+        r += htmltext('<h3>%s</h3>') % _('Actions')
+        r += htmltext('<ul class="sidebar--buttons">')
+        r += htmltext('<li><a class="button button-paragraph" href="duplicate" rel="popup">%s</a>') % _(
+            'Duplicate'
+        )
+        if get_publisher().snapshot_class:
+            r += htmltext('<li><a class="button button-paragraph" href="history/save">%s</a>') % _(
+                'Save snapshot'
+            )
+        r += htmltext('<li><a class="button button-paragraph" rel="popup" href="overwrite">%s</a>') % _(
+            'Overwrite'
+        )
+        r += htmltext('</ul>')
+        r += htmltext('<h3>%s</h3>') % _('Navigation')
+        r += htmltext('<ul class="sidebar--buttons">')
+        r += htmltext('<li><a class="button button-paragraph" href="history/">%s</a></li>') % _('History')
+        r += htmltext('<li><a class="button button-paragraph" href="inspect">%s</a></li>') % _('Inspector')
+        r += htmltext('</ul>')
         return r.getvalue()
 
     def delete(self):
@@ -209,6 +226,41 @@ class BlockDirectory(FieldsDirectory):
             content_type='application/x-wcs-form',
         )
 
+    def overwrite(self):
+        form = Form(enctype='multipart/form-data')
+        form.widgets.append(
+            HtmlWidget(
+                '<div class="warningnotice"><p>%s</p></div>'
+                % _('Field data will be lost if overwriting with an incompatible block.')
+            )
+        )
+        form.add(FileWidget, 'file', title=_('File'), required=True)
+        form.add_submit('submit', _('Submit'))
+        form.add_submit('cancel', _('Cancel'))
+        if form.get_widget('cancel').parse():
+            return redirect('.')
+
+        if form.is_submitted() and not form.has_errors():
+            try:
+                return self.overwrite_submit(form)
+            except ValueError:
+                pass
+
+        get_response().breadcrumb.append(('overwrite', _('Overwrite')))
+        get_response().set_title(title=_('Overwrite'))
+        r = TemplateIO(html=True)
+        r += htmltext('<h2>%s</h2>') % _('Overwrite')
+        r += form.render()
+        return r.getvalue()
+
+    def overwrite_submit(self, form):
+        blockdef = BlocksDirectory.import_blockdef(form)
+        self.objectdef.name = blockdef.name
+        self.objectdef.digest_template = blockdef.digest_template
+        self.objectdef.fields = blockdef.fields
+        self.objectdef.store(comment=_('Overwritten'))
+        return redirect('.')
+
     def settings(self):
         get_response().breadcrumb.append(('settings', _('Settings')))
         form = Form()
@@ -409,7 +461,8 @@ class BlocksDirectory(Directory):
         r += form.render()
         return r.getvalue()
 
-    def import_submit(self, form):
+    @classmethod
+    def import_blockdef(cls, form):
         fp = form.get_widget('file').parse().fp
 
         error, reason = False, None
@@ -433,6 +486,10 @@ class BlocksDirectory(Directory):
             form.set_error('file', msg)
             raise ValueError()
 
+        return blockdef
+
+    def import_submit(self, form):
+        blockdef = self.import_blockdef(form)
         initial_blockdef_name = blockdef.name
         blockdef_names = [x.name for x in BlockDef.select()]
         copy_no = 1

wcs/admin/fields.py

@@ -527,8 +527,8 @@ class FieldsDirectory(Directory):
                 )
                 return r.getvalue()
 
-        r += htmltext('<div id="new-field">')
         r += htmltext('<h3>%s</h3>') % _('New Field')
+        r += htmltext('<div id="new-field">')
         get_request().form = None  # ignore the eventual ?page=x
         form = self.get_new_field_form(page_id)
         r += form.render()

wcs/admin/forms.py

@@ -332,10 +332,12 @@ class OptionsDirectory(Directory):
     def management(self):
         form = Form(enctype='multipart/form-data')
         form.add(
-            CheckboxWidget,
-            'include_download_all_button',
-            title=_('Include button to download all files'),
-            value=self.formdef.include_download_all_button,
+            CheckboxesWidget,
+            'management_sidebar_items',
+            title=_('Sidebar elements'),
+            options=[(x[0], x[1], x[0]) for x in self.formdef.get_management_sidebar_available_items()],
+            value=self.formdef.get_management_sidebar_items(),
+            inline=False,
         )
         form.add(
             CheckboxWidget,
@@ -494,6 +496,7 @@ class OptionsDirectory(Directory):
                 'submission_lateral_template',
                 'drafts_lifespan',
                 'user_support',
+                'management_sidebar_items',
             ]
             for attr in attrs:
                 widget = form.get_widget(attr)
@@ -503,6 +506,10 @@ class OptionsDirectory(Directory):
                         if has_error:
                             continue
                     new_value = widget.parse()
+                    if attr == 'management_sidebar_items':
+                        new_value = set(new_value)
+                        if new_value == self.formdef.__class__.management_sidebar_items:
+                            new_value = {'__default__'}
                     if attr == 'digest_template':
                         if self.formdef.default_digest_template != new_value:
                             self.changed = True
@@ -777,7 +784,10 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
                 'options/management',
                 _('Management'),
                 _('Custom')
-                if (self.formdef.skip_from_360_view or self.formdef.include_download_all_button)
+                if (
+                    self.formdef.skip_from_360_view
+                    or self.formdef.management_sidebar_items != {'__default__'}
+                )
                 else _('Default'),
             ),
             'tracking_code': self.add_option_line(

wcs/admin/logged_errors.py

@@ -28,6 +28,28 @@ from wcs.qommon.form import CheckboxesWidget, DateWidget, Form
 from wcs.sql_criterias import Equal, Less, NotEqual, NotNull, Null, Or
 
 
+class ErrorFrame:
+    def __init__(self, context):
+        self.context = context or {}
+
+    def source(self):
+        if self.context.get('source_url'):
+            return {
+                'url': self.context.get('source_url'),
+                'label': self.context.get('source_label'),
+            }
+        return None
+
+    def get_frame_lines(self):
+        for key, value in self.context.items():
+            key_label = {
+                'condition': _('Condition'),
+                'condition_type': _('Condition type'),
+            }.get(key)
+            if key_label:
+                yield {'label': key_label, 'value': value}
+
+
 class LoggedErrorDirectory(Directory):
     _q_exports = ['', 'delete', 'ack']
     do_not_call_in_templates = True
@@ -63,6 +85,10 @@ class LoggedErrorDirectory(Directory):
             'text': _('Text'),
         }.get(self.error.expression_type, _('Unknown'))
 
+    def get_context_frames(self):
+        for frame_context in reversed(self.error.context.get('stack') or []):
+            yield ErrorFrame(frame_context)
+
     def get_tabs(self):
         r = TemplateIO(html=True)
         parts = (

wcs/admin/workflows.py

@@ -46,12 +46,12 @@ from wcs.qommon.form import (
     Form,
     HtmlWidget,
     RadiobuttonsWidget,
+    RichTextWidget,
     SingleSelectWidget,
     SlugWidget,
     StringWidget,
     UrlWidget,
     VarnameWidget,
-    WysiwygTextWidget,
 )
 from wcs.sql_criterias import Equal
 from wcs.workflows import (
@@ -991,7 +991,7 @@ class WorkflowStatusPage(Directory):
             value=(self.status.forced_endpoint is True),
         )
         form.add(
-            WysiwygTextWidget,
+            RichTextWidget,
             'backoffice_info_text',
             title=_('Information text for backoffice'),
             value=self.status.backoffice_info_text,
@@ -1590,7 +1590,7 @@ class GlobalActionPage(WorkflowStatusPage):
     def options(self):
         form = Form(enctype='multipart/form-data')
         form.add(
-            WysiwygTextWidget,
+            RichTextWidget,
             'backoffice_info_text',
             title=_('Information text for backoffice'),
             value=self.action.backoffice_info_text,

wcs/backoffice/management.py

@@ -1147,6 +1147,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
             'submission-agent-id',
             'date',
             'distance',
+            'criticality-level',
         ]
         return types
 
@@ -1169,6 +1170,8 @@ class FormPage(Directory, TempfileDirectoryMixin):
             FakeField('user-function', 'user-function', _('Current User Function')),
             FakeField('submission-agent', 'submission-agent-id', _('Submission Agent'), addable=False),
         ]
+        if self.formdef.workflow.criticality_levels:
+            fake_fields.append(FakeField('criticality-level', 'criticality-level', _('Criticality Level')))
         default_filters = self.get_default_filters(mode)
 
         filter_fields = []
@@ -1376,6 +1379,19 @@ class FormPage(Directory, TempfileDirectoryMixin):
                 )
                 r += render_widget(widget, operators)
 
+            elif filter_field.key == 'criticality-level':
+                options = [('', pgettext_lazy('criticality-level', 'All'), '')] + [
+                    (str(i), x.name, str(i)) for i, x in enumerate(self.formdef.workflow.criticality_levels)
+                ]
+                widget = SingleSelectWidget(
+                    filter_field_key,
+                    title=filter_field.label,
+                    options=options,
+                    value=filter_field_value,
+                    render_br=False,
+                )
+                r += render_widget(widget, operators=[])
+
             elif filter_field.key in ('item', 'items'):
                 filter_field.required = False
 
@@ -1457,18 +1473,12 @@ class FormPage(Directory, TempfileDirectoryMixin):
                 )
                 r += render_widget(widget, operators)
 
-            elif filter_field.key in ('string', 'text', 'email', 'numeric'):
+            elif filter_field.key in ('string', 'text', 'email', 'numeric', 'date'):
                 widget = StringWidget(
                     filter_field_key, title=filter_field.label, value=filter_field_value, render_br=False
                 )
                 r += render_widget(widget, operators)
 
-            elif filter_field.key == 'date':
-                widget = DateWidget(
-                    filter_field_key, title=filter_field.label, value=filter_field_value, render_br=False
-                )
-                r += render_widget(widget, operators)
-
         # field filter dialog content
         r += htmltext('<div style="display: none;">')
         r += htmltext('<ul id="field-filter" class="objects-list">')
@@ -2002,6 +2012,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
             FakeField('user-function', 'user-function', _('Current User Function')),
             FakeField('submission-agent', 'submission-agent-id', _('Submission Agent')),
             FakeField('distance', 'distance', _('Distance')),
+            FakeField('criticality-level', 'criticality-level', _('Criticality Level')),
         ]
         criterias = []
 
@@ -2066,6 +2077,10 @@ class FormPage(Directory, TempfileDirectoryMixin):
             if filter_field.key == 'distance' and filters_dict.get('filter-distance'):
                 filters_dict['filter-distance-value'] = filters_dict['filter-distance']
 
+            if filter_field.key == 'criticality-level' and filters_dict.get('filter-criticality-level'):
+                if filters_dict['filter-criticality-level'] != 'on':
+                    filters_dict['filter-criticality-level-value'] = filters_dict['filter-criticality-level']
+
             if filter_field.key == 'user-id' and not filters_dict.get('filter-user-function'):
                 # convert uuid based filter into local id filter.
                 # do not apply if there's filter-user-function as it indicates the filtering
@@ -2218,7 +2233,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
                     filter_field_value = False
                 else:
                     raise RequestError('Invalid value "%s" for "%s"' % (filter_field_value, filter_field_key))
-            elif filter_field.key in ('item', 'items', 'string', 'email', 'numeric'):
+            elif filter_field.key in ('item', 'items', 'string', 'email', 'numeric', 'date'):
                 if Template.is_template_string(filter_field_value, ezt_support=False):
                     if keep_templates:
                         # use Equal criteria here, the only use is in CardDef.get_data_source_referenced_varnames
@@ -2306,6 +2321,9 @@ class FormPage(Directory, TempfileDirectoryMixin):
                     raise RequestError('Distance filter missing a center')
                 center = misc.normalize_geolocation({'lat': center_lat, 'lon': center_lon})
                 criterias.append(Distance(center, float(filter_field_value)))
+            elif filter_field.key == 'criticality-level':
+                level = 100 + int(filter_field_value)
+                criterias.append(Equal('criticality_level', level))
             elif filter_field.key in ('item', 'items', 'bool', 'string', 'text', 'email', 'date', 'numeric'):
                 criterias.append(
                     lazy_manager.get_criteria_from_operator(
@@ -3419,6 +3437,7 @@ class FormBackOfficeStatusPage(FormStatusPage):
 
     def get_extra_context_bar(self, parent=None):
         formdata = self.filled
+        management_sidebar_items = self.formdef.get_management_sidebar_items()
 
         r = TemplateIO(html=True)
 
@@ -3426,7 +3445,6 @@ class FormBackOfficeStatusPage(FormStatusPage):
             r += htmltext('<p><a class="button" id="back-to-listing" href="..">%s</a></p>') % _(
                 'Back to Listing'
             )
-            r += htmltext('<div class="extra-context">')
             if (
                 formdata.backoffice_submission
                 and formdata.submission_agent_id == str(get_request().user.id)
@@ -3435,9 +3453,13 @@ class FormBackOfficeStatusPage(FormStatusPage):
             ):
                 # keep displaying tracking code to submission agent for 30
                 # minutes after submission
+                r += htmltext('<div class="extra-context">')
                 r += htmltext('<h3>%s</h3>') % _('Tracking Code')
                 r += htmltext('<p>%s</p>') % formdata.tracking_code
+                r += htmltext('</div>')
 
+        if not formdata.is_draft() and 'general' in management_sidebar_items:
+            r += htmltext('<div class="extra-context sidebar-general-information">')
             r += htmltext('<h3>%s</h3>') % _('General Information')
             r += htmltext('<p>')
             tm = misc.localstrftime(formdata.receipt_time)
@@ -3489,43 +3511,54 @@ class FormBackOfficeStatusPage(FormStatusPage):
                     'date': formdata.anonymised.strftime(misc.date_format())
                 }
                 r += htmltext('</div>')
+            r += htmltext('</div>')  # .extra-context
 
-            if formdata.formdef.include_download_all_button:
-                has_attached_files = False
-                for value in (formdata.data or {}).values():
-                    if isinstance(value, PicklableUpload):
-                        has_attached_files = True
-                    if isinstance(value, dict) and isinstance(value.get('data'), list):
-                        # block fields
-                        for subvalue in value.get('data'):
-                            for subvalue_elem in subvalue.values():
-                                if isinstance(subvalue_elem, PicklableUpload):
-                                    has_attached_files = True
-                                    break
-                    if has_attached_files:
-                        break
-
+        if not formdata.is_draft() and 'download-files' in management_sidebar_items:
+            has_attached_files = False
+            for value in (formdata.data or {}).values():
+                if isinstance(value, PicklableUpload):
+                    has_attached_files = True
+                if isinstance(value, dict) and isinstance(value.get('data'), list):
+                    # block fields
+                    for subvalue in value.get('data'):
+                        for subvalue_elem in subvalue.values():
+                            if isinstance(subvalue_elem, PicklableUpload):
+                                has_attached_files = True
+                                break
                 if has_attached_files:
-                    r += htmltext('<p><a class="button" href="download-as-zip">%s</a></p>') % _(
-                        'Download all files as .zip'
-                    )
+                    break
 
-            r += htmltext('</div>')
+            if has_attached_files:
+                r += htmltext('<div class="extra-context sidebar-download-files">')
+                r += htmltext('<p><a class="button" href="download-as-zip">%s</a></p>') % _(
+                    'Download all files as .zip'
+                )
+                r += htmltext('</div>')
 
-        r += self.get_extra_submission_context_bar()
-        r += self.get_extra_submission_channel_bar()
-        r += self.get_extra_submission_user_id_bar(parent=parent)
-        r += self.get_extra_geolocation_bar()
-        if formdata.formdef.lateral_template:
+        if 'submission-context' in management_sidebar_items:
+            r += self.get_extra_submission_context_bar()
+            r += self.get_extra_submission_channel_bar()
+
+        if 'user' in management_sidebar_items:
+            r += self.get_extra_submission_user_id_bar(parent=parent)
+
+        if 'geolocation' in management_sidebar_items:
+            r += self.get_extra_geolocation_bar()
+
+        if 'custom-template' in management_sidebar_items and formdata.formdef.lateral_template:
             r += htmltext('<div data-async-url="%slateral-block"></div>' % formdata.get_url(backoffice=True))
 
-        if not isinstance(formdata.formdef, CardDef) and formdata.user_id:
+        if (
+            'pending-forms' in management_sidebar_items
+            and not isinstance(formdata.formdef, CardDef)
+            and formdata.user_id
+        ):
             r += htmltext(
                 '<div data-async-url="%suser-pending-forms"></div>' % formdata.get_url(backoffice=True)
             )
 
         if not formdata.is_draft() and self.can_go_in_inspector():
-            r += htmltext('<div class="extra-context">')
+            r += htmltext('<div class="extra-context sidebar-data-inspector">')
             r += htmltext('<p><a href="%sinspect">' % formdata.get_url(backoffice=True))
             r += htmltext('%s</a></p>') % _('Data Inspector')
             r += htmltext('</div>')
@@ -3537,7 +3570,7 @@ class FormBackOfficeStatusPage(FormStatusPage):
         r = TemplateIO(html=True)
         if formdata.submission_context or formdata.submission_channel:
             extra_context = formdata.submission_context or {}
-            r += htmltext('<div class="extra-context">')
+            r += htmltext('<div class="extra-context sidebar-submission-context">')
             if extra_context.get('orig_formdef_id'):
                 object_type = extra_context.get('orig_object_type', 'formdef')
                 if object_type == 'formdef':
@@ -3580,7 +3613,7 @@ class FormBackOfficeStatusPage(FormStatusPage):
                 r += htmltext('<p>%s</p>') % extra_context.get('comments')
             if extra_context.get('summary_url'):
                 r += htmltext('<div data-content-url="%s"></div>' % (extra_context.get('summary_url')))
-            r += htmltext('</div>')
+            r += htmltext('</div>')  # closes .extra-context from get_extra_submission_context_bar
 
         return r.getvalue()
 
@@ -3588,7 +3621,7 @@ class FormBackOfficeStatusPage(FormStatusPage):
         formdata = self.filled
         r = TemplateIO(html=True)
         if formdata and formdata.user_id and formdata.get_user():
-            r += htmltext('<div class="extra-context">')
+            r += htmltext('<div class="extra-context sidebar--user">')
             r += htmltext('<h3>%s</h3>') % _('Associated User')
             users_cfg = get_cfg('users', {})
             sidebar_user_template = users_cfg.get('sidebar_template')
@@ -3625,7 +3658,7 @@ class FormBackOfficeStatusPage(FormStatusPage):
         formdata = self.filled
         r = TemplateIO(html=True)
         if formdata.formdef.geolocations and formdata.geolocations:
-            r += htmltext('<div class="geolocations">')
+            r += htmltext('<div class="extra-context geolocations sidebar-geolocations">')
             for geoloc_key in formdata.formdef.geolocations:
                 if geoloc_key not in formdata.geolocations:
                     continue
@@ -3684,7 +3717,7 @@ class FormBackOfficeStatusPage(FormStatusPage):
         self.filled.related_user_forms = formdatas
 
         if formdatas:
-            r += htmltext('<div class="user-pending-forms">')
+            r += htmltext('<div class="extra-context user-pending-forms">')
             r += htmltext('<h3>%s</h3>') % _('User Pending Forms')
             categories = {}
             formdata_by_category = {}

wcs/backoffice/root.py

@@ -81,6 +81,7 @@ class RootDirectory(AccessControlled, Directory):
             except KeyError:
                 pass
         get_response().add_javascript(['jquery.js', 'qommon.js', 'gadjo.js'])
+        get_response().add_css_include('../xstatic/css/godo.css')
         if path and path[0] == 'categories':
             # legacy /backoffice/categories/<...>, redirect.
             return redirect('/backoffice/forms/' + '/'.join(path))

wcs/carddata.py

@@ -14,31 +14,18 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
-from quixote import get_publisher, get_request, get_session
+from quixote import get_publisher, get_request, get_response, get_session
 
 from wcs.formdata import FormData
 
 from .qommon import _
-from .sql_criterias import Equal
+from .qommon.afterjobs import AfterJob
+from .sql_criterias import Equal, Null, Or, get_field_id
 
 
 class CardData(FormData):
     uuid = None
 
-    def get_formdef(self):
-        if self._formdef:
-            return self._formdef
-        from .carddef import CardDef
-
-        id = self._names.split('-', 1)[1]
-        try:
-            self._formdef = CardDef.get_by_urlname(id)
-        except KeyError:
-            self._formdef = None
-        return self._formdef
-
-    formdef = property(get_formdef)
-
     def get_data_source_structured_item(
         self, digest_key='default', group_by=None, with_related_urls=False, with_files_urls=False
     ):
@@ -133,3 +120,98 @@ class CardData(FormData):
         }
         token = get_session().create_token('card-file-by-token', context)
         return '/api/card-file-by-token/%s' % token.id
+
+    def update_related(self):
+        if self.formdef.reverse_relations:
+            job = UpdateRelationsAfterJob(carddata=self)
+            if get_response():
+                job.store()
+                get_response().add_after_job(job)
+            else:
+                job.execute()
+        self._has_changed_digest = False
+
+
+class UpdateRelationsAfterJob(AfterJob):
+    label = _('Updating relations')
+
+    def __init__(self, carddata):
+        super().__init__(carddef_id=carddata.formdef.id, carddata_id=carddata.id)
+
+    def execute(self):
+        from .carddef import CardDef
+        from .formdef import FormDef
+
+        if getattr(get_publisher(), '_update_related_seen', None) is None:
+            get_publisher()._update_related_seen = set()
+
+        # keep track of objects that have been updated, to avoid cycles
+        update_related_seen = get_publisher()._update_related_seen
+
+        try:
+            carddef = CardDef.get(self.kwargs['carddef_id'])
+            carddata = carddef.data_class().get(self.kwargs['carddata_id'])
+        except KeyError:
+            # card got removed (probably the afterjob met some unexpected delay), ignore.
+            return
+
+        klass = {'carddef': CardDef, 'formdef': FormDef}
+
+        # check all known reverse relations
+        for obj_ref in {x['obj'] for x in carddef.reverse_relations}:
+            obj_type, obj_slug = obj_ref.split(':')
+            obj_class = klass.get(obj_type)
+            try:
+                objdef = obj_class.get_by_slug(obj_slug)
+            except KeyError:
+                continue
+            criterias = []
+            fields = []
+
+            # get fields referencing the card model (only item and items fields, as string
+            # field with data source is just for completion, and computed field with data
+            # source, do not store a display value.
+            for field in objdef.iter_fields(include_block_fields=True):
+                if field.key not in ('item', 'items'):
+                    continue
+                data_source = getattr(field, 'data_source', None)
+                if not data_source:
+                    continue
+                data_source_type = data_source.get('type')
+                if (
+                    not data_source_type.startswith('carddef:')
+                    or data_source_type.split(':')[1] != carddef.slug
+                ):
+                    continue
+                fields.append(field)
+                criterias.append(Equal(get_field_id(field), carddata.identifier, field=field))
+            if not criterias:
+                continue
+
+            def update_data(field, data):
+                display_value = data.get(f'{field.id}_display')
+                field.set_value(data, data.get(field.id))
+                return bool(data.get(f'{field.id}_display') != display_value)
+
+            # look for all formdata, including drafts, excluding anonymised
+            select_criterias = [Null('anonymised'), Or(criterias)]
+            for objdata in objdef.data_class().select_iterator(clause=select_criterias, itersize=200):
+                objdata_seen_key = f'{objdata.formdef.xml_root_node}:{objdata.formdef.slug}:{objdata.id}'
+                if objdata_seen_key in update_related_seen:
+                    # do not allow updates to cycle back
+                    continue
+                objdata_changed = False
+                for field in fields:
+                    if getattr(field, 'block_field', None):
+                        blockdata_changed = False
+                        for block_row_data in objdata.data[field.block_field.id]['data']:
+                            blockdata_changed |= update_data(field, block_row_data)
+                        if blockdata_changed:
+                            # if block data changed, maybe block digest changed too
+                            update_data(field.block_field, objdata.data)
+                        objdata_changed |= blockdata_changed
+                    else:
+                        objdata_changed |= update_data(field, objdata.data)
+                if objdata_changed:
+                    update_related_seen.add(objdata_seen_key)
+                    objdata.store()

wcs/conditions.py

@@ -49,21 +49,22 @@ class Condition:
         local_variables = self.get_data()
         return getattr(self, 'evaluate_' + self.type)(local_variables)
 
-    def evaluate(self):
-        try:
-            return self.unsafe_evaluate()
-        except Exception as e:
-            if self.record_errors:
-                summary = _('Failed to evaluate condition')
-                get_publisher().record_error(
-                    summary,
-                    formdata=self.context.get('formdata'),
-                    status_item=self.context.get('status_item'),
-                    expression=self.value,
-                    expression_type=self.type,
-                    exception=e,
-                )
-            raise RuntimeError()
+    def evaluate(self, source_label=None, source_url=None):
+        with get_publisher().error_context(
+            condition=self.value, condition_type=self.type, source_label=source_label, source_url=source_url
+        ):
+            try:
+                return self.unsafe_evaluate()
+            except Exception as e:
+                if self.record_errors:
+                    summary = _('Failed to evaluate condition')
+                    get_publisher().record_error(
+                        summary,
+                        formdata=self.context.get('formdata'),
+                        status_item=self.context.get('status_item'),
+                        exception=e,
+                    )
+                raise RuntimeError()
 
     def evaluate_python(self, local_variables):
         global_variables = get_publisher().get_global_eval_dict()

wcs/ctl/management/commands/__init__.py

@@ -64,6 +64,7 @@ class TenantCommand(BaseCommand):
         except UnknownTenantError:
             raise CommandError('unknown tenant')
         publisher.install_lang()
+        publisher.setup_timezone()
         publisher.substitutions.feed(publisher)
         return publisher
 

wcs/fields/base.py

@@ -15,6 +15,7 @@
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
 import collections
+import copy
 import datetime
 import html
 import re
@@ -236,6 +237,11 @@ class Field:
         for k, v in kwargs.items():
             setattr(self, k.replace('-', '_'), v)
 
+    def __getstate__(self):
+        odict = copy.copy(self.__dict__)
+        odict.pop('_formdef', None)
+        return odict
+
     @classmethod
     def init(cls):
         pass
@@ -243,6 +249,11 @@ class Field:
     def get_type_label(self):
         return self.description
 
+    def get_admin_url(self):
+        if not getattr(self, '_formdef', None):
+            return ''
+        return self._formdef.get_field_admin_url(field=self)
+
     @property
     def include_in_listing(self):
         return 'listings' in (self.display_locations or [])
@@ -584,16 +595,24 @@ class Field:
         return changed
 
     @staticmethod
-    def evaluate_condition(dict_vars, formdef, condition, record_errors=True):
+    def evaluate_condition(
+        dict_vars, formdef, condition, source_label=None, source_url=None, record_errors=True
+    ):
         from .page import PageCondition
 
-        return PageCondition(
-            condition, {'dict_vars': dict_vars, 'formdef': formdef}, record_errors
-        ).evaluate()
+        return PageCondition(condition, {'dict_vars': dict_vars, 'formdef': formdef}, record_errors).evaluate(
+            source_label=source_label, source_url=source_url
+        )
 
     def is_visible(self, dict, formdef):
         try:
-            return self.evaluate_condition(dict, formdef, self.condition)
+            return self.evaluate_condition(
+                dict,
+                formdef,
+                self.condition,
+                source_label=_('Field: %s') % self.ellipsized_label,
+                source_url=self.get_admin_url(),
+            )
         except RuntimeError:
             return True
 

wcs/fields/block.py

@@ -342,9 +342,8 @@ class BlockField(WidgetField):
 
     def __getstate__(self):
         # do not store _block cache
-        odict = copy.copy(self.__dict__)
-        if '_block' in odict:
-            del odict['_block']
+        odict = super().__getstate__()
+        odict.pop('_block', None)
         return odict
 
     def __setstate__(self, ndict):

wcs/formdata.py

@@ -310,15 +310,7 @@ class FormData(StorableObject):
     _formdef = None
 
     def get_formdef(self):
-        if self._formdef:
-            return self._formdef
-        from .formdef import FormDef
-
-        id = self._names.split('-', 1)[1]
-        try:
-            self._formdef = FormDef.get_by_urlname(id)
-        except KeyError:
-            self._formdef = None
+        assert self._formdef
         return self._formdef
 
     formdef = property(get_formdef)

wcs/formdef.py

@@ -174,6 +174,14 @@ class FormDef(StorableObject):
     expiration_date = None
     has_captcha = False
     skip_from_360_view = False
+    management_sidebar_items = {
+        'general',
+        'submission-context',
+        'user',
+        'geolocation',
+        'custom-template',
+        'pending-forms',
+    }
     include_download_all_button = False
     appearance_keywords = None
     digest_templates = None
@@ -221,7 +229,6 @@ class FormDef(StorableObject):
         'enable_tracking_codes',
         'confirmation',
         'always_advertise',
-        'include_download_all_button',
         'has_captcha',
         'skip_from_360_view',
     ]
@@ -231,6 +238,7 @@ class FormDef(StorableObject):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.fields = []
+        self.management_sidebar_items = {'__default__'}
 
     def __eq__(self, other):
         return bool(
@@ -267,6 +275,12 @@ class FormDef(StorableObject):
                     changed = True
                     break
 
+        if self.include_download_all_button:  # 2023-12-30
+            self.management_sidebar_items = self.__class__.management_sidebar_items.copy()
+            self.management_sidebar_items.add('download-files')
+            self.include_download_all_button = False
+            changed = True
+
         for f in self.fields or []:
             changed |= f.migrate()
 
@@ -287,6 +301,29 @@ class FormDef(StorableObject):
                 sql.clean_global_views(conn, cur)
             cur.close()
 
+    def get_management_sidebar_available_items(self):
+        return [
+            ('general', _('General Information')),
+            ('download-files', _('Button to download all files')),
+            ('submission-context', _('Submission context')),
+            ('user', _('Associated User')),
+            ('geolocation', _('Geolocation')),
+            ('custom-template', _('Custom template')),
+            ('pending-forms', _('User Pending Forms')),
+        ]
+
+    def management_sidebar_items_labels(self):
+        # return ordered labels
+        management_sidebar_items = self.get_management_sidebar_items()
+        for key, label in self.get_management_sidebar_available_items():
+            if key in management_sidebar_items:
+                yield label
+
+    def get_management_sidebar_items(self):
+        if self.management_sidebar_items == {'__default__'}:
+            return self.__class__.management_sidebar_items
+        return self.management_sidebar_items or []
+
     @property
     def data_class_name(self):
         return '_wcs_%s' % self.url_name.title()
@@ -1080,6 +1117,9 @@ class FormDef(StorableObject):
         if self.required_authentication_contexts:
             root['required_authentication_contexts'] = self.required_authentication_contexts[:]
 
+        if self.management_sidebar_items:
+            root['management_sidebar_items'] = sorted(self.management_sidebar_items)
+
         if isinstance(self, CardDef):
             all_carddefs = CardDef.select(ignore_errors=True)
             all_carddefs = [c for c in all_carddefs if c]
@@ -1247,6 +1287,9 @@ class FormDef(StorableObject):
                 str(x) for x in value.get('required_authentication_contexts')
             ]
 
+        if value.get('management_sidebar_items'):
+            formdef.management_sidebar_items = {str(x) for x in value.get('management_sidebar_items')}
+
         return formdef
 
     def export_to_xml(self, include_id=False):
@@ -1396,6 +1439,11 @@ class FormDef(StorableObject):
             for auth_context in self.required_authentication_contexts:
                 ET.SubElement(element, 'method').text = force_str(auth_context)
 
+        if self.management_sidebar_items:
+            element = ET.SubElement(root, 'management_sidebar_items')
+            for item in sorted(self.management_sidebar_items):
+                ET.SubElement(element, 'item').text = force_str(item)
+
         if self.digest_templates:
             digest_templates = ET.SubElement(root, 'digest_templates')
             for key, value in self.digest_templates.items():
@@ -1604,6 +1652,12 @@ class FormDef(StorableObject):
             for child in node:
                 formdef.required_authentication_contexts.append(str(child.text))
 
+        if tree.find('management_sidebar_items') is not None:
+            node = tree.find('management_sidebar_items')
+            formdef.management_sidebar_items = set()
+            for child in node:
+                formdef.management_sidebar_items.add(str(child.text))
+
         if tree.find('digest_templates') is not None:
             digest_templates_node = tree.find('digest_templates')
             formdef.digest_templates = {}
@@ -1956,6 +2010,8 @@ class FormDef(StorableObject):
                 o.fields = pickle.load(fd, **PICKLE_KWARGS)
             except EOFError:
                 pass  # old format
+        for field in o.fields or []:
+            field._formdef = o  # keep formdef reference
         return o
 
     @classmethod

wcs/forms/common.py

@@ -77,6 +77,10 @@ class FileDirectory(Directory):
         if component and component not in (file.base_filename, urllib.parse.quote(file.base_filename)):
             raise errors.TraversalError()
 
+        if not hasattr(file, 'has_redirect_url'):
+            # not an appropriate file object
+            raise errors.TraversalError()
+
         if file.has_redirect_url():
             redirect_url = file.get_redirect_url(backoffice=get_request().is_in_backoffice())
             if not redirect_url:

wcs/forms/root.py

@@ -422,6 +422,7 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
             'current_page_index': current_position,
             'current_page_no': current_position,  # legacy, for themes
             'page_labels': page_labels,
+            'pages': self.pages,
         }
 
     def step(self):
@@ -945,9 +946,13 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
             # create a fake FormData with current submission data
             formdata.user = get_request().user
         formdata._formdef = self.formdef
-        if draft_formdata and draft_formdata.submission_context:
-            # restore submission context, this is required to get access to form_parent_* variables
-            formdata.submission_context = draft_formdata.submission_context
+        if draft_formdata:
+            if draft_formdata.submission_context:
+                # restore submission context, this is required to get access to form_parent_* variables
+                formdata.submission_context = draft_formdata.submission_context
+            if draft_formdata.workflow_data:
+                # restore workflow_data, this is used for partial edit
+                formdata.workflow_data = draft_formdata.workflow_data
         formdata.data = session_data
         formdata.prefilling_data = formdata.data.get('prefilling_data', {})
         computed_values = get_session().get_by_magictoken('%s-computed' % magictoken) or {}
@@ -1013,6 +1018,7 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
             # if there's a form with a single page (at all, not as the result of conditions),
             # and no confirmation page, add native quixote CSRF protection.
             form.add(FormTokenWidget, form.TOKEN_NAME)
+        form.add_hidden('previous-page-id', '')
         form.attrs['data-live-url'] = self.formdef.get_url(language=get_publisher().current_language) + 'live'
         form.attrs['data-live-validation-url'] = (
             self.formdef.get_url(language=get_publisher().current_language) + 'live-validation'
@@ -1027,7 +1033,9 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
         return form
 
     def create_view_form(self, *args, **kwargs):
-        return self.formdef.create_view_form(*args, **kwargs)
+        form = self.formdef.create_view_form(*args, **kwargs)
+        form.add_hidden('previous-page-id', '')
+        return form
 
     def check_authentication_context(self):
         if not self.formdef.required_authentication_contexts:
@@ -1070,7 +1078,8 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
     def pages(self):
         if self._pages:
             return self._pages
-        current_data = self.get_transient_formdata().data
+        transient_formdata = self.get_transient_formdata()
+        current_data = transient_formdata.data
 
         pages = [x for x in self.formdef.fields if x.key == 'page']
         has_page_fields = bool(pages)
@@ -1082,12 +1091,27 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
             # submitted data) (#27247).
             hidden_pages = [x for x in pages if not x.is_visible(current_data, self.formdef)]
 
-        if self.edit_mode and self.edit_action and self.edit_action.operation_mode in ('single', 'partial'):
+        if self.edit_mode and self.edit_action:
+            operation_mode = self.edit_action.operation_mode
+            page_identifier = self.edit_action.page_identifier
+        elif (
+            not self.edit_mode
+            and transient_formdata.workflow_data
+            and '_create_formdata_draft_edit' in transient_formdata.workflow_data
+        ):
+            operation_mode = transient_formdata.workflow_data['_create_formdata_draft_edit']['operation_mode']
+            page_identifier = transient_formdata.workflow_data['_create_formdata_draft_edit'][
+                'page_identifier'
+            ]
+        else:
+            operation_mode = 'full'
+
+        if operation_mode in ('single', 'partial'):
             edit_pages = []
             for page in pages:
-                if self.edit_action.page_identifier == page.varname or edit_pages:
+                if page_identifier == page.varname or edit_pages:
                     edit_pages.append(page)
-                    if self.edit_action.operation_mode == 'single':
+                    if operation_mode == 'single':
                         break
             edit_pages = [x for x in edit_pages if x not in hidden_pages]
             if not edit_pages:
@@ -1374,7 +1398,7 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
                     if errored:
                         form.add(HiddenErrorWidget, 'post_condition%d' % i)
                         form.set_error('post_condition%d' % i, 'error')
-                        page_error_messages.append(error_message)
+                        page_error_messages.append(get_publisher().translate(error_message))
 
             honeypot_error = False
             if get_request().form.get('f00'):  # 🍯
@@ -1595,8 +1619,14 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
 
     def previous_page(self, page_no, magictoken):
         try:
-            new_page_no = page_no - 1
-            previous_page = self.pages[new_page_no]
+            previous_page_id = get_request().form.get('previous-page-id')
+            if previous_page_id:
+                new_page_no, previous_page = [
+                    x for x in enumerate(self.pages[:page_no]) if x[1].id == previous_page_id
+                ][0]
+            else:
+                new_page_no = page_no - 1
+                previous_page = self.pages[new_page_no]
         except IndexError:
             new_page_no = 0
             previous_page = self.pages[0]
@@ -2470,14 +2500,42 @@ TextsDirectory.register(
     'form-recorded',
     _('Message when a form has been recorded'),
     category=_('Forms'),
-    default=_('The form has been recorded on {{ form_receipt_datetime }} with the number {{ form_number }}.'),
+    default=_(
+        '''
+The form has been recorded on {{ form_receipt_datetime }} with the number {{ form_number }}.
+{% if form_submission_agent_display_name %}
+It has been submitted for you by {{ form_submission_agent_display_name }}
+{% if form_submission_channel == "phone" %}after a phone call.
+{% elif form_submission_channel == "email" %}after an email.
+{% elif form_submission_channel == "mail" %}after a mail.
+{% elif form_submission_channel == "social-network" %}after a message on a social network.
+{% elif form_submission_channel == "counter" %}after your passage at the counter.
+{% else %}.
+{% endif %}
+{% endif %}
+              '''
+    ),
 )
 
 TextsDirectory.register(
     'form-recorded-allow-one',
     _('Message when a form has been recorded, and the form is set to only allow one per user'),
     category=_('Forms'),
-    default=_('The form has been recorded on {{ form_receipt_datetime }}.'),
+    default=_(
+        '''
+The form has been recorded on {{ form_receipt_datetime }}.
+{% if form_submission_agent_display_name %}
+It has been submitted for you by {{ form_submission_agent_display_name }}
+{% if form_submission_channel == "phone" %}after a phone call.
+{% elif form_submission_channel == "email" %}after an email.
+{% elif form_submission_channel == "mail" %}after a mail.
+{% elif form_submission_channel == "social-network" %}after a message on a social network.
+{% elif form_submission_channel == "counter" %}after your passage at the counter.
+{% else %}.
+{% endif %}
+{% endif %}
+              '''
+    ),
 )
 
 TextsDirectory.register(

wcs/locale/fr/LC_MESSAGES/django.po

@@ -4,8 +4,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: wcs 0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-03-12 09:34+0100\n"
-"PO-Revision-Date: 2024-03-12 09:34+0100\n"
+"POT-Creation-Date: 2024-03-15 07:48+0100\n"
+"PO-Revision-Date: 2024-03-15 07:48+0100\n"
 "Last-Translator: Thomas Noël <tnoel@entrouvert.com>\n"
 "Language-Team: french\n"
 "Language: fr\n"
@@ -174,6 +174,21 @@ msgstr "Ce bloc de champs contient plus de %d champs."
 msgid "Applications"
 msgstr "Applications"
 
+#: admin/blocks.py admin/comment_templates.py admin/data_sources.py
+#: admin/mail_templates.py admin/settings.py admin/wscalls.py
+#: backoffice/i18n.py backoffice/management.py
+#: templates/wcs/backoffice/category.html templates/wcs/backoffice/formdef.html
+#: templates/wcs/backoffice/i18n.html templates/wcs/backoffice/snapshots.html
+#: templates/wcs/backoffice/test_sidebar.html
+#: templates/wcs/backoffice/workflow.html
+msgid "Export"
+msgstr "Exporter"
+
+#: admin/blocks.py admin/settings.py backoffice/root.py
+#: templates/wcs/backoffice/settings/import.html
+msgid "Settings"
+msgstr "Paramètres"
+
 #: admin/blocks.py admin/fields.py
 msgid "There are not yet any fields defined."
 msgstr "Il n’y a pas encore de champs configurés."
@@ -183,6 +198,23 @@ msgstr "Il n’y a pas encore de champs configurés."
 msgid "Usage"
 msgstr "Utilisation"
 
+#: admin/blocks.py templates/wcs/backoffice/blocks.html
+#: templates/wcs/backoffice/cards.html templates/wcs/backoffice/categories.html
+#: templates/wcs/backoffice/comment-templates.html
+#: templates/wcs/backoffice/data-sources.html
+#: templates/wcs/backoffice/formdef.html templates/wcs/backoffice/forms.html
+#: templates/wcs/backoffice/mail-templates.html
+#: templates/wcs/backoffice/snapshots.html
+#: templates/wcs/backoffice/test-webservice-responses.html
+#: templates/wcs/backoffice/test_sidebar.html
+#: templates/wcs/backoffice/tests.html
+#: templates/wcs/backoffice/workflow-global-action.html
+#: templates/wcs/backoffice/workflow.html
+#: templates/wcs/backoffice/workflows.html
+#: templates/wcs/backoffice/wscalls.html
+msgid "Actions"
+msgstr "Actions"
+
 #: admin/blocks.py admin/comment_templates.py admin/data_sources.py
 #: admin/fields.py admin/forms.py admin/mail_templates.py admin/tests.py
 #: admin/workflows.py qommon/admin/menu.py
@@ -194,22 +226,28 @@ msgstr "Utilisation"
 msgid "Duplicate"
 msgstr "Dupliquer"
 
-#: admin/blocks.py admin/comment_templates.py admin/data_sources.py
-#: admin/mail_templates.py admin/settings.py admin/wscalls.py
-#: backoffice/i18n.py backoffice/management.py
-#: templates/wcs/backoffice/category.html templates/wcs/backoffice/formdef.html
-#: templates/wcs/backoffice/i18n.html templates/wcs/backoffice/snapshots.html
-#: templates/wcs/backoffice/test_sidebar.html
-#: templates/wcs/backoffice/workflow.html
-msgid "Export"
-msgstr "Exporter"
-
 #: admin/blocks.py admin/comment_templates.py admin/data_sources.py
 #: admin/mail_templates.py admin/wscalls.py backoffice/snapshots.py
 #: templates/wcs/backoffice/formdef.html templates/wcs/backoffice/workflow.html
 msgid "Save snapshot"
 msgstr "Enregistrer une sauvegarde"
 
+#: admin/blocks.py admin/forms.py
+msgid "Overwrite"
+msgstr "Écraser"
+
+#: admin/blocks.py templates/wcs/backoffice/blocks.html
+#: templates/wcs/backoffice/cards.html templates/wcs/backoffice/category.html
+#: templates/wcs/backoffice/comment-templates.html
+#: templates/wcs/backoffice/data-sources.html
+#: templates/wcs/backoffice/formdef.html templates/wcs/backoffice/forms.html
+#: templates/wcs/backoffice/mail-templates.html
+#: templates/wcs/backoffice/test_sidebar.html
+#: templates/wcs/backoffice/tests.html templates/wcs/backoffice/workflow.html
+#: templates/wcs/backoffice/workflows.html
+msgid "Navigation"
+msgstr "Navigation"
+
 #: admin/blocks.py admin/comment_templates.py admin/data_sources.py
 #: admin/mail_templates.py admin/wscalls.py backoffice/snapshots.py
 #: templates/wcs/backoffice/category.html templates/wcs/backoffice/formdef.html
@@ -223,11 +261,6 @@ msgstr "Historique"
 msgid "Inspector"
 msgstr "Inspecteur"
 
-#: admin/blocks.py admin/settings.py backoffice/root.py
-#: templates/wcs/backoffice/settings/import.html
-msgid "Settings"
-msgstr "Paramètres"
-
 #: admin/blocks.py
 msgid "You are about to irrevocably delete this block."
 msgstr "Vous allez définitivement supprimer ce bloc."
@@ -259,6 +292,23 @@ msgstr "%(name)s (Copie %(no)d)"
 msgid "Duplicate Fields Block"
 msgstr "Dupliquer le bloc de champs"
 
+#: admin/blocks.py
+msgid "Field data will be lost if overwriting with an incompatible block."
+msgstr ""
+"Les données des champs seront perdues si le bloc est écrasé par un bloc "
+"incompatible."
+
+#: admin/blocks.py admin/categories.py admin/comment_templates.py
+#: admin/data_sources.py admin/forms.py admin/mail_templates.py
+#: admin/settings.py admin/tests.py admin/workflows.py admin/wscalls.py
+#: backoffice/data_management.py backoffice/i18n.py wf/export_to_model.py
+msgid "File"
+msgstr "Fichier"
+
+#: admin/blocks.py admin/forms.py
+msgid "Overwritten"
+msgstr "Écrasement"
+
 #: admin/blocks.py
 msgid "The identifier can not be modified as the block is in use."
 msgstr "L’identifiant ne peut pas être modifié car le bloc est utilisé."
@@ -310,13 +360,6 @@ msgstr "Ajouter"
 msgid "New Fields Block"
 msgstr "Nouveau bloc de champs"
 
-#: admin/blocks.py admin/categories.py admin/comment_templates.py
-#: admin/data_sources.py admin/forms.py admin/mail_templates.py
-#: admin/settings.py admin/tests.py admin/workflows.py admin/wscalls.py
-#: backoffice/data_management.py backoffice/i18n.py wf/export_to_model.py
-msgid "File"
-msgstr "Fichier"
-
 #: admin/blocks.py
 msgid "Import Fields Block"
 msgstr "Importer un bloc de champs"
@@ -973,7 +1016,7 @@ msgstr ""
 msgid "Previous page"
 msgstr "Page précédente"
 
-#: admin/fields.py wf/editable.py
+#: admin/fields.py wf/create_formdata.py wf/editable.py
 msgid "All pages"
 msgstr "Toutes les pages"
 
@@ -1105,7 +1148,7 @@ msgstr "Inclure une page de confirmation"
 msgid "Confirmation Page"
 msgstr "Page de confirmation"
 
-#: admin/forms.py api.py formdata.py
+#: admin/forms.py api.py formdata.py wf/create_formdata.py
 msgid "Draft"
 msgstr "Brouillon"
 
@@ -1157,9 +1200,9 @@ msgstr "Commencer par un CAPTCHA pour les utilisateurs anonymes"
 msgid "CAPTCHA"
 msgstr "CAPTCHA"
 
-#: admin/forms.py templates/wcs/backoffice/formdef-inspect.html
-msgid "Include button to download all files"
-msgstr "Inclure un bouton pour télécharger tous les fichiers"
+#: admin/forms.py
+msgid "Sidebar elements"
+msgstr "Contenu de la barre latérale"
 
 #: admin/forms.py templates/wcs/backoffice/formdef-inspect.html
 msgid "Skip from per user view"
@@ -1597,10 +1640,6 @@ msgstr "Attention les demandes seront supprimées."
 msgid "Address"
 msgstr "Adresse"
 
-#: admin/forms.py
-msgid "Overwrite"
-msgstr "Écraser"
-
 #: admin/forms.py admin/workflows.py
 #, python-format
 msgid "Error loading form (%s)."
@@ -1614,10 +1653,6 @@ msgstr "Vous devez entrer un fichier ou une URL."
 msgid "Overwritten (removal of incompatible fields)"
 msgstr "Écrasement (avec suppression des champs incompatibles)"
 
-#: admin/forms.py
-msgid "Overwritten"
-msgstr "Écrasement"
-
 #: admin/forms.py
 msgid "Summary of changes"
 msgstr "Résumé des modifications"
@@ -1724,6 +1759,14 @@ msgstr "Ré-indexation des données pour le nouveau workflow"
 msgid "Back"
 msgstr "Retour"
 
+#: admin/logged_errors.py
+msgid "Condition"
+msgstr "Condition"
+
+#: admin/logged_errors.py
+msgid "Condition type"
+msgstr "Type de condition"
+
 #: admin/logged_errors.py
 #, python-format
 msgid "Logged Errors - %s"
@@ -4382,6 +4425,10 @@ msgstr "Fonction de l’utilisateur connecté"
 msgid "Submission Agent"
 msgstr "Agent à la saisie"
 
+#: backoffice/management.py
+msgid "Criticality Level"
+msgstr "Niveau de criticité"
+
 #: backoffice/management.py
 msgid "Current view"
 msgstr "Vue actuelle"
@@ -4414,6 +4461,11 @@ msgstr "Statuts à afficher"
 msgid "Current user"
 msgstr "Utilisateur connecté"
 
+#: backoffice/management.py
+msgctxt "criticality-level"
+msgid "All"
+msgstr "Tous"
+
 #: backoffice/management.py fields/base.py fields/bool.py formdata.py
 #: statistics/views.py templates/wcs/backoffice/data-source.html workflows.py
 msgid "No"
@@ -4651,7 +4703,7 @@ msgstr ""
 "Vous avez accédé à ce formulaire via son code de suivi, vous le voyez donc "
 "aussi comme l’usager."
 
-#: backoffice/management.py
+#: backoffice/management.py formdef.py
 msgid "General Information"
 msgstr "Informations générales"
 
@@ -4700,11 +4752,11 @@ msgstr "Ouvrir"
 msgid "Comments"
 msgstr "Commentaire"
 
-#: backoffice/management.py
+#: backoffice/management.py formdef.py
 msgid "Associated User"
 msgstr "Usager associé"
 
-#: backoffice/management.py
+#: backoffice/management.py formdef.py
 msgid "User Pending Forms"
 msgstr "Formulaires de cet usager en attente"
 
@@ -5118,6 +5170,10 @@ msgstr "Web"
 msgid "File Import"
 msgstr "Importation de fichier"
 
+#: carddata.py
+msgid "Updating relations"
+msgstr "Mise à jour des relations"
+
 #: carddef.py
 #, python-format
 msgid "No such card model: %s"
@@ -5354,6 +5410,11 @@ msgstr "Tableaux de traitement"
 msgid "Failed to evaluate prefill on field \"%s\""
 msgstr "Erreur à l’évaluation du préremplissage du champ « %s »"
 
+#: fields/base.py
+#, python-format
+msgid "Field: %s"
+msgstr "Champ : %s"
+
 #: fields/base.py fields/item.py fields/items.py
 #, python-format
 msgid "datasource is unavailable (field id: %s)"
@@ -6184,6 +6245,18 @@ msgctxt "item"
 msgid "forms"
 msgstr "demandes"
 
+#: formdef.py
+msgid "Button to download all files"
+msgstr "Bouton pour télécharger tous les fichiers"
+
+#: formdef.py
+msgid "Submission context"
+msgstr "Informations sur la saisie"
+
+#: formdef.py
+msgid "Custom template"
+msgstr "Gabarit personnalisé"
+
 #: formdef.py
 #, python-format
 msgid "Could not render submission lateral template (%s)"
@@ -6684,11 +6757,39 @@ msgstr "Message quand un formulaire a été enregistré"
 
 #: forms/root.py
 msgid ""
+"\n"
 "The form has been recorded on {{ form_receipt_datetime }} with the number "
-"{{ form_number }}."
+"{{ form_number }}.\n"
+"{% if form_submission_agent_display_name %}\n"
+"It has been submitted for you by {{ form_submission_agent_display_name }}\n"
+"{% if form_submission_channel == \"phone\" %}after a phone call.\n"
+"{% elif form_submission_channel == \"email\" %}after an email.\n"
+"{% elif form_submission_channel == \"mail\" %}after a mail.\n"
+"{% elif form_submission_channel == \"social-network\" %}after a message on a "
+"social network.\n"
+"{% elif form_submission_channel == \"counter\" %}after your passage at the "
+"counter.\n"
+"{% else %}.\n"
+"{% endif %}\n"
+"{% endif %}\n"
+"              "
 msgstr ""
+"\n"
 "Le formulaire a été enregistré le {{ form_receipt_datetime }} avec le numéro "
-"{{ form_number }}."
+"{{ form_number }}.\n"
+"{% if form_submission_agent_display_name %}\n"
+"Il a été saisie pour vous par {{ form_submission_agent_display_name }}\n"
+"{% if form_submission_channel == \"phone\" %}suite à un appel téléphonique.\n"
+"{% elif form_submission_channel == \"email\" %}suite à un courriel.\n"
+"{% elif form_submission_channel == \"mail\" %}suite à un courrier.\n"
+"{% elif form_submission_channel == \"social-network\" %}suite à un message "
+"sur un réseau social.\n"
+"{% elif form_submission_channel == \"counter\" %}suite à votre passage au "
+"guichet.\n"
+"{% else %}.\n"
+"{% endif %}\n"
+"{% endif %}\n"
+"              "
 
 #: forms/root.py
 msgid ""
@@ -6699,8 +6800,39 @@ msgstr ""
 "pour n’autoriser qu’un seul exemplaire par utilisateur"
 
 #: forms/root.py
-msgid "The form has been recorded on {{ form_receipt_datetime }}."
-msgstr "Le formulaire a été enregistré le {{ form_receipt_datetime }}."
+msgid ""
+"\n"
+"The form has been recorded on {{ form_receipt_datetime }}.\n"
+"{% if form_submission_agent_display_name %}\n"
+"It has been submitted for you by {{ form_submission_agent_display_name }}\n"
+"{% if form_submission_channel == \"phone\" %}after a phone call.\n"
+"{% elif form_submission_channel == \"email\" %}after an email.\n"
+"{% elif form_submission_channel == \"mail\" %}after a mail.\n"
+"{% elif form_submission_channel == \"social-network\" %}after a message on a "
+"social network.\n"
+"{% elif form_submission_channel == \"counter\" %}after your passage at the "
+"counter.\n"
+"{% else %}.\n"
+"{% endif %}\n"
+"{% endif %}\n"
+"              "
+
+msgstr ""
+"\n"
+"Le formulaire a été enregistré le {{ form_receipt_datetime }}.\n"
+"{% if form_submission_agent_display_name %}\n"
+"Il a été saisie pour vous par {{ form_submission_agent_display_name }}\n"
+"{% if form_submission_channel == \"phone\" %}suite à un appel téléphonique.\n"
+"{% elif form_submission_channel == \"email\" %}suite à un courriel.\n"
+"{% elif form_submission_channel == \"mail\" %}suite à un courrier.\n"
+"{% elif form_submission_channel == \"social-network\" %}suite à un message "
+"sur un réseau social.\n"
+"{% elif form_submission_channel == \"counter\" %}suite à votre passage au "
+"guichet.\n"
+"{% else %}.\n"
+"{% endif %}\n"
+"{% endif %}\n"
+"              "
 
 #: forms/root.py
 msgid "Message when a form is displayed before validation"
@@ -9295,39 +9427,10 @@ msgstr[1] "%(fields_count)s champs"
 msgid "Field Blocks"
 msgstr "Blocs de champs"
 
-#: templates/wcs/backoffice/blocks.html templates/wcs/backoffice/cards.html
-#: templates/wcs/backoffice/categories.html
-#: templates/wcs/backoffice/comment-templates.html
-#: templates/wcs/backoffice/data-sources.html
-#: templates/wcs/backoffice/formdef.html templates/wcs/backoffice/forms.html
-#: templates/wcs/backoffice/mail-templates.html
-#: templates/wcs/backoffice/snapshots.html
-#: templates/wcs/backoffice/test-webservice-responses.html
-#: templates/wcs/backoffice/test_sidebar.html
-#: templates/wcs/backoffice/tests.html
-#: templates/wcs/backoffice/workflow-global-action.html
-#: templates/wcs/backoffice/workflow.html
-#: templates/wcs/backoffice/workflows.html
-#: templates/wcs/backoffice/wscalls.html
-msgid "Actions"
-msgstr "Actions"
-
 #: templates/wcs/backoffice/blocks.html
 msgid "New field block"
 msgstr "Nouveau bloc de champs"
 
-#: templates/wcs/backoffice/blocks.html templates/wcs/backoffice/cards.html
-#: templates/wcs/backoffice/category.html
-#: templates/wcs/backoffice/comment-templates.html
-#: templates/wcs/backoffice/data-sources.html
-#: templates/wcs/backoffice/formdef.html templates/wcs/backoffice/forms.html
-#: templates/wcs/backoffice/mail-templates.html
-#: templates/wcs/backoffice/test_sidebar.html
-#: templates/wcs/backoffice/tests.html templates/wcs/backoffice/workflow.html
-#: templates/wcs/backoffice/workflows.html
-msgid "Navigation"
-msgstr "Navigation"
-
 #: templates/wcs/backoffice/card-data-import-form.html
 msgid "You can add data to this card by uploading a JSON file."
 msgstr ""
@@ -9592,12 +9695,12 @@ msgid "Display to unlogged users"
 msgstr "Afficher aux utilisateurs non-connectés"
 
 #: templates/wcs/backoffice/formdef-inspect.html
-msgid "Tracking codes"
-msgstr "Codes de suivi"
+msgid "Management sidebar elements"
+msgstr "Contenu de la barre latéral pour le traitement"
 
 #: templates/wcs/backoffice/formdef-inspect.html
-msgid "default value"
-msgstr "valeur par défaut"
+msgid "Tracking codes"
+msgstr "Codes de suivi"
 
 #: templates/wcs/backoffice/formdef-inspect.html
 msgid "Redirection when disabled"
@@ -9723,6 +9826,21 @@ msgstr "non trouvé"
 msgid "There are no agendas."
 msgstr "Il n’y a pas d’agendas."
 
+#: templates/wcs/backoffice/includes/forms.html
+#, python-format
+msgid "Published from %(date1)s until %(date2)s"
+msgstr "Publié du %(date1)s au %(date2)s"
+
+#: templates/wcs/backoffice/includes/forms.html
+#, python-format
+msgid "Published from %(date1)s"
+msgstr "Publié à partir du %(date1)s"
+
+#: templates/wcs/backoffice/includes/forms.html
+#, python-format
+msgid "Published until %(date2)s"
+msgstr "Publié jusqu’au %(date2)s"
+
 #: templates/wcs/backoffice/includes/mail-templates.html
 msgid "There are no mail templates defined."
 msgstr "Il n’y a pas de modèle de courriel défini."
@@ -10423,6 +10541,11 @@ msgstr "Étapes"
 msgid "Step %(page_no)s: %(page_label)s"
 msgstr "Étape %(page_no)s : %(page_label)s"
 
+#: templates/wcs/formdata_steps.html
+#, python-format
+msgid "Go back to step: %(page_label)s"
+msgstr "REtourner à l’étape : %(page_label)s"
+
 #: templates/wcs/formdata_steps.html
 #, python-format
 msgid "Step %(page_no)s of %(page_count)s:"
@@ -10638,6 +10761,10 @@ msgstr "dans"
 msgid "not in"
 msgstr "pas dans"
 
+#: variables.py
+msgid "contains"
+msgstr "contient"
+
 #: variables.py
 #, python-format
 msgid "Invalid value \"%(value)s\" for filter \"%(filter)s\""
@@ -11029,6 +11156,22 @@ msgstr "Libellé de l’action"
 msgid "Create new draft"
 msgstr "Créer en tant que brouillon"
 
+#: wf/create_formdata.py
+msgid "Operation mode when a draft is created"
+msgstr "Mode d’édition quand un brouillon est créé"
+
+#: wf/create_formdata.py wf/editable.py
+msgid "Single page"
+msgstr "Une seule page"
+
+#: wf/create_formdata.py wf/editable.py
+msgid "From specific page"
+msgstr "À partir d’une page"
+
+#: wf/create_formdata.py wf/editable.py
+msgid "Page Identifier"
+msgstr "Identifiant de page"
+
 #: wf/create_formdata.py
 msgid "Backoffice submission"
 msgstr "Saisie backoffice"
@@ -11280,18 +11423,6 @@ msgstr "Statut après modification"
 msgid "Don't select any if you don't want status change processing"
 msgstr "N’en sélectionnez aucun si vous ne voulez pas de changement de statut"
 
-#: wf/editable.py
-msgid "Single page"
-msgstr "Une seule page"
-
-#: wf/editable.py
-msgid "From specific page"
-msgstr "À partir d’une page"
-
-#: wf/editable.py
-msgid "Page Identifier"
-msgstr "Identifiant de page"
-
 #: wf/editable.py wf/wscall.py workflows.py
 msgid "Set marker to jump back to current status"
 msgstr "Poser un marqueur qui permettra de revenir au statut actuel"

wcs/logged_errors.py

@@ -18,6 +18,7 @@ import re
 
 from django.utils.formats import number_format
 from django.utils.timezone import now
+from quixote import get_publisher
 from quixote.html import htmlescape, htmltext
 
 from wcs.carddef import CardDef
@@ -42,6 +43,7 @@ class LoggedError:
     status_item_id = None
     expression = None
     expression_type = None
+    context = None
     traceback = None
     exception_class = None
     exception_message = None
@@ -93,6 +95,8 @@ class LoggedError:
         if status:
             error.status_id = status.id
 
+        error.context = get_publisher().get_error_context()
+
         error.first_occurence_timestamp = now()
         error.tech_id = error.build_tech_id()
         error.occurences_count += 1
@@ -111,6 +115,7 @@ class LoggedError:
         self.traceback = error.traceback
         self.expression = error.expression
         self.expression_type = error.expression_type
+        self.context = error.context
         # exception should be the same (same tech_id), record just in case
         self.exception_class = error.exception_class
         self.exception_message = error.exception_message

wcs/publisher.py

@@ -24,8 +24,10 @@ import shutil
 import sys
 import traceback
 import zipfile
+import zoneinfo
 from contextlib import ExitStack, contextmanager
 
+from django.utils import timezone
 from django.utils.timezone import localtime
 
 from . import custom_views, data_sources, formdef, sessions
@@ -233,6 +235,16 @@ class WcsPublisher(QommonPublisher):
         self.session_manager_class = sessions.StorageSessionManager
         self.set_session_manager(self.session_manager_class(session_class=self.session_class))
 
+    def start_request(self):
+        self.setup_timezone()
+        super().start_request()
+
+    def setup_timezone(self):
+        try:
+            timezone.activate(zoneinfo.ZoneInfo(self.get_site_option('timezone')))
+        except zoneinfo.ZoneInfoNotFoundError:
+            timezone.deactivate()  # use value from django settings
+
     def get_enabled_languages(self):
         return self.cfg.get('language', {}).get('languages') or []
 
@@ -455,6 +467,7 @@ class WcsPublisher(QommonPublisher):
             for _formdef in FormDef.select() + CardDef.select():
                 sql.do_formdef_tables(_formdef)
             sql.migrate_global_views(conn, cur)
+            sql.init_search_tokens()
             cur.close()
 
     def record_deprecated_usage(self, *args, **kwargs):
@@ -592,9 +605,12 @@ class WcsPublisher(QommonPublisher):
 
     def cleanup(self):
         self._cached_user_fields_formdef = None
+        self._update_related_seen = None
+        self._error_context = None
         from . import sql
 
         sql.cleanup_connection()
+        timezone.deactivate()
 
     @contextmanager
     def complex_data(self):
@@ -659,6 +675,22 @@ class WcsPublisher(QommonPublisher):
         finally:
             self.keep_all_block_rows_mode = False
 
+    # stacked contexts to include in logged errors
+    _error_context = None
+
+    @contextmanager
+    def error_context(self, **kwargs):
+        if not self._error_context:
+            self._error_context = []
+        self._error_context.append(kwargs)
+        try:
+            yield True
+        finally:
+            self._error_context.pop()
+
+    def get_error_context(self):
+        return {'stack': self._error_context} if self._error_context else None
+
     def clean_deleted_users(self, **kwargs):
         for user_id in self.user_class.get_to_delete_ids():
             self.user_class.remove_object(user_id)

wcs/qommon/cron.py

@@ -121,6 +121,7 @@ def cron_worker(publisher, since, job_name=None):
         CronJob.log('running jobs: %r' % sorted([x.name or x for x in jobs]))
     for job in jobs:
         publisher.install_lang()
+        publisher.setup_timezone()
         publisher.reset_formdata_state()
         publisher.set_sql_application_name(f'wcs-cron-{job.name}')
         try:

wcs/qommon/form.py

@@ -3711,7 +3711,9 @@ class HiddenErrorWidget(HiddenWidget):
 
 class SingleSelectWidgetWithOther(CompositeWidget):
     def __init__(self, name, value=None, **kwargs):
+        other_widget_class = kwargs.pop('other_widget_class', StringWidget)
         CompositeWidget.__init__(self, name, value=value, **kwargs)
+        kwargs.pop('attrs', None)
         if 'title' in kwargs:
             del kwargs['title']
         options = kwargs.get('options')[:]
@@ -3727,8 +3729,23 @@ class SingleSelectWidgetWithOther(CompositeWidget):
         else:
             choice_value = '__other'
             other_value = value
-        self.add(SingleSelectWidget, 'choice', value=choice_value, **kwargs)
-        self.add(StringWidget, 'other', value=other_value, size=35)
+        self.add(
+            SingleSelectWidget,
+            'choice',
+            value=choice_value,
+            attrs={'data-dynamic-display-parent': 'true'},
+            **kwargs,
+        )
+        self.add(
+            other_widget_class,
+            'other',
+            value=other_value,
+            size=35,
+            attrs={
+                'data-dynamic-display-value': '__other',
+                'data-dynamic-display-child-of': f'{name}$choice',
+            },
+        )
 
     def _parse(self, request):
         self.value = self.get('choice')

wcs/qommon/misc.py

@@ -450,7 +450,11 @@ def get_variadic_url(url, variables, encode_query=True):
         return url
 
     # django template
-    if '{{' in url or '{%' in url:
+    if (
+        '{{' in url
+        or '{%' in url
+        or (get_publisher() and get_publisher().has_site_option('disable-ezt-support'))
+    ):
         try:
             with no_complex(variables):
                 url = Template(url).render(variables)

wcs/qommon/publisher.py

@@ -442,11 +442,13 @@ class QommonPublisher(Publisher):
                 'allow-tracking-code-in-url': 'true',
                 'disabled-fields': 'ranked-items, table, table-select, tablerows',
                 'disable-rtf-support': 'true',
+                'enable-card-identifier-template': 'true',
                 'enable-intermediate-anonymisation': 'true',
                 'relatable-hosts': '',
                 'sync-map-and-address-fields': 'true',
                 'unused-files-behaviour': 'remove',
                 'rich-text-wf-displaymsg': 'auto-ckeditor',
+                'timezone': 'Europe/Paris',
             },
         }
         if self.site_options is None:
@@ -690,6 +692,11 @@ class QommonPublisher(Publisher):
         for error in self.loggederror_class.select(clause=clauses):
             self.loggederror_class.remove_object(error.id)
 
+    def clean_search_tokens(self, **kwargs):
+        from wcs import sql
+
+        sql.purge_obsolete_search_tokens()
+
     @classmethod
     def register_cronjobs(cls):
         cls.register_cronjob(CronJob(cls.clean_sessions, minutes=[0], name='clean_sessions'))
@@ -702,6 +709,9 @@ class QommonPublisher(Publisher):
         cls.register_cronjob(
             CronJob(cls.clean_loggederrors, hours=[3], minutes=[0], name='clean_loggederrors')
         )
+        cls.register_cronjob(
+            CronJob(cls.clean_search_tokens, weekdays=[0], hours=[1], minutes=[0], name='clean_search_tokens')
+        )
 
     _initialized = False
 

wcs/qommon/static/css/dc2/admin.scss

@@ -222,13 +222,8 @@ div#new-action, div#new-trigger, div#new-field {
 	}
 }
 
-div#new-field {
-	margin: 2em 0 4px 0;
-	padding: 5px 5px;
-}
-
-div#new-field form {
-	margin-bottom: 2em;
+form#import-fields {
+	margin-top: 1em;
 }
 
 aside#sidebar div.news h3,
@@ -3141,3 +3136,25 @@ div[role="tabpanel"] > div.infonotice:first-child {
 form div.widget[data-widget-name="model_file_mode"] {
 	margin-bottom: 0;
 }
+
+.extra-info.publication-dates {
+	&::before {
+		content: "";
+		display: block;
+	}
+}
+
+#panel-general ul.logged-error-frames {
+	padding: 0;
+	margin: 0;
+	li {
+		padding: 0;
+		margin: 0;
+	}
+	.logged-error-frames--context {
+		list-style: none;
+	}
+	> li:nth-child(2n) {
+		background: #eee;
+	}
+}

wcs/qommon/static/css/qommon.scss

@@ -30,7 +30,7 @@ div.list-add {
 	display: block;
 }
 
-div.SingleSelectWidgetWithOther .content .widget {
+div.SingleSelectWidgetWithOther .content .widget:not(.widget-hidden) {
 	display: inline-block;
 }
 

wcs/qommon/static/js/qommon.forms.js

@@ -147,10 +147,11 @@ $(function() {
   var autosave_timeout_id = null;
   var autosave_is_running = false;
   var autosave_button_to_click_on_complete = null;
+  var last_auto_save = $('form[data-has-draft]').serialize();
 
   if ($('form[data-has-draft]:not([data-autosave=false])').length == 1) {
-    var last_auto_save = $('form[data-has-draft]').serialize();
     var error_counter = 0;
+
     function autosave() {
       var $form = $('form[data-has-draft]');
       if ($form.hasClass('disabled-during-submit')) return;
@@ -439,6 +440,7 @@ $(function() {
   }
 
   add_js_behaviours($('form[data-live-url], form[data-backoffice-preview]'));
+  last_auto_save = $('form[data-has-draft]').serialize();
 
   // Form with error
   const errornotice = document.querySelector('form:not([data-backoffice-preview]) .errornotice');
@@ -1083,3 +1085,20 @@ document.addEventListener('DOMContentLoaded', function(){
     initLiveValidation(blockWidgets)
   })
 })
+
+document.addEventListener('DOMContentLoaded', function(){
+  const previous_page_id_input = document.querySelector('[name="previous-page-id"]')
+  if (!previous_page_id_input) return
+  document.querySelectorAll('.wcs-step[data-page-id]').forEach((step, idx) => {
+    step.addEventListener('keydown', function(e) {
+      if (e.key !== "Enter" && e.key !== " ") return
+      e.preventDefault()
+      previous_page_id_input.value = step.dataset.pageId
+      document.querySelector('button[name="previous"]').dispatchEvent(new MouseEvent('click'))
+    })
+    step.addEventListener('click', function() {
+      previous_page_id_input.value = step.dataset.pageId
+      document.querySelector('button[name="previous"]').dispatchEvent(new MouseEvent('click'))
+    })
+  })
+})

wcs/qommon/static/js/qommon.js

@@ -51,21 +51,6 @@ $(function() {
     });
   }
 
-  function prepate_select_or_other_widgets() {
-    $('.SingleSelectWidgetWithOther').each(function(idx, elem) {
-      var $widget = $(elem);
-      $widget.find('select').off('change input').on('change input', function() {
-        var val = $(this).val();
-        if (val == '__other') {
-          $widget.find('.StringWidget').show();
-        } else {
-          $widget.find('.StringWidget').hide();
-        }
-      });
-      $widget.find('select').trigger('change');
-    });
-  }
-
   function prepare_confirmation_buttons() {
     $('button[data-ask-for-confirmation]').off('click').on('click', function() {
       var text = $(this).data('ask-for-confirmation');
@@ -108,7 +93,6 @@ $(function() {
   function prepare_widgets() {
     prepare_dynamic_widgets();
     prepare_autocomplete_widgets();
-    prepate_select_or_other_widgets();
     prepare_select_empty_label();
     prepare_confirmation_buttons();
   }

wcs/qommon/templates/qommon/forms/widgets/mini-rich-text.html

@@ -1,8 +1,6 @@
 {% extends "qommon/forms/widget.html" %}
 {% block widget-control %}
-  <textarea hidden id="form_{{widget.get_name_for_id}}" name="{{widget.name}}">
-    {{widget.value|default:""}}
-  </textarea>
+  <textarea hidden id="form_{{widget.get_name_for_id}}" name="{{widget.name}}">{{widget.value|default:""}}</textarea>
   <godo-editor
     style="width: 100%"
     {% for attr in widget.attrs.items %}{{attr.0}}="{{attr.1}}" {% endfor %}

wcs/qommon/templatetags/qommon.py

@@ -922,6 +922,11 @@ def between(queryset):
     return queryset.apply_between()
 
 
+@register_queryset_filter(name='icontains', attr='apply_icontains')
+def icontains(queryset):
+    return queryset.apply_icontains()
+
+
 @register.filter
 def count(queryset):
     if hasattr(queryset, '__len__'):

wcs/sql.py

@@ -96,6 +96,20 @@ SQL_TYPE_MAPPING = {
 }
 
 
+def _table_exists(cur, table_name):
+    cur.execute('SELECT 1 FROM pg_class WHERE relname = %s;', (table_name,))
+    rows = cur.fetchall()
+    return len(rows) > 0
+
+
+def _trigger_exists(cur, table_name, trigger_name):
+    cur.execute(
+        'SELECT 1 FROM pg_trigger WHERE tgrelid = %s::regclass AND tgname = %s;', (table_name, trigger_name)
+    )
+    rows = cur.fetchall()
+    return len(rows) > 0
+
+
 class WcsPgConnection(psycopg2.extensions.connection):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -1177,6 +1191,7 @@ def do_loggederrors_table():
                                 status_item_id VARCHAR,
                                 expression VARCHAR,
                                 expression_type VARCHAR,
+                                context JSONB,
                                 traceback TEXT,
                                 exception_class VARCHAR,
                                 exception_message VARCHAR,
@@ -1199,6 +1214,8 @@ def do_loggederrors_table():
     # migrations
     if 'kind' not in existing_fields:
         cur.execute('''ALTER TABLE %s ADD COLUMN kind VARCHAR''' % table_name)
+    if 'context' not in existing_fields:
+        cur.execute('''ALTER TABLE %s ADD COLUMN context JSONB''' % table_name)
 
     # delete obsolete fields
     for field in existing_fields - needed_fields:
@@ -1579,6 +1596,8 @@ def do_global_views(conn, cur):
                 % (name, category.id)
             )
 
+    init_search_tokens_triggers(cur)
+
 
 def clean_global_views(conn, cur):
     # Purge of any dead data
@@ -1671,11 +1690,154 @@ def init_global_table(conn=None, cur=None):
                 endpoint_status=endpoint_status_filter,
             )
         )
+        init_search_tokens_data(cur)
 
     if own_conn:
         cur.close()
 
 
+def init_search_tokens(conn=None, cur=None):
+    own_cur = False
+    if cur is None:
+        own_cur = True
+        conn, cur = get_connection_and_cursor()
+
+    # Create table
+    cur.execute('CREATE TABLE IF NOT EXISTS wcs_search_tokens(token TEXT PRIMARY KEY);')
+
+    # Create triggers
+    init_search_tokens_triggers(cur)
+
+    # Fill table
+    init_search_tokens_data(cur)
+
+    # Index at the end, small performance trick... not that useful, but it's free...
+    cur.execute('CREATE EXTENSION IF NOT EXISTS pg_trgm;')
+    cur.execute(
+        'CREATE INDEX IF NOT EXISTS wcs_search_tokens_trgm ON wcs_search_tokens USING gin(token gin_trgm_ops);'
+    )
+
+    # And last: functions to use this brand new table
+    cur.execute('CREATE OR REPLACE AGGREGATE tsquery_agg_or  (tsquery) (sfunc=tsquery_or,  stype=tsquery);')
+    cur.execute('CREATE OR REPLACE AGGREGATE tsquery_agg_and (tsquery) (sfunc=tsquery_and, stype=tsquery);')
+    cur.execute(
+        r"""CREATE OR REPLACE FUNCTION public.wcs_tsquery(text)
+ RETURNS tsquery
+ LANGUAGE sql
+ STABLE
+AS $function$
+with
+        tokenized as (select unnest(regexp_split_to_array($1, '\s+')) w),
+        super_tokenized as (
+            select w,
+                coalesce((select plainto_tsquery(perfect.token) from wcs_search_tokens perfect where perfect.token = plainto_tsquery(w)::text),
+                         tsquery_agg_or(plainto_tsquery(partial.token) order by partial.token <-> w desc),
+                         plainto_tsquery(w)) tokens
+            from tokenized
+            left join wcs_search_tokens partial on partial.token % w and w not similar to '%[0-9]{2,}%'
+            group by w)
+select tsquery_agg_and(tokens) from super_tokenized;
+$function$;"""
+    )
+
+    if own_cur:
+        cur.close()
+
+
+def init_search_tokens_triggers(cur):
+    # We define only appending triggers, ie on INSERT and UPDATE.
+    # It would be far heavier to maintain deletions here, and having extra data has
+    # no or marginal side effect on search performances, and absolutely no impact
+    # on search results.
+    # Instead, a weekly cron job will delete obsolete entries, thus making it sure no
+    # personal data is kept uselessly.
+
+    # First part: the appending function
+    cur.execute(
+        """CREATE OR REPLACE FUNCTION wcs_search_tokens_trigger_fn ()
+ RETURNS trigger
+ LANGUAGE plpgsql
+AS $function$
+BEGIN
+        INSERT INTO wcs_search_tokens SELECT unnest(tsvector_to_array(NEW.fts)) ON CONFLICT(token) DO NOTHING;
+        RETURN NEW;
+END;
+$function$;"""
+    )
+
+    if not (_table_exists(cur, 'wcs_search_tokens')):
+        # abort trigger creation if tokens table doesn't exist yet
+        return
+
+    if _table_exists(cur, 'wcs_all_forms') and not _trigger_exists(
+        cur, 'wcs_all_forms', 'wcs_all_forms_fts_trg_upd'
+    ):
+        # Second part: insert and update triggers for wcs_all_forms
+        cur.execute(
+            """CREATE TRIGGER wcs_all_forms_fts_trg_ins 
+            AFTER INSERT ON wcs_all_forms 
+            FOR EACH ROW WHEN (NEW.fts IS NOT NULL) 
+            EXECUTE PROCEDURE wcs_search_tokens_trigger_fn();"""
+        )
+        cur.execute(
+            """CREATE TRIGGER wcs_all_forms_fts_trg_upd
+            AFTER UPDATE OF fts ON wcs_all_forms
+            FOR EACH ROW WHEN (NEW.fts IS NOT NULL)
+            EXECUTE PROCEDURE wcs_search_tokens_trigger_fn();"""
+        )
+
+    if _table_exists(cur, 'searchable_formdefs') and not _trigger_exists(
+        cur, 'searchable_formdefs', 'searchable_formdefs_fts_trg_upd'
+    ):
+        # Third part: insert and update triggers for searchable_formdefs
+        cur.execute(
+            """CREATE TRIGGER searchable_formdefs_fts_trg_ins 
+            AFTER INSERT ON searchable_formdefs 
+            FOR EACH ROW WHEN (NEW.fts IS NOT NULL) 
+            EXECUTE PROCEDURE wcs_search_tokens_trigger_fn();"""
+        )
+        cur.execute(
+            """CREATE TRIGGER searchable_formdefs_fts_trg_upd 
+            AFTER UPDATE OF fts ON searchable_formdefs 
+            FOR EACH ROW WHEN (NEW.fts IS NOT NULL) 
+            EXECUTE PROCEDURE wcs_search_tokens_trigger_fn();"""
+        )
+
+
+def init_search_tokens_data(cur):
+    if not (_table_exists(cur, 'wcs_search_tokens')):
+        # abort table data initialization if tokens table doesn't exist yet
+        return
+
+    if _table_exists(cur, 'wcs_all_forms'):
+        cur.execute(
+            """INSERT INTO wcs_search_tokens 
+        SELECT unnest(tsvector_to_array(fts)) FROM wcs_all_forms
+        ON CONFLICT(token) DO NOTHING;"""
+        )
+    if _table_exists(cur, 'searchable_formdefs'):
+        cur.execute(
+            """INSERT INTO wcs_search_tokens 
+        SELECT unnest(tsvector_to_array(fts)) FROM searchable_formdefs 
+        ON CONFLICT(token) DO NOTHING;"""
+        )
+
+
+def purge_obsolete_search_tokens(cur=None):
+    own_cur = False
+    if cur is None:
+        own_cur = True
+        _, cur = get_connection_and_cursor()
+
+    cur.execute(
+        """DELETE FROM wcs_search_tokens 
+    WHERE token NOT IN (SELECT unnest(tsvector_to_array(fts)) FROM wcs_all_forms) 
+    AND token NOT IN (SELECT unnest(tsvector_to_array(fts)) FROM wcs_all_forms);"""
+    )
+    if own_cur:
+        cur.close()
+
+
 class SqlMixin:
     _table_name = None
     _numerical_id = True
@@ -2267,6 +2429,7 @@ class SqlDataMixin(SqlMixin):
     def __init__(self, id=None):
         self.id = id
         self.data = {}
+        self._has_changed_digest = False
 
     _evolution = None
 
@@ -2403,6 +2566,7 @@ class SqlDataMixin(SqlMixin):
 
     def _set_auto_fields(self, cur):
         if self.set_auto_fields():
+            self._has_changed_digest = True
             sql_statement = (
                 '''UPDATE %s
                                   SET id_display = %%(id_display)s,
@@ -2619,6 +2783,11 @@ class SqlDataMixin(SqlMixin):
                 if isinstance(value, str) and len(value) < 10000:
                     # avoid overlong strings, typically base64-encoded values
                     fts_strings[weight].add(value)
+                    # normalize values looking like phonenumbers, because
+                    # phonenumbers are normalized by the FTS criteria
+                    if len(value) < 30 and value != normalize_phone_number_for_fts_if_needed(value):
+                        # use weight 'D' to give preference to fields with the phonenumber validation
+                        fts_strings['D'].add(normalize_phone_number_for_fts_if_needed(value))
                 elif type(value) in (tuple, list):
                     for val in value:
                         fts_strings[weight].add(val)
@@ -2848,7 +3017,9 @@ class SqlCardData(SqlDataMixin, wcs.carddata.CardData):
     def store(self, *args, **kwargs):
         if self.uuid is None:
             self.uuid = str(uuid.uuid4())
-        return super().store(*args, **kwargs)
+        super().store(*args, **kwargs)
+        if self._has_changed_digest:
+            self.update_related()
 
 
 class SqlUser(SqlMixin, wcs.users.User):
@@ -3727,6 +3898,7 @@ class LoggedError(SqlMixin, wcs.logged_errors.LoggedError):
         ('status_item_id', 'varchar'),
         ('expression', 'varchar'),
         ('expression_type', 'varchar'),
+        ('context', 'jsonb'),
         ('traceback', 'text'),
         ('exception_class', 'varchar'),
         ('exception_message', 'varchar'),
@@ -4796,7 +4968,6 @@ class SearchableFormDef(SqlMixin):
                 % (cls._table_name, cls._table_name)
             )
         cls.do_indexes(cur)
-        cur.close()
 
         from wcs.carddef import CardDef
         from wcs.formdef import FormDef
@@ -4805,6 +4976,8 @@ class SearchableFormDef(SqlMixin):
             CardDef.select(ignore_errors=True), FormDef.select(ignore_errors=True)
         ):
             cls.update(obj=objectdef)
+        init_search_tokens(cur)
+        cur.close()
 
     @classmethod
     def update(cls, obj=None, removed_obj_type=None, removed_obj_id=None):
@@ -4842,7 +5015,7 @@ class SearchableFormDef(SqlMixin):
     def search(cls, obj_type, string):
         _, cur = get_connection_and_cursor()
         cur.execute(
-            'SELECT object_id FROM searchable_formdefs WHERE fts @@ plainto_tsquery(%s)',
+            'SELECT object_id FROM searchable_formdefs WHERE fts @@ wcs_tsquery(%s)',
             (FtsMatch.get_fts_value(string),),
         )
         ids = [x[0] for x in cur.fetchall()]
@@ -5107,7 +5280,7 @@ def get_period_total(
 # latest migration, number + description (description is not used
 # programmaticaly but will make sure git conflicts if two migrations are
 # separately added with the same number)
-SQL_LEVEL = (105, 'change test result json structure')
+SQL_LEVEL = (107, 'improved fts method')
 
 
 def migrate_global_views(conn, cur):
@@ -5235,10 +5408,11 @@ def migrate():
         # 50: switch role uuid column to varchar
         do_role_table()
         migrate_legacy_roles()
-    if sql_level < 53:
+    if sql_level < 106:
         # 47: store LoggedErrors in SQL
         # 48: remove acked attribute from LoggedError
         # 53: add kind column to logged_errors table
+        # 106: add context column to logged_errors table
         do_loggederrors_table()
     if sql_level < 94:
         # 3: introduction of _structured for user fields
@@ -5440,6 +5614,10 @@ def migrate():
         for formdef in FormDef.select() + CardDef.select():
             do_formdef_tables(formdef, rebuild_views=False, rebuild_global_views=False)
 
+    if sql_level < 107:
+        # 107: new fts mechanism with tokens table
+        init_search_tokens()
+
     if sql_level != SQL_LEVEL[0]:
         cur.execute(
             '''UPDATE wcs_meta SET value = %s, updated_at=NOW() WHERE key = %s''',

wcs/sql_criterias.py

@@ -25,7 +25,7 @@ from wcs.qommon import misc
 
 
 def like_escape(value):
-    value = value.replace('\\', '\\\\')
+    value = str(value or '').replace('\\', '\\\\')
     value = value.replace('_', '\\_')
     value = value.replace('%', '\\%')
     return value
@@ -336,12 +336,31 @@ class Intersects(Criteria):
 
 
 class ILike(Criteria):
+    sql_op = 'ILIKE'
+
     def __init__(self, attribute, value, **kwargs):
         super().__init__(attribute, value, **kwargs)
         self.value = '%' + like_escape(self.value) + '%'
 
-    def as_sql(self):
-        return '%s ILIKE %%(c%s)s' % (self.attribute, id(self.value))
+
+phone_re = re.compile(
+    r'''.*?(?P<phone>         # a phone number
+        ((\+[1-9])|(\b0))     # starting with an international prefix, or 0
+        [-\(\)\d\.\s/]{6,20}  # then a bunch of numbers/symbols
+        \b)                   # till the end of the "word"''',
+    re.X,
+)
+
+
+def normalize_phone_number_for_fts_if_needed(value):
+    phone_match = phone_re.match(value)
+    if phone_match and not re.match(r'^\d+-\d+$', phone_match.group('phone').strip()):
+        # if it looks like a phone number, normalize it to its
+        # "international/E164" format to match what's stored in the
+        # database.
+        phone_value = misc.normalize_phone_number_for_fts(phone_match.group('phone').strip())
+        value = value.replace(phone_match.group('phone').strip(), phone_value)
+    return value
 
 
 class FtsMatch(Criteria):
@@ -350,20 +369,7 @@ class FtsMatch(Criteria):
         self.attribute = 'fts'
         self.value = self.get_fts_value(value)
         if extra_normalize:
-            phone_match = re.match(
-                r'''.*?(?P<phone>         # a phone number
-                    ((\+[1-9])|(\b0))     # starting with an international prefix, or 0
-                    [-\(\)\d\.\s/]{6,20}  # then a bunch of numbers/symbols
-                    \b)                   # till the end of the "word"''',
-                self.value,
-                re.X,
-            )
-            if phone_match and not re.match(r'^\d+-\d+$', phone_match.group('phone').strip()):
-                # if it looks like a phone number, normalize it to its
-                # "international/E164" format to match what's stored in the
-                # database.
-                phone_value = misc.normalize_phone_number_for_fts(phone_match.group('phone').strip())
-                self.value = self.value.replace(phone_match.group('phone').strip(), phone_value)
+            self.value = normalize_phone_number_for_fts_if_needed(self.value)
 
     @classmethod
     def get_fts_value(cls, value):
@@ -373,6 +379,11 @@ class FtsMatch(Criteria):
         return 'fts @@ plainto_tsquery(%%(c%s)s)' % id(self.value)
 
 
+class WcsFtsMatch(FtsMatch):
+    def as_sql(self):
+        return 'fts @@ wcs_tsquery(%%(c%s)s)' % id(self.value)
+
+
 class ElementEqual(Criteria):
     def __init__(self, attribute, key, value, **kwargs):
         super().__init__(attribute, value)

wcs/templates/wcs/backoffice/formdef-inspect.html

@@ -56,13 +56,17 @@
           {% if formdef.roles %}
             <li><span class="parameter">{% trans "Display to unlogged users" %}{% trans ":" %}</span> {{ formdef.always_advertise|yesno }}</li>
           {% endif %}
-          <li><span class="parameter">{% trans "Include button to download all files" %}{% trans ":" %}</span> {{ formdef.include_download_all_button|yesno }}</li>
+          <li><span class="parameter">{% trans "Management sidebar elements" %}{% trans ":" %}</span>
+            <ul>
+              {% for label in formdef.management_sidebar_items_labels %} <li>{{ label }}</li>{% endfor %}
+            </ul>
+          </li>
           <li><span class="parameter">{% trans "Skip from per user view" %}{% trans ":" %}</span> {{ formdef.skip_from_360_view|yesno }}</li>
           <li><span class="parameter">{% trans "Tracking codes" %}{% trans ":" %}</span> {{ formdef.enable_tracking_codes|yesno }}</li>
           {% if formdef.enable_tracking_codes %}
             <li><span class="parameter">{% trans "Fields to check after entering the tracking code" %}{% trans ":" %}</span> {{ tracking_code_verify_fields_labels|default:"-" }}</li>
           {% endif %}
-          <li><span class="parameter">{% trans "Lifespan of drafts (in days)" %}{% trans ":" %}</span> {{ formdef.drafts_lifespan|default_if_none:_('default value') }}</li>
+          <li><span class="parameter">{% trans "Lifespan of drafts (in days)" %}{% trans ":" %}</span> {{ formdef.get_drafts_lifespan }}</li>
           <li><span class="parameter">{% trans "Templates" %}</span>
             <ul>
               <li><span class="parameter">{% trans "Digest" %}{% trans ":" %}</span> {{ formdef.default_digest_template|default:"-" }}</li>

wcs/templates/wcs/backoffice/includes/forms.html

@@ -10,6 +10,23 @@
             <li {% if item.disabled %}class="disabled"{% endif %}><a href="{{ item.get_admin_url }}">
               {% if not application %}{% include 'wcs/backoffice/includes/application_icons.html' with object=item %}{% endif %}
               {{ item.name }}
+              {% if item.publication_date or item.expiration_date %}
+                <span class="extra-info publication-dates">
+                  {% if item.publication_date and item.expiration_date %}
+                    {% blocktrans trimmed with date1=item.publication_date|date:"DATETIME_FORMAT" date2=item.expiration_date|date:"DATETIME_FORMAT" %}
+                      Published from {{ date1 }} until {{ date2 }}
+                    {% endblocktrans %}
+                  {% elif item.publication_date %}
+                    {% blocktrans trimmed with date1=item.publication_date|date:"DATETIME_FORMAT" %}
+                      Published from {{ date1 }}
+                    {% endblocktrans %}
+                  {% else %}
+                    {% blocktrans trimmed with date2=item.expiration_date|date:"DATETIME_FORMAT" %}
+                      Published until {{ date2 }}
+                    {% endblocktrans %}
+                  {% endif %}
+                </span>
+              {% endif %}
               {% if item.disabled and item.disabled_redirection %}
                 <span class="extra-info">- {% trans "redirection" %}</span>
               {% endif %}

wcs/templates/wcs/backoffice/logged-error.html

@@ -43,6 +43,17 @@
           {% if error.expression or error.expression_type %}
             <li>{{ view.error_expression_type_label }}{% trans ":" %} <code>{{ error.expression }}</code></li>
           {% endif %}
+          {% if error.context %}
+            <li><ul class="logged-error-frames">
+              {% for frame in view.get_context_frames %}
+                <li>{% if frame.source %}<a href="{{ frame.source.url }}">{{ frame.source.label }}</a>{% endif %}
+                  <ul class="logged-error-frames--context">
+                    {% for frame_context in frame.get_frame_lines %}
+                      <li>{{ frame_context.label }}{% trans ":" %} <code>{{ frame_context.value }}</code></li>
+                    {% endfor %}
+                  </ul></li>
+              {% endfor %}</ul></li>
+          {% endif %}
           {% if error.exception_class or error.exception_message %}
             <li>{% trans "Error message:" %} <code>{{ error.exception_class }}: {{ error.exception_message }}</code></li>
           {% endif %}

wcs/templates/wcs/formdata_steps.html

@@ -17,7 +17,14 @@
                      {% if forloop.last %}last{% endif %}
                      {% if forloop.counter == current_page_no %}current{% endif %}
                      {% if forloop.counter < current_page_no %}step-before{% endif %}
-                     {% if forloop.counter > current_page_no %}step-after{% endif %}" >
+                     {% if forloop.counter > current_page_no %}step-after{% endif %}"
+              {% if forloop.counter < current_page_no %}
+                aria-label="{% blocktrans with page_label=page_label %}Go back to step: {{ page_label }}{% endblocktrans %}"
+                role="button"
+                tabindex="0"
+                data-page-id="{% with page=pages|get:forloop.counter0 %}{{ page.id }}{% endwith %}"
+              {% endif %}
+            >
               <abbr
                 aria-label="{% blocktrans with page_no=forloop.counter page_count=page_labels|length %}Step {{ page_no }} of {{ page_count }}:{% endblocktrans %}"
                 title="{% blocktrans with page_no=forloop.counter page_count=page_labels|length %}Step {{ page_no }} of {{ page_count }}{% endblocktrans %}"

wcs/variables.py

@@ -30,6 +30,7 @@ from wcs.sql_criterias import (
     Equal,
     Greater,
     GreaterOrEqual,
+    ILike,
     Less,
     LessOrEqual,
     Not,
@@ -303,9 +304,21 @@ class LazyFormDefObjectsManager:
             ('in', _('in')),
             ('not_in', _('not in')),
         ]
+        text_operators = [
+            ('icontains', _('contains')),
+        ]
         if field.key == 'internal-id':
             return equality_operators + comparison_operators
-        if field.key in ['date', 'item', 'items', 'string', 'text', 'numeric']:
+        if field.key in ['string', 'text']:
+            return (
+                equality_operators
+                + comparison_operators
+                + more_comparison_operators
+                + in_operators
+                + empty_operators
+                + text_operators
+            )
+        if field.key in ['date', 'item', 'items', 'numeric']:
             return (
                 equality_operators
                 + comparison_operators
@@ -316,7 +329,7 @@ class LazyFormDefObjectsManager:
         if field.key == 'bool':
             return equality_operators + empty_operators
         if field.key == 'email':
-            return equality_operators + in_operators + empty_operators
+            return equality_operators + in_operators + empty_operators + text_operators
         return None
 
     def format_value(self, op, value, field):
@@ -412,6 +425,7 @@ class LazyFormDefObjectsManager:
             'gt': Greater,
             'gte': GreaterOrEqual,
             'in': Contains,
+            'icontains': ILike,
         }
 
         if isinstance(value, list) and op in ['eq', 'ne']:
@@ -588,6 +602,9 @@ class LazyFormDefObjectsManager:
     def apply_between(self):
         return self.apply_op('between')
 
+    def apply_icontains(self):
+        return self.apply_op('icontains')
+
     def getlist(self, key):
         return LazyList(self, key)
 
@@ -1886,7 +1903,7 @@ class LazyUser:
         self._user = user
 
     def inspect_keys(self):
-        return ['display_name', 'email', 'var', 'nameid']
+        return ['display_name', 'email', 'var', 'nameid', 'has_deleted_account']
 
     @property
     def display_name(self):
@@ -1896,6 +1913,10 @@ class LazyUser:
     def email(self):
         return self._user.email
 
+    @property
+    def has_deleted_account(self):
+        return bool(self._user.deleted_timestamp)
+
     @property
     def var(self):
         return LazyFormDataVar(self._user.get_formdef().fields, self._user.form_data)

wcs/wf/attachment.py

@@ -167,9 +167,10 @@ class AddAttachmentWorkflowStatusItem(WorkflowStatusItem):
                 self.store_in_backoffice_filefield(
                     formdata, self.backoffice_filefield_id, filename, content_type, outstream.read()
                 )
-            if self.attach_to_history:
-                f.fp.seek(0)
-                evo.add_part(AttachmentEvolutionPart.from_upload(f, varname=self.varname))
+            f.fp.seek(0)
+            evo_part = AttachmentEvolutionPart.from_upload(f, varname=self.varname)
+            evo_part.display_in_history = self.attach_to_history
+            evo.add_part(evo_part)
 
     def get_parameters(self):
         parameters = (

wcs/wf/create_formdata.py

@@ -387,6 +387,8 @@ class CreateFormdataWorkflowStatusItem(WorkflowStatusItem):
     map_fields_by_varname = False
     attach_to_history = False
     cached_field_labels = None
+    draft_edit_operation_mode = 'full'  # or 'single' or 'partial'
+    page_identifier = None
 
     def migrate(self):
         changed = super().migrate()
@@ -412,6 +414,13 @@ class CreateFormdataWorkflowStatusItem(WorkflowStatusItem):
         parameters = super().get_inspect_parameters()
         if self.user_association_mode != 'custom' and 'user_association_template' in parameters:
             parameters.remove('user_association_template')
+        if not self.draft:
+            if 'draft_edit_operation_mode' in parameters:
+                parameters.remove('draft_edit_operation_mode')
+            if 'page_identifier' in parameters:
+                parameters.remove('page_identifier')
+        if self.draft_edit_operation_mode not in ('single', 'partial') and 'page_identifier' in parameters:
+            parameters.remove('page_identifier')
         return parameters
 
     def add_parameters_widgets(self, form, parameters, prefix='', formdef=None, **kwargs):
@@ -443,7 +452,46 @@ class CreateFormdataWorkflowStatusItem(WorkflowStatusItem):
                 **{'data-autocomplete': 'true'},
             )
         if 'draft' in parameters:
-            form.add(CheckboxWidget, '%sdraft' % prefix, title=_('Create new draft'), value=self.draft)
+            form.add(
+                CheckboxWidget,
+                '%sdraft' % prefix,
+                title=_('Create new draft'),
+                value=self.draft,
+                attrs={'data-dynamic-display-parent': 'true'},
+                tab=('draft', _('Draft')),
+            )
+        if 'draft_edit_operation_mode' in parameters:
+            form.add(
+                RadiobuttonsWidget,
+                '%sdraft_edit_operation_mode' % prefix,
+                title=_('Operation mode when a draft is created'),
+                options=[
+                    ('full', _('All pages'), 'full'),
+                    ('single', _('Single page'), 'single'),
+                    ('partial', _('From specific page'), 'partial'),
+                ],
+                tab=('draft', _('Draft')),
+                value=self.draft_edit_operation_mode,
+                attrs={
+                    'data-dynamic-display-parent': 'true',
+                    'data-dynamic-display-child-of': f'{prefix}draft',
+                    'data-dynamic-display-checked': 'true',
+                },
+                extra_css_class='widget-inline-radio',
+                default_value=self.__class__.draft_edit_operation_mode,
+            )
+        if 'page_identifier' in parameters:
+            form.add(
+                StringWidget,
+                '%spage_identifier' % prefix,
+                title=_('Page Identifier'),
+                value=self.page_identifier,
+                tab=('draft', _('Draft')),
+                attrs={
+                    'data-dynamic-display-child-of': '%sdraft_edit_operation_mode' % prefix,
+                    'data-dynamic-display-value-in': 'single|partial',
+                },
+            )
         if 'backoffice_submission' in parameters:
             form.add(
                 CheckboxWidget,
@@ -566,6 +614,12 @@ class CreateFormdataWorkflowStatusItem(WorkflowStatusItem):
                         mapped_subfield_id = f'{field.id}${subfield.id}'
                         if mapped_subfield_id in mapped_field_ids:
                             self.cached_field_labels[mapped_subfield_id] = f'{field.label} - {subfield.label}'
+        if not self.draft:
+            # cleanup
+            if 'draft_edit_operation_mode' in self.get_parameters():
+                delattr(self, 'draft_edit_operation_mode')
+            if 'page_identifier' in self.get_parameters():
+                delattr(self, 'page_identifier')
 
     def get_mappings_parameter_view_value(self):
         to_id_fields = {str(field.id): field for field in self.formdef.get_widget_fields()}
@@ -595,11 +649,13 @@ class CreateFormdataWorkflowStatusItem(WorkflowStatusItem):
     def get_parameters(self):
         return (
             'action_label',
-            'draft',
             'formdef_slug',
             'map_fields_by_varname',
             'mappings',
             'backoffice_submission',
+            'draft',
+            'draft_edit_operation_mode',
+            'page_identifier',
             'user_association_mode',
             'user_association_template',
             'keep_submission_context',
@@ -737,6 +793,13 @@ class CreateFormdataWorkflowStatusItem(WorkflowStatusItem):
         if self.draft:
             new_formdata.status = 'draft'
             new_formdata.receipt_time = localtime()
+            if self.draft_edit_operation_mode != 'full':
+                new_formdata.workflow_data = {
+                    '_create_formdata_draft_edit': {
+                        'operation_mode': self.draft_edit_operation_mode,
+                        'page_identifier': self.page_identifier,
+                    },
+                }
             new_formdata.store()
             if formdef.enable_tracking_codes:
                 code.formdata = new_formdata  # this will .store() the code

wcs/wf/sendmail.py

@@ -174,6 +174,7 @@ class SendmailWorkflowStatusItem(WorkflowStatusItem):
                 add_element_label=self.get_add_role_label(),
                 element_kwargs={
                     'render_br': False,
+                    'other_widget_class': ComputedExpressionWidget,
                     'options': [(None, '---', None)] + self.get_list_of_roles(include_logged_in_users=False),
                 },
             )

wcs/workflows.py

@@ -372,6 +372,7 @@ class AttachmentEvolutionPart(EvolutionPart):
     render_for_fts = None
     storage = None
     storage_attrs = None
+    display_in_history = True
 
     def __init__(
         self,
@@ -452,6 +453,9 @@ class AttachmentEvolutionPart(EvolutionPart):
 
         return odict
 
+    def is_hidden(self):
+        return bool(not self.display_in_history)
+
     def view(self):
         show_link = True
         if self.has_redirect_url():
@@ -730,6 +734,18 @@ class WorkflowVariablesFieldsFormDef(FormDef):
     def is_readonly(self):
         return self.workflow.is_readonly()
 
+    def migrate(self):
+        changed = False
+        for field in self.fields or []:
+            changed |= field.migrate()
+            if getattr(field, 'prefill', None):  # 2024-03-11
+                # prefill attribute is no longer advertised for workflow variables,
+                # reset its value if it had one, so ancient python prefills do not
+                # persist.
+                field.prefill = None
+                changed = True
+        return changed
+
 
 class WorkflowBackofficeFieldsFormDef(FormDef):
     """Class to handle workflow backoffice fields, it loads and saves from/to
@@ -816,6 +832,9 @@ class Workflow(StorableObject):
             for field in self.backoffice_fields_formdef.fields:
                 changed |= field.migrate()
 
+        if self.variables_formdef:
+            changed |= self.variables_formdef.migrate()
+
         if not self.global_actions:
             self.global_actions = []
 
@@ -3049,7 +3068,9 @@ class WorkflowStatusItem(XmlSerialisable):
         return ''
 
     def get_admin_url(self):
-        return self.parent.get_admin_url() + 'items/%s/' % self.id
+        if self.parent:
+            return self.parent.get_admin_url() + 'items/%s/' % self.id
+        return ''
 
     def get_inspect_details(self):
         return getattr(self, 'label', '')
@@ -3114,7 +3135,10 @@ class WorkflowStatusItem(XmlSerialisable):
     def check_condition(self, formdata, record_errors=True):
         context = {'formdata': formdata, 'status_item': self}
         try:
-            return Condition(self.condition, context, record_errors=record_errors).evaluate()
+            return Condition(self.condition, context, record_errors=record_errors).evaluate(
+                source_label=str(self.description),
+                source_url=self.get_admin_url(),
+            )
         except RuntimeError:
             return False
 
@@ -3514,7 +3538,7 @@ class WorkflowStatusItem(XmlSerialisable):
                             uploads.append(complex_value)
 
         # 2. python expressions
-        if attachments:
+        if attachments and not get_publisher().has_site_option('forbid-python-expressions'):
             global_eval_dict = get_publisher().get_global_eval_dict()
             local_eval_dict = get_publisher().substitutions.get_context_variables()
             for attachment in attachments: