sql: test purge of search tokens (#86527)

introduce a new mechanism to implement FTS with fuzzy-match.
This is made possible by adding and maintaining a table of the
FTS tokens, wcs_search_tokens, fed with searchable_formdefs
and wcs_all_forms.
When a query is issued, its tokens are matched against the
tokens with a fuzzy match when no direct match is found, and
the query is then rebuilt.

tests/admin_pages/test_block.py

@@ -387,6 +387,13 @@ def test_block_use_in_formdef(pub):
     resp = resp.form.submit('submit')
     assert resp.pyquery('#form_error_max_items').text() == 'required field'
 
+    # check there's no crash if block is missing
+    block.remove_self()
+    resp = app.get(formdef.get_admin_url() + 'fields/')
+    assert resp.pyquery('#fields-list .type-block .type').text() == 'Block of fields (foobar, missing)'
+    resp = resp.click('Edit', href='%s/' % formdef.fields[0].id)
+    assert resp.pyquery('.field-edit--subtitle').text() == 'Block of fields (foobar, missing)'
+
 
 def test_block_use_in_workflow_backoffice_fields(pub):
     create_superuser(pub)
@@ -495,7 +502,7 @@ def test_removed_block_in_form_fields_list(pub):
 
     app = login(get_app(pub))
     resp = app.get('/backoffice/forms/%s/fields/' % formdef.id)
-    assert 'Field Block (removed, missing)' in resp.text
+    assert 'Block of fields (removed, missing)' in resp.text
 
 
 def test_block_edit_field_warnings(pub):
@@ -678,3 +685,37 @@ def test_block_test_results(pub):
     resp.form['varname'] = 'test_3'
     resp = resp.form.submit('submit').follow()
     assert TestResult.count() == 1
+
+
+def test_block_documentation(pub):
+    create_superuser(pub)
+
+    BlockDef.wipe()
+    blockdef = FormDef()
+    blockdef.name = 'block title'
+    blockdef.fields = [fields.BoolField(id='1', label='Bool')]
+    blockdef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get(blockdef.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(blockdef.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    blockdef.refresh_from_storage()
+    assert blockdef.documentation == '<p>doc</p>'
+    resp = app.get(blockdef.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')
+
+    resp = app.get(blockdef.get_admin_url() + 'fields/1/')
+    assert resp.pyquery('.documentation[hidden]')
+    assert resp.pyquery('#sidebar[hidden]')
+    resp = app.post_json(
+        blockdef.get_admin_url() + 'fields/1/update-documentation', {'content': '<p>doc</p>'}
+    )
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    blockdef.refresh_from_storage()
+    assert blockdef.fields[0].documentation == '<p>doc</p>'
+    resp = app.get(blockdef.get_admin_url() + 'fields/1/')
+    assert resp.pyquery('.documentation:not([hidden])')
+    assert resp.pyquery('#sidebar:not([hidden])')

tests/admin_pages/test_card.py

@@ -437,6 +437,10 @@ def test_card_id_template(pub):
     resp = resp.click('Templates')
     assert 'id_template' not in resp.text
 
+    # check a severe warning is displayed on field removal
+    resp = app.get(carddef.fields[0].get_admin_url() + 'delete')
+    assert 'This field may be used in the card custom identifiers' in resp.pyquery('.errornotice').text()
+
 
 def test_card_digest_template(pub):
     create_superuser(pub)
@@ -1176,3 +1180,35 @@ def test_cards_management_options(pub):
     assert_option_display(resp, 'Templates', 'Custom')
     resp = resp.click('Management', href='options/management')
     assert resp.form['history_pane_default_mode'].value == 'expanded'
+
+
+def test_card_documentation(pub):
+    create_superuser(pub)
+
+    CardDef.wipe()
+    carddef = FormDef()
+    carddef.name = 'card title'
+    carddef.fields = [fields.BoolField(id='1', label='Bool')]
+    carddef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get(carddef.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(carddef.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    carddef.refresh_from_storage()
+    assert carddef.documentation == '<p>doc</p>'
+    resp = app.get(carddef.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')
+
+    resp = app.get(carddef.get_admin_url() + 'fields/1/')
+    assert resp.pyquery('.documentation[hidden]')
+    assert resp.pyquery('#sidebar[hidden]')
+    resp = app.post_json(carddef.get_admin_url() + 'fields/1/update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    carddef.refresh_from_storage()
+    assert carddef.fields[0].documentation == '<p>doc</p>'
+    resp = app.get(carddef.get_admin_url() + 'fields/1/')
+    assert resp.pyquery('.documentation:not([hidden])')
+    assert resp.pyquery('#sidebar:not([hidden])')

tests/admin_pages/test_datasource.py

@@ -195,7 +195,6 @@ def test_data_sources_new(pub):
     resp = app.get('/backoffice/settings/data-sources/')
     resp = resp.click('New Data Source')
     resp.forms[0]['name'] = 'a new data source'
-    resp.forms[0]['description'] = 'description of the data source'
 
     resp.forms[0]['data_source$type'] = 'python'
     resp.forms[0]['data_source$value'] = repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}])
@@ -209,13 +208,11 @@ def test_data_sources_new(pub):
     assert 'Edit Data Source' in resp.text
 
     assert NamedDataSource.get(1).name == 'a new data source'
-    assert NamedDataSource.get(1).description == 'description of the data source'
 
     # add a second one
     resp = app.get('/backoffice/settings/data-sources/')
     resp = resp.click('New Data Source')
     resp.forms[0]['name'] = 'an other data source'
-    resp.forms[0]['description'] = 'description of the data source'
     resp.forms[0]['data_source$type'] = 'python'
     resp = resp.forms[0].submit('data_source$apply')
     resp.forms[0]['data_source$value'] = repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}])
@@ -408,7 +405,6 @@ def test_data_sources_category(pub):
     resp = app.get('/backoffice/settings/data-sources/categories/')
     resp = resp.click('New Category')
     resp.form['name'] = 'a new category'
-    resp.form['description'] = 'description of the category'
     resp = resp.form.submit('submit')
     assert DataSourceCategory.count() == 1
     category = DataSourceCategory.select()[0]
@@ -440,7 +436,6 @@ def test_data_sources_category(pub):
     resp = app.get('/backoffice/settings/data-sources/categories/')
     resp = resp.click('New Category')
     resp.form['name'] = 'a second category'
-    resp.form['description'] = 'description of the category'
     resp = resp.form.submit('submit')
     assert DataSourceCategory.count() == 2
     category2 = [x for x in DataSourceCategory.select() if x.id != category.id][0]
@@ -872,13 +867,10 @@ def test_data_sources_edit(pub):
 
     resp = resp.click(href='edit')
     assert resp.forms[0]['name'].value == 'foobar'
-    resp.forms[0]['description'] = 'data source description'
     resp = resp.forms[0].submit('submit')
     assert resp.location == 'http://example.net/backoffice/settings/data-sources/1/'
     resp = resp.follow()
 
-    assert NamedDataSource.get(1).description == 'data source description'
-
     resp = app.get('/backoffice/settings/data-sources/1/edit')
     assert '>Data Attribute</label>' in resp.text
     assert '>Id Attribute</label>' in resp.text
@@ -1146,3 +1138,22 @@ def test_data_sources_agenda_refresh(mock_collect, pub, chrono_url):
     resp = resp.follow()
     assert 'Agendas will be updated in the background.' in resp.text
     assert NamedDataSource.count() == 2
+
+
+def test_datasource_documentation(pub):
+    create_superuser(pub)
+
+    NamedDataSource.wipe()
+    data_source = NamedDataSource(name='foobar')
+    data_source.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get(data_source.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(data_source.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    data_source.refresh_from_storage()
+    assert data_source.documentation == '<p>doc</p>'
+    resp = app.get(data_source.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')

tests/admin_pages/test_form.py

@@ -3723,6 +3723,12 @@ def test_form_edit_field_warnings(pub):
         .startswith('There are at least 2201 data fields, including fields in blocks.')
     )
 
+    # no crash if default_items_count is none
+    formdef.fields[1].default_items_count = None
+    formdef.store()
+    resp = app.get('/backoffice/forms/%s/fields/' % formdef.id)
+    assert not resp.pyquery('.warningnotice')
+
     FormDef.wipe()
 
 
@@ -5173,3 +5179,35 @@ def test_admin_form_sql_integrity_error(pub):
         == 'There are integrity errors in the database column types.'
     )
     assert resp.pyquery('.errornotice li').text() == 'String, expected: character varying, got: boolean.'
+
+
+def test_form_documentation(pub):
+    create_superuser(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [fields.BoolField(id='1', label='Bool')]
+    formdef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get(formdef.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(formdef.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    formdef.refresh_from_storage()
+    assert formdef.documentation == '<p>doc</p>'
+    resp = app.get(formdef.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')
+
+    resp = app.get(formdef.get_admin_url() + 'fields/1/')
+    assert resp.pyquery('.documentation[hidden]')
+    assert resp.pyquery('#sidebar[hidden]')
+    resp = app.post_json(formdef.get_admin_url() + 'fields/1/update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    formdef.refresh_from_storage()
+    assert formdef.fields[0].documentation == '<p>doc</p>'
+    resp = app.get(formdef.get_admin_url() + 'fields/1/')
+    assert resp.pyquery('.documentation:not([hidden])')
+    assert resp.pyquery('#sidebar:not([hidden])')

tests/admin_pages/test_tests.py

@@ -99,6 +99,9 @@ def test_tests_page(pub):
     resp = resp.click('Second test')
     assert 'This test is empty' in resp.text
 
+    resp = resp.click('History')
+    assert 'Creation (empty)' in resp.text
+
     # test run with empty test is allowed
     app.get('/backoffice/forms/1/tests/results/run').follow()
 
@@ -374,6 +377,129 @@ def test_tests_status_page_image_field(pub):
     resp.follow(status=404)
 
 
+def test_tests_history_page(pub):
+    user = create_superuser(pub)
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.fields = [fields.StringField(id='1', varname='test_field', label='Test Field')]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
+    formdata.data['1'] = 'This is a test'
+    formdata.user_id = user.id
+    formdata.store()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.name = 'Test 1'
+    testdef.workflow_tests.actions = [
+        workflow_tests.ButtonClick(id='1', button_name='xxx'),
+    ]
+    # create one snapshot
+    testdef.store()
+
+    response = WebserviceResponse()
+    response.testdef_id = testdef.id
+    response.name = 'Fake response'
+    response.url = 'http://example.com/json'
+    response.payload = '{"foo": "bar"}'
+    response.store()
+
+    # create second snapshot
+    testdef.name = 'Test 2'
+    testdef.store()
+
+    # create third snapshot
+    testdef.name = 'Test 3'
+    testdef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
+    resp = resp.click('History')
+    assert [x.attrib['class'] for x in resp.pyquery.find('.snapshots-list tr')] == [
+        'new-day',
+        'collapsed',
+        'collapsed',
+    ]
+
+    # export snapshot
+    resp_export = resp.click('Export', index=1)
+    assert resp_export.content_type == 'application/x-wcs-snapshot'
+    assert '>Test 2<' in resp_export.text
+
+    # view snapshot
+    view_resp = resp.click('View', index=1)
+    assert '<h2>Test 2</h2>' in view_resp.text
+    assert 'Options' not in resp.text
+    assert 'Delete' not in resp.text
+    assert 'Edit' not in resp.text
+
+    resp = view_resp.click('Workflow tests')
+    assert 'Simulate click on action button' in resp.text
+    assert 'Add' not in resp.text
+    assert 'Delete' not in resp.text
+    assert 'Duplicate' not in resp.text
+
+    resp = resp.click('Edit')
+    assert '>Submit<' not in resp.text
+
+    resp = view_resp.click('Webservice responses')
+    assert 'New' not in resp.text
+    assert 'Remove' not in resp.text
+    assert 'Duplicate' not in resp.text
+
+    resp = resp.click('Fake response')
+    assert 'Edit webservice response' in resp.text
+    assert '>Submit<' not in resp.text
+
+    resp = view_resp.click('Inspect')
+
+    assert 'form_var_test_field' in resp.text
+
+    resp.form['django-condition'] = 'form_var_test_field == "This is a test"'
+    resp = resp.form.submit()
+    assert 'Condition result' in resp.text
+    assert 'result-true' in resp.text
+
+    # restore as new
+    assert TestDef.count() == 1
+    assert WorkflowTests.count() == 1
+    assert WebserviceResponse.count() == 1
+
+    resp = view_resp.click('Restore version')
+    resp.form['action'] = 'as-new'
+    resp = resp.form.submit('submit').follow()
+
+    assert TestDef.count() == 2
+    assert WorkflowTests.count() == 2
+    assert WebserviceResponse.count() == 2
+    assert '<h2>Test 2</h2>' in resp.text
+
+    # restore as current
+    resp = view_resp.click('Restore version')
+    resp.form['action'] = 'overwrite'
+    resp = resp.form.submit('submit').follow()
+
+    assert TestDef.count() == 2
+    assert WorkflowTests.count() == 2
+    assert WebserviceResponse.count() == 2
+    assert '<h2>Test 2</h2>' in resp.text
+
+    # restore first version as current, making sure webservice response is deleted
+    resp = resp.click('History')
+    resp = resp.click('Restore', index=2)
+    resp.form['action'] = 'overwrite'
+    resp = resp.form.submit('submit').follow()
+
+    assert TestDef.count() == 2
+    assert WorkflowTests.count() == 2
+    assert WebserviceResponse.count() == 1
+    assert '<h2>Test 1</h2>' in resp.text
+
+
 def test_tests_edit(pub):
     create_superuser(pub)
     user = pub.user_class(name='test user')
@@ -1457,6 +1583,21 @@ def test_tests_webservice_response(pub):
 
     assert 'must start with http://' in resp.text
 
+    testdef2 = TestDef.create_from_formdata(formdef, formdata)
+    testdef.name = 'Second test'
+    testdef2.store()
+
+    resp = app.get('/backoffice/forms/1/tests/%s/webservice-responses/' % testdef2.id)
+    assert 'Test response' not in resp.text
+
+    resp = resp.click('Import from other test')
+    resp.form['testdef_id'] = testdef.id
+    resp = resp.form.submit().follow()
+
+    assert 'Test response' in resp.text
+    assert len(testdef.get_webservice_responses()) == 1
+    assert len(testdef2.get_webservice_responses()) == 1
+
 
 def test_tests_test_users_management(pub):
     create_superuser(pub)
@@ -1509,9 +1650,11 @@ def test_tests_test_users_management(pub):
 
     real_user = pub.user_class(name='new user')
     real_user.email = 'jane@example.com'
+    real_user.roles = [role.id]
     real_user.form_data = {
         '1': 'Jane',
         '2': 'Doe',
+        '3': 'jane@example.com',
     }
     real_user.store()
 
@@ -1524,6 +1667,7 @@ def test_tests_test_users_management(pub):
     user = pub.user_class.select([NotNull('test_uuid')], order_by='id')[1]
     assert user.name == 'User test 2'
     assert user.email == 'jane@example.com'
+    assert user.roles == [role.id]
     assert user.form_data['1'] == 'Jane'
     assert user.form_data['2'] == 'Doe'
 
@@ -1542,8 +1686,32 @@ def test_tests_test_users_management(pub):
 
     assert 'A test user with this email already exists.' in resp.text
 
+    user_test_2_export_resp = resp.click('Export')
+
     resp = app.get('/backoffice/forms/test-users/')
     resp = resp.click('Remove', href=str(user.id))
     resp = resp.form.submit().follow()
 
     assert 'User test 2' not in resp.text
+
+    resp = resp.click('Import')
+    resp.form['file'] = Upload('export.json', user_test_2_export_resp.body, 'application/json')
+    resp = resp.form.submit().follow()
+
+    assert 'Test users have been successfully imported.' in resp.text
+    assert 'User test 2' in resp.text
+
+    user = pub.user_class.select([NotNull('test_uuid')], order_by='id')[1]
+    assert user.name == 'User test 2'
+    assert user.email == 'jane@example.com'
+    assert user.roles == [role.id]
+    assert user.form_data['1'] == 'Jane'
+    assert user.form_data['2'] == 'Doe'
+
+    global_export_resp = resp.click('Export')
+
+    resp = resp.click('Import')
+    resp.form['file'] = Upload('export.json', global_export_resp.body, 'application/json')
+    resp = resp.form.submit().follow()
+
+    assert 'Some already existing users were not imported.' in resp.text

tests/admin_pages/test_workflow.py

@@ -2349,7 +2349,7 @@ def test_workflows_backoffice_fields(pub):
         'foobar Text (line)',
         'foobar2 Text (line)',
         'foobar3 Title',
-        'foobar4 Field Block (Test Block)',
+        'foobar4 Block of fields (Test Block)',
     ]
 
     workflow.refresh_from_storage()
@@ -3262,11 +3262,11 @@ def test_workflows_create_formdata_fields_with_same_label(pub):
         ('', False, '---'),
         ('0', False, 'string1 - Text (line) (foo)'),
         ('1', True, 'string1 - Text (line) (bar)'),
-        ('2', False, 'block1 - Field Block (Test Block) (foo2)'),
+        ('2', False, 'block1 - Block of fields (Test Block) (foo2)'),
         ('2$123', False, 'block1 (foo2) - Test - Text (line)'),
-        ('3', False, 'block1 - Field Block (Test Block) (bar2)'),
+        ('3', False, 'block1 - Block of fields (Test Block) (bar2)'),
         ('3$123', False, 'block1 (bar2) - Test - Text (line)'),
-        ('4', False, 'block2 - Field Block (Test Block)'),
+        ('4', False, 'block2 - Block of fields (Test Block)'),
         ('4$123', False, 'block2 - Test - Text (line)'),
     ]
 
@@ -4429,3 +4429,115 @@ def test_workflow_test_results(pub):
     assert TestResult.count() == 2
     result = TestResult.select(order_by='id')[1]
     assert result.reason == 'Workflow: New status "new status"'
+
+
+def test_workflow_documentation(pub):
+    create_superuser(pub)
+
+    Workflow.wipe()
+    workflow = Workflow(name='Workflow One')
+    status = workflow.add_status(name='New status')
+    status.add_action('anonymise')
+    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
+    workflow.backoffice_fields_formdef.fields = [
+        fields.StringField(id='bo234', label='bo field 1'),
+    ]
+    workflow.variables_formdef = WorkflowVariablesFieldsFormDef(workflow=workflow)
+    workflow.variables_formdef.fields = [
+        fields.StringField(id='va123', label='bo field 1'),
+    ]
+    global_action = workflow.add_global_action('action1')
+    workflow.store()
+
+    app = login(get_app(pub))
+    resp = app.get(workflow.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    assert app.post_json(workflow.get_admin_url() + 'update-documentation', {}).json.get('err') == 1
+    resp = app.post_json(workflow.get_admin_url() + 'update-documentation', {'content': ''})
+    assert resp.json == {'err': 0, 'empty': True, 'changed': False}
+    resp = app.post_json(workflow.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    workflow.refresh_from_storage()
+    assert workflow.documentation == '<p>doc</p>'
+
+    # check forbidden HTML is cleaned
+    resp = app.post_json(
+        workflow.get_admin_url() + 'update-documentation',
+        {'content': '<p>iframe</p><iframe src="xx"></iframe>'},
+    )
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    workflow.refresh_from_storage()
+    assert workflow.documentation == '<p>iframe</p>'
+
+    resp = app.get(workflow.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')
+
+    resp = app.get(workflow.get_admin_url() + 'variables/fields/')
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(
+        workflow.get_admin_url() + 'variables/fields/update-documentation', {'content': '<p>doc</p>'}
+    )
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    workflow.refresh_from_storage()
+    assert workflow.variables_formdef.documentation == '<p>doc</p>'
+    resp = app.get(workflow.get_admin_url() + 'variables/fields/')
+    assert resp.pyquery('.documentation:not([hidden])')
+
+    resp = app.get(workflow.get_admin_url() + 'variables/fields/va123/')
+    assert resp.pyquery('.documentation[hidden]')
+    assert resp.pyquery('#sidebar[hidden]')
+    resp = app.post_json(
+        workflow.get_admin_url() + 'variables/fields/va123/update-documentation', {'content': '<p>doc</p>'}
+    )
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    workflow.refresh_from_storage()
+    assert workflow.variables_formdef.fields[0].documentation == '<p>doc</p>'
+    resp = app.get(workflow.get_admin_url() + 'variables/fields/va123/')
+    assert resp.pyquery('.documentation:not([hidden])')
+    assert resp.pyquery('#sidebar:not([hidden])')
+
+    resp = app.get(workflow.get_admin_url() + 'backoffice-fields/fields/')
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(
+        workflow.get_admin_url() + 'backoffice-fields/fields/update-documentation', {'content': '<p>doc</p>'}
+    )
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    workflow.refresh_from_storage()
+    assert workflow.backoffice_fields_formdef.documentation == '<p>doc</p>'
+    resp = app.get(workflow.get_admin_url() + 'backoffice-fields/fields/')
+    assert resp.pyquery('.documentation:not([hidden])')
+
+    resp = app.get(workflow.get_admin_url() + 'backoffice-fields/fields/bo234/')
+    assert resp.pyquery('.documentation[hidden]')
+    assert resp.pyquery('#sidebar[hidden]')
+    resp = app.post_json(
+        workflow.get_admin_url() + 'backoffice-fields/fields/bo234/update-documentation',
+        {'content': '<p>doc</p>'},
+    )
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    workflow.refresh_from_storage()
+    assert workflow.backoffice_fields_formdef.fields[0].documentation == '<p>doc</p>'
+    resp = app.get(workflow.get_admin_url() + 'backoffice-fields/fields/bo234/')
+    assert resp.pyquery('.documentation:not([hidden])')
+    assert resp.pyquery('#sidebar:not([hidden])')
+
+    resp = app.get(global_action.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(global_action.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    workflow.refresh_from_storage()
+    assert workflow.global_actions[0].documentation == '<p>doc</p>'
+    resp = app.get(global_action.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')
+
+    resp = app.get(status.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(status.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    workflow.refresh_from_storage()
+    assert workflow.possible_status[0].documentation == '<p>doc</p>'
+    resp = app.get(status.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')
+
+    resp = app.get(workflow.get_admin_url() + 'inspect')
+    assert resp.pyquery('.documentation').length == 5

tests/admin_pages/test_workflow_tests.py

@@ -1,5 +1,4 @@
 import datetime
-import os
 
 import pytest
 from django.utils.html import escape
@@ -24,12 +23,6 @@ def pub():
     pub.cfg['identification'] = {'methods': ['password']}
     pub.write_cfg()
 
-    if not pub.site_options.has_section('options'):
-        pub.site_options.add_section('options')
-    pub.site_options.set('options', 'enable-workflow-tests', 'true')
-    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
-        pub.site_options.write(fd)
-
     pub.user_class.wipe()
     FormDef.wipe()
     TestDef.wipe()
@@ -41,34 +34,6 @@ def teardown_module(module):
     clean_temporary_pub()
 
 
-def test_workflow_tests_link_feature_flag(pub):
-    create_superuser(pub)
-
-    formdef = FormDef()
-    formdef.name = 'test title'
-    formdef.store()
-
-    formdata = formdef.data_class()()
-    formdata.just_created()
-    formdata.store()
-
-    testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.name = 'First test'
-    testdef.store()
-
-    app = login(get_app(pub))
-
-    resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
-    assert 'Workflow tests' in resp.text
-
-    pub.site_options.set('options', 'enable-workflow-tests', 'false')
-    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
-        pub.site_options.write(fd)
-
-    resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
-    assert 'Workflow tests' not in resp.text
-
-
 def test_workflow_tests_options(pub):
     create_superuser(pub)
     user = pub.user_class(name='test user')

tests/admin_pages/test_wscall.py

@@ -36,7 +36,6 @@ def teardown_module(module):
 def wscall():
     NamedWsCall.wipe()
     wscall = NamedWsCall(name='xxx')
-    wscall.description = 'description'
     wscall.notify_on_errors = True
     wscall.record_on_errors = True
     wscall.request = {
@@ -68,7 +67,6 @@ def test_wscalls_new(pub, value):
     assert resp.form['notify_on_errors'].value is None
     assert resp.form['record_on_errors'].value == 'yes'
     resp.form['name'] = 'a new webservice call'
-    resp.form['description'] = 'description'
     resp.form['notify_on_errors'] = value
     resp.form['record_on_errors'] = value
     resp.form['request$url'] = 'http://remote.example.net/json'
@@ -111,14 +109,12 @@ def test_wscalls_edit(pub, wscall):
     assert resp.form['notify_on_errors'].value == 'yes'
     assert resp.form['record_on_errors'].value == 'yes'
     assert 'slug' in resp.form.fields
-    resp.form['description'] = 'bla bla bla'
     resp.form['notify_on_errors'] = False
     resp.form['record_on_errors'] = False
     resp = resp.form.submit('submit')
     assert resp.location == 'http://example.net/backoffice/settings/wscalls/xxx/'
     resp = resp.follow()
 
-    assert NamedWsCall.get('xxx').description == 'bla bla bla'
     assert NamedWsCall.get('xxx').notify_on_errors is False
     assert NamedWsCall.get('xxx').record_on_errors is False
 
@@ -235,7 +231,6 @@ def test_wscalls_empty_param_values(pub):
     resp = app.get('/backoffice/settings/wscalls/')
     resp = resp.click('New webservice call')
     resp.form['name'] = 'a new webservice call'
-    resp.form['description'] = 'description'
     resp.form['request$qs_data$element0key'] = 'foo'
     resp.form['request$post_data$element0key'] = 'bar'
     resp = resp.form.submit('submit').follow()
@@ -253,7 +248,6 @@ def test_wscalls_timeout(pub):
     resp = app.get('/backoffice/settings/wscalls/')
     resp = resp.click('New webservice call')
     resp.form['name'] = 'a new webservice call'
-    resp.form['description'] = 'description'
     resp.form['request$timeout'] = 'plop'
     resp = resp.form.submit('submit')
     assert resp.pyquery('[data-widget-name="request$timeout"].widget-with-error')
@@ -300,3 +294,22 @@ def test_wscalls_usage(pub, wscall):
     formdef.store()
     resp = app.get(usage_url)
     assert 'No usage detected.' in resp.text
+
+
+def test_wscall_documentation(pub):
+    create_superuser(pub)
+
+    NamedWsCall.wipe()
+    wscall = NamedWsCall(name='foobar')
+    wscall.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get(wscall.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(wscall.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    wscall.refresh_from_storage()
+    assert wscall.documentation == '<p>doc</p>'
+    resp = app.get(wscall.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')

tests/api/test_all.py

@@ -145,6 +145,26 @@ def test_validate_condition(pub):
     resp = get_app(pub).get('/api/validate-condition?type=unknown&value_unknown=2')
     assert resp.json['msg'] == 'unknown condition type'
 
+    resp = get_app(pub).get('/api/validate-condition?type=django&value_django=today > "2023"')
+    assert resp.json == {'msg': ''}
+    resp = get_app(pub).get(
+        '/api/validate-condition?type=django&value_django=today > "2023"&warn-on-datetime=false'
+    )
+    assert resp.json == {'msg': ''}
+    resp = get_app(pub).get(
+        '/api/validate-condition?type=django&value_django=today > "2023"&warn-on-datetime=true'
+    )
+    assert resp.json['msg'].startswith('Warning: conditions are only evaluated when entering')
+
+    resp = get_app(pub).get(
+        '/api/validate-condition?type=django&value_django=x|age_in_days > 10&warn-on-datetime=true'
+    )
+    assert resp.json['msg'].startswith('Warning: conditions are only evaluated when entering')
+    resp = get_app(pub).get(
+        '/api/validate-condition?type=django&value_django=x|age_in_days|abs > 10&warn-on-datetime=true'
+    )
+    assert resp.json['msg'].startswith('Warning: conditions are only evaluated when entering')
+
 
 def test_reverse_geocoding(pub):
     with responses.RequestsMock() as rsps:
@@ -285,3 +305,57 @@ def test_afterjobs_base_directory(pub):
     get_app(pub).get('/api/jobs/', status=403)
     # base directory is 404
     get_app(pub).get(sign_url('/api/jobs/?orig=coucou', '1234'), status=404)
+
+
+def test_preview_payload_structure(pub, admin_user):
+    get_app(pub).get('/api/preview-payload-structure', status=403)
+    app = login(get_app(pub))
+    resp = app.get('/api/preview-payload-structure')
+
+    assert resp.pyquery('div.payload-preview').length == 1
+    assert '<h2>Payload structure preview</h2>' in resp.text
+    assert resp.pyquery('div.payload-preview').text() == '{}'
+    params = {
+        'request$post_data$added_elements': 1,
+        'request$post_data$element1key': 'user/first_name',
+        'request$post_data$element1value$value_template': 'Foo',
+        'request$post_data$element1value$value_python': '',
+        'request$post_data$element2key': 'user/last_name',
+        'request$post_data$element2value$value_template': 'Bar',
+        'request$post_data$element2value$value_python': '',
+        'request$post_data$element3key': 'user/0',
+    }
+    resp = app.get('/api/preview-payload-structure', params=params)
+    assert resp.pyquery('div.payload-preview div.errornotice').length == 0
+    assert resp.pyquery('div.payload-preview').text() == '{"user": {"first_name": "Foo","last_name": "Bar"}}'
+    params.update(
+        {
+            'request$post_data$element3value$value_template': 'value',
+            'request$post_data$element3value$value_python': '',
+        }
+    )
+    resp = app.get('/api/preview-payload-structure', params=params)
+
+    assert resp.pyquery('div.payload-preview div.errornotice').length == 1
+    assert 'Unable to preview payload' in resp.pyquery('div.payload-preview div.errornotice').text()
+    assert (
+        'Following error occured: there is a mix between lists and dicts'
+        in resp.pyquery('div.payload-preview div.errornotice').text()
+    )
+
+    params = {
+        'post_data$element1key': '0/0',
+        'post_data$element1value$value_template': 'Foo',
+        'post_data$element1value$value_python': '',
+        'post_data$element2key': '0/1',
+        'post_data$element2value$value_template': '{{ form_name }}',
+        'post_data$element2value$value_python': '',
+        'post_data$element3key': '1/0',
+        'post_data$element3value$value_template': '',
+        'post_data$element10key': '1/1',
+        'post_data$element10value$value_template': '10',
+        'post_data$element100key': '1/2',
+        'post_data$element100value$value_template': '100',
+    }
+    resp = app.get('/api/preview-payload-structure', params=params)
+    assert resp.pyquery('div.payload-preview').text() == '[["Foo",{{ form_name }}],["","10","100"]]'

tests/api/test_formdata.py

@@ -29,6 +29,7 @@ from wcs.qommon import ods
 from wcs.qommon.http_request import HTTPRequest
 from wcs.qommon.upload_storage import PicklableUpload
 from wcs.wf.comment import WorkflowCommentPart
+from wcs.wf.form import WorkflowFormEvolutionPart, WorkflowFormFieldsFormDef
 from wcs.workflows import AttachmentEvolutionPart, Workflow, WorkflowBackofficeFieldsFormDef
 
 from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
@@ -472,6 +473,106 @@ def test_formdata_backoffice_fields(pub, local_user):
     assert resp.json['workflow']['fields']['backoffice_blah'] == 'Hello world'
 
 
+@pytest.mark.parametrize('user', ['query-email', 'api-access'])
+@pytest.mark.parametrize('auth', ['signature', 'http-basic'])
+def test_formdata_workflow_form(pub, local_user, user, auth):
+    app = get_app(pub)
+
+    if user == 'api-access':
+        ApiAccess.wipe()
+        access = ApiAccess()
+        access.name = 'test'
+        access.access_identifier = 'test'
+        access.access_key = '12345'
+        access.store()
+
+        if auth == 'http-basic':
+
+            def get_url(url, **kwargs):
+                app.set_authorization(('Basic', ('test', '12345')))
+                return app.get(url, **kwargs)
+
+        else:
+
+            def get_url(url, **kwargs):
+                return app.get(sign_uri(url, orig=access.access_identifier, key=access.access_key), **kwargs)
+
+    else:
+        if auth == 'http-basic':
+            pytest.skip('http basic authentication requires ApiAccess')
+
+        def get_url(url, **kwargs):
+            return app.get(sign_uri(url, user=local_user), **kwargs)
+
+    role = pub.role_class(name='test')
+    role.id = '123'
+    role.store()
+
+    Workflow.wipe()
+    workflow = Workflow(name='foo')
+    st = workflow.add_status('st1')
+    form_action = st.add_action('form')
+    form_action.varname = 'blah'
+    form_action.formdef = WorkflowFormFieldsFormDef(item=form_action)
+    form_action.formdef.fields = [
+        fields.FileField(id='1', label='file', varname='file'),
+        fields.StringField(id='2', label='str', varname='str'),
+    ]
+    workflow.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = []
+    formdef.workflow_id = workflow.id
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.store()
+
+    formdef.data_class().wipe()
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    data = {'1': PicklableUpload('test.txt', 'text/plain'), '2': 'text'}
+    data['1'].receive([b'hello world wf form'])
+    formdata.evolution[-1].parts = [
+        WorkflowFormEvolutionPart(form_action, data),
+    ]
+    formdata.store()
+
+    if user == 'api-access':
+        access.roles = [role]
+        access.store()
+    else:
+        local_user.roles = [role.id]
+        local_user.store()
+
+    resp = get_url('/api/forms/test/%s/' % formdata.id, status=200)
+    assert resp.json['evolution'][0]['parts'] == [
+        {
+            'data': {
+                'file': 'test.txt',
+                'file_raw': {
+                    'content': 'aGVsbG8gd29ybGQgd2YgZm9ybQ==',
+                    'content_type': 'text/plain',
+                    'filename': 'test.txt',
+                },
+                'file_url': None,
+                'str': 'text',
+            },
+            'key': 'blah',
+            'type': 'workflow-form',
+        }
+    ]
+
+    resp = get_url('/api/forms/test/%s/?include-files-content=off' % formdata.id, status=200)
+    assert resp.json['evolution'][0]['parts'] == [
+        {
+            'data': {'file': 'test.txt', 'file_url': None, 'str': 'text'},
+            'key': 'blah',
+            'type': 'workflow-form',
+        }
+    ]
+
+
 def test_formdata_duplicated_varnames(pub, local_user):
     pub.role_class.wipe()
     role = pub.role_class(name='test')

tests/api/test_formdef.py

@@ -13,7 +13,7 @@ from django.utils.encoding import force_str
 from django.utils.timezone import localtime
 from quixote import get_publisher
 
-from wcs import fields, qommon, sql
+from wcs import fields, qommon
 from wcs.api_access import ApiAccess
 from wcs.api_utils import sign_url
 from wcs.blocks import BlockDef
@@ -429,8 +429,8 @@ def test_backoffice_submission_formdef_list_search(pub, local_user, access, auth
     resp = get_url('/api/formdefs/?backoffice-submission=on&q=test')
     assert len(resp.json['data']) == 2
 
-    #resp = get_url('/api/formdefs/?backoffice-submission=on&q=tes')
-    #assert len(resp.json['data']) == 2
+    resp = get_url('/api/formdefs/?backoffice-submission=on&q=tes')
+    assert len(resp.json['data']) == 2
 
     resp = get_url('/api/formdefs/?backoffice-submission=on&q=xyz')
     assert len(resp.json['data']) == 0
@@ -450,11 +450,6 @@ def test_backoffice_submission_formdef_list_search(pub, local_user, access, auth
     formdef.backoffice_submission_roles = [role.id]
     formdef.fields = []
     formdef.store()
-    conn, cur = sql.get_connection_and_cursor()
-    cur.execute('SELECT * FROM wcs_search_tokens')
-    print(cur.fetchall())
-    cur.execute("SELECT wcs_tsquery('salubrité')")
-    print(cur.fetchall())
     resp = get_url('/api/formdefs/?backoffice-submission=on&q=salubrité')
     assert len(resp.json['data']) == 1
 

tests/backoffice_pages/test_all.py

@@ -1603,8 +1603,18 @@ def test_backoffice_map(pub):
     resp = app.get('/backoffice/management/form-title/')
     assert 'Plot on a Map' in resp.text
     resp = resp.click('Plot on a Map')
-    assert 'data-geojson-url' in resp.text
-    assert 'tiles.entrouvert.org/' in resp.text
+    assert (
+        resp.pyquery('.qommon-map')[0].attrib['data-geojson-url']
+        == 'http://example.net/backoffice/management/form-title/geojson?'
+    )
+    assert (
+        resp.pyquery('.qommon-map')[0].attrib['data-tile-urltemplate']
+        == 'https://tiles.entrouvert.org/hdm/{z}/{x}/{y}.png'
+    )
+    assert (
+        resp.pyquery('.qommon-map')[0].attrib['data-map-attribution']
+        == 'Map data &copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a>'
+    )
 
     if not pub.site_options.has_section('options'):
         pub.site_options.add_section('options')
@@ -1615,7 +1625,10 @@ def test_backoffice_map(pub):
 
     resp = app.get('/backoffice/management/form-title/')
     resp = resp.click('Plot on a Map')
-    assert 'tile.example.net/' in resp.text
+    assert (
+        resp.pyquery('.qommon-map')[0].attrib['data-tile-urltemplate']
+        == 'https://{s}.tile.example.net/{z}/{x}/{y}.png'
+    )
 
     # check query string is kept
     resp = app.get('/backoffice/management/form-title/map?filter=all')
@@ -6193,3 +6206,50 @@ def test_backoffice_form_tracking_code_workflow_action(pub):
     formdata.refresh_from_storage()
     assert isinstance(formdata.evolution[-1].parts[0], WorkflowCommentPart)
     assert formdata.evolution[-1].who == str(user.id)
+
+
+def test_backoffice_compact_table_view(pub):
+    user = create_user(pub)
+
+    workflow = Workflow(name='workflow')
+    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
+    workflow.backoffice_fields_formdef.fields = [
+        fields.FileField(id='bo1', label='bo field 1'),
+    ]
+    workflow.add_status('status1')
+    workflow.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [fields.StringField(id='1', label='string')]
+    formdef.workflow = workflow
+    formdef.workflow_roles = {'_receiver': user.roles[0]}
+    formdef.store()
+    formdef.data_class().wipe()
+
+    formdata = formdef.data_class()()
+    formdata.data = {'1': 'data', 'bo1': 'data'}
+    formdata.just_created()
+    formdata.store()
+
+    if not pub.site_options.has_section('options'):
+        pub.site_options.add_section('options')
+    pub.site_options.set('options', 'enable-compact-dataview', 'true')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+
+    app = get_app(pub)
+
+    resp = login(app).get(formdata.get_backoffice_url())
+    assert resp.pyquery('#compact-table-dataview-switch input')
+    assert not resp.pyquery('#compact-table-dataview-switch input:checked')
+    assert resp.pyquery('.dataview:not(.compact-dataview)')
+    assert not resp.pyquery('.dataview.compact-dataview')
+
+    app.post_json('/api/user/preferences', {'use-compact-table-dataview': True}, status=200)
+    resp = app.get(formdata.get_backoffice_url())
+    assert resp.pyquery('#compact-table-dataview-switch input')
+    assert resp.pyquery('#compact-table-dataview-switch input:checked')
+    assert not resp.pyquery('.dataview:not(.compact-dataview)')
+    assert resp.pyquery('.dataview.compact-dataview')

tests/backoffice_pages/test_applications.py

@@ -734,7 +734,7 @@ def test_blockdefs(pub, application_with_icon, application_without_icon, icon):
         )
     else:
         assert len(resp.pyquery('h3:contains("Applications") + .button-paragraph img')) == 0
-    assert 'Field blocks outside applications' in resp
+    assert 'Blocks of fields outside applications' in resp
 
     # check application view
     resp = resp.click(href='application/%s/' % application.slug)
@@ -751,7 +751,7 @@ def test_blockdefs(pub, application_with_icon, application_without_icon, icon):
 
     # check elements outside applications
     resp = resp.click(href='application/')
-    assert resp.pyquery('h2').text() == 'Field blocks outside applications'
+    assert resp.pyquery('h2').text() == 'Blocks of fields outside applications'
     assert len(resp.pyquery('ul.objects-list li')) == 1
     assert resp.pyquery('ul.objects-list li:nth-child(1)').text() == 'block1'
 

tests/backoffice_pages/test_carddata.py

@@ -814,7 +814,7 @@ def test_backoffice_cards_import_data_csv_blockfield(pub):
     assert sample_resp.text.splitlines()[0] == '"String","Block"'
     assert (
         sample_resp.text.splitlines()[1]
-        == '"value","will be ignored - type Field Block (foobar) not supported"'
+        == '"value","will be ignored - type Block of fields (foobar) not supported"'
     )
 
     # block is required, error

tests/backoffice_pages/test_columns.py

@@ -149,14 +149,14 @@ def test_backoffice_submission_agent_column(pub):
 
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/')
-    assert 'submission_agent' not in resp.forms['listing-settings'].fields
+    assert 'submission-agent' not in resp.forms['listing-settings'].fields
 
     formdef.backoffice_submission_roles = [role]
     formdef.store()
 
     resp = app.get('/backoffice/management/form-title/')
     assert resp.text.count('</th>') == 6  # four columns
-    resp.forms['listing-settings']['submission_agent'].checked = True
+    resp.forms['listing-settings']['submission-agent'].checked = True
     resp = resp.forms['listing-settings'].submit()
     assert resp.text.count('</th>') == 7  # five columns
     assert resp.text.count('data-link') == 1  # 1 row
@@ -172,7 +172,7 @@ def test_backoffice_submission_agent_column(pub):
         formdata.store()
 
     resp = app.get('/backoffice/management/form-title/')
-    resp.forms['listing-settings']['submission_agent'].checked = True
+    resp.forms['listing-settings']['submission-agent'].checked = True
     resp = resp.forms['listing-settings'].submit()
     assert resp.text.count('>agent<') == 1
 

tests/backoffice_pages/test_filters.py

@@ -1426,52 +1426,102 @@ def test_backoffice_submission_agent_filter(pub):
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/?limit=100')
     # enable submission-agent column
-    resp.forms['listing-settings']['submission_agent'].checked = True
+    resp.forms['listing-settings']['submission-agent'].checked = True
     resp = resp.forms['listing-settings'].submit()
-    assert resp.text.count('>userA<') > 0
-    assert resp.text.count('>userB<') > 0
-    # check the filter is hidden
-    assert resp.pyquery.find('li[hidden] input[name=filter-submission-agent]')
+    assert resp.text.count('>userA</td>') > 0
+    assert resp.text.count('>userB</td>') > 0
 
-    base_url = resp.request.url
-    resp = app.get(base_url + '&filter-submission-agent=on&filter-submission-agent-value=%s' % user1.id)
-    assert resp.text.count('>userA<') > 0
-    assert resp.text.count('>userB<') == 0
-    assert resp.text.count('<tr') == 2
-    assert resp.pyquery.find('input[value=userA]')  # displayed in sidebar
-    # check it persits on filter changes
+    # enable submission-agent filter
+    resp.forms['listing-settings']['filter-submission-agent'].checked = True
     resp = resp.forms['listing-settings'].submit()
-    assert resp.text.count('>userA<') > 0
-    assert resp.text.count('>userB<') == 0
-    assert resp.text.count('<tr') == 2
 
-    resp = app.get(base_url + '&filter-submission-agent=on&filter-submission-agent-value=%s' % user2.id)
-    assert resp.text.count('>userA<') == 0
-    assert resp.text.count('>userB<') > 0
-    assert resp.text.count('<tr') == 2
+    # check everything is still displayed
+    assert resp.forms['listing-settings']['filter-submission-agent-value'].value == ''
+    assert resp.text.count('>userA</td>') > 0
+    assert resp.text.count('>userB</td>') > 0
 
-    resp = app.get(
-        '/backoffice/management/form-title/?limit=100&filter-submission-agent=on&filter-submission-agent-value=%s'
-        % user2.id
-    )
-    assert resp.text.count('<tr') == 2
+    # check available filter values
+    assert [x.text for x in resp.pyquery('select[name="filter-submission-agent-value"] option')] == [
+        None,
+        'Current user',
+        'admin',
+    ]
+
+    # add userA and userB to role for backoffice submission
+    user1.roles = user.roles
+    user1.store()
+    user2.roles = user.roles
+    user2.store()
+
+    # refresh
+    resp = resp.forms['listing-settings'].submit()
+    assert [x.text for x in resp.pyquery('select[name="filter-submission-agent-value"] option')] == [
+        None,
+        'Current user',
+        'admin',
+        'userA',
+        'userB',
+    ]
+
+    resp.forms['listing-settings']['filter-submission-agent-value'].value = str(user1.id)
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('>userA</td>') > 0
+    assert resp.text.count('>userB</td>') == 0
+    assert resp.pyquery('tbody tr').length == 1
+    # check it persists on filter changes
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('>userA</td>') > 0
+    assert resp.text.count('>userB</td>') == 0
+    assert resp.pyquery('tbody tr').length == 1
+
+    resp.forms['listing-settings']['filter-submission-agent-value'].value = str(user2.id)
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('>userA</td>') == 0
+    assert resp.text.count('>userB</td>') > 0
+    assert resp.pyquery('tbody tr').length == 1
+
+    # filter on current user
+    resp.forms['listing-settings']['filter-submission-agent-value'].value = '__current__'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('>userA</td>') == 0
+    assert resp.text.count('>userB</td>') == 0
+    assert resp.pyquery('tbody tr').length == 0
+
+    old_formdata_agent_id, formdata.submission_agent_id = formdata.submission_agent_id, user.id
+    formdata.store()
+
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('>userA</td>') == 0
+    assert resp.text.count('>admin</td>') == 1
+    assert resp.pyquery('tbody tr').length == 1
+
+    # restore second formadata user
+    formdata.submission_agent_id = old_formdata_agent_id
+    formdata.store()
 
     # filter on uuid
     user1.name_identifiers = ['0123456789']
     user1.store()
-    resp = app.get(base_url + '&filter-submission-agent-uuid=0123456789')
-    assert resp.text.count('>userA<') > 0
-    assert resp.text.count('>userB<') == 0
-    assert resp.pyquery.find('input[value=userA]')  # displayed in sidebar
+    resp = app.get(
+        '/backoffice/management/form-title/?filter-submission-agent-uuid=0123456789&submission-agent=on'
+    )
+    assert resp.forms['listing-settings']['filter-submission-agent-value'].value == str(user1.id)
+    assert resp.text.count('>userA</td>') > 0
+    assert resp.text.count('>userB</td>') == 0
     # check it persists on filter changes
     resp = resp.forms['listing-settings'].submit()
-    assert resp.text.count('>userA<') > 0
-    assert resp.text.count('>userB<') == 0
+    assert resp.text.count('>userA</td>') > 0
+    assert resp.text.count('>userB</td>') == 0
 
     # check with unknown uuid
-    resp = app.get(base_url + '&filter-submission-agent-uuid=XXX')
-    assert resp.text.count('>userA<') == 0
-    assert resp.text.count('>userB<') == 0
+    resp = app.get('/backoffice/management/form-title/?filter-submission-agent-uuid=XXX&submission-agent=on')
+    assert resp.forms['listing-settings']['filter-submission-agent-value'].value == '-1'
+    assert resp.text.count('>userA</td>') == 0
+    assert resp.text.count('>userB</td>') == 0
+    # check it persists on submits
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('>userA</td>') == 0
+    assert resp.text.count('>userB</td>') == 0
 
 
 def test_workflow_function_filter(pub):

tests/form_pages/test_all.py

@@ -516,7 +516,7 @@ def test_form_submit(pub):
     assert 'The form has been recorded' in next_page.text
     assert 'None' not in next_page.text
     assert formdef.data_class().count() == 1
-    assert '<div class="section foldable folded" id="summary">' in next_page.text
+    assert next_page.pyquery('#summary').attr['class'] == 'section foldable folded'
     assert next_page.pyquery('#summary .disclose-message')
     assert formdef.data_class().select()[0].submission_context['language'] == 'en'
 
@@ -1362,7 +1362,7 @@ def test_form_submit_with_user(pub, emails):
     next_page = next_page.follow()
     assert 'The form has been recorded' in next_page.text
     assert formdef.data_class().count() == 1
-    assert '<div class="section foldable folded" id="summary">' in next_page.text
+    assert next_page.pyquery('#summary').attr['class'] == 'section foldable folded'
     # check the user received a copy by email
     assert emails.get('New form (test)')
     assert emails.get('New form (test)')['email_rcpt'] == ['foo@localhost']
@@ -5197,10 +5197,34 @@ def test_form_honeypot(pub):
     resp.forms[0]['f0'] = 'plop'
     resp.forms[0]['f00'] = 'honey?'
     resp = resp.forms[0].submit('submit')
-    assert 'Honey pot should be left untouched.' in resp
+    assert 'Honey pots should be left untouched.' in resp
     assert formdef.data_class().count() == 0  # check no drafts have been saved
 
 
+def test_form_honeypot_level2(pub):
+    pub.load_site_options()
+    pub.site_options.set('options', 'honeypots', 'level2')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+    formdef = create_formdef()
+    formdef.fields = [fields.StringField(id='0', label='string', required=False)]
+    formdef.store()
+    formdef.data_class().wipe()
+    resp = get_app(pub).get('/test/')
+    resp.forms[0]['f0'] = 'plop'
+    assert resp.forms[0]['f002'].value == ''
+    resp = resp.forms[0].submit('submit')
+    assert 'Honey pots should be left untouched.' in resp
+    assert formdef.data_class().count() == 0  # check no drafts have been saved
+
+    resp = get_app(pub).get('/test/')
+    resp.forms[0]['f0'] = 'plop'
+    resp.forms[0]['f002'].value = resp.pyquery('form')[0].attrib['data-honey-pot-value']
+    resp = resp.forms[0].submit('submit')  # -> validation
+    resp = resp.forms[0].submit('submit')  # -> submit
+    assert formdef.data_class().count() == 1
+
+
 def test_structured_workflow_options(pub):
     create_user_and_admin(pub)
 

tests/form_pages/test_draft.py

@@ -596,6 +596,92 @@ def test_nothing_to_update_add_row(pub):
     assert 'Technical error saving draft, please try again.' in resp.text
 
 
+def test_draft_with_block_data(pub):
+    FormDef.wipe()
+    BlockDef.wipe()
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [fields.StringField(id='123', required=True, label='Test1')]
+    block.store()
+
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.PageField(id='0', label='1st page'),
+        fields.PageField(id='2', label='2nd page'),
+        fields.BlockField(id='3', label='block', block_slug='foobar', max_items=3),
+    ]
+    formdef.store()
+
+    formdef.data_class().wipe()
+
+    create_user(pub)
+    app = get_app(pub)
+    resp = login(app, username='foo', password='foo').get('/test/')
+
+    resp = resp.form.submit('submit')  # -> page 2
+    resp.form['f3$element0$f123'] = 'foo'
+    resp = resp.form.submit('submit')  # -> confirmation page
+
+    resp = app.get('/test/')
+    resp = resp.click('Continue with draft').follow()
+    assert resp.forms[1]['f3$element0$f123'].value == 'foo'
+    resp = resp.forms[1].submit('previous')  # -> page 2
+    assert resp.forms[1]['f3$element0$f123'].value == 'foo'
+    resp = resp.forms[1].submit('submit')  # -> confirmation page
+    resp = resp.forms[1].submit('submit')  # -> submit
+    assert formdef.data_class().count() == 1
+    formdata = formdef.data_class().select()[0]
+    assert formdata.data == {
+        '3': {'data': [{'123': 'foo'}], 'schema': {'123': 'string'}},
+        '3_display': 'foobar',
+    }
+
+
+def test_draft_with_block_data_tracking_code(pub):
+    FormDef.wipe()
+    BlockDef.wipe()
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [fields.StringField(id='123', required=True, label='Test1')]
+    block.store()
+
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.PageField(id='0', label='1st page'),
+        fields.PageField(id='2', label='2nd page'),
+        fields.BlockField(id='3', label='block', block_slug='foobar', max_items=3),
+    ]
+    formdef.enable_tracking_codes = True
+    formdef.store()
+
+    formdef.data_class().wipe()
+
+    resp = get_app(pub).get('/test/')
+    resp = resp.form.submit('submit')  # -> page 2
+    resp.form['f3$element0$f123'] = 'foo'
+    resp = resp.form.submit('submit')  # -> confirmation page
+    tracking_code = get_displayed_tracking_code(resp)
+
+    resp = get_app(pub).get('/')
+    resp.form['code'] = tracking_code
+    resp = resp.form.submit().follow().follow().follow()
+    assert resp.forms[1]['f3$element0$f123'].value == 'foo'
+    resp = resp.forms[1].submit('previous')  # -> page 2
+    assert resp.forms[1]['f3$element0$f123'].value == 'foo'
+    resp = resp.forms[1].submit('submit')  # -> confirmation page
+    resp = resp.forms[1].submit('submit')  # -> submit
+    assert formdef.data_class().count() == 1
+    formdata = formdef.data_class().select()[0]
+    assert formdata.data == {
+        '3': {'data': [{'123': 'foo'}], 'schema': {'123': 'string'}},
+        '3_display': 'foobar',
+    }
+
+
 def test_draft_store_page_id(pub):
     formdef = create_formdef()
     formdef.enable_tracking_codes = True

tests/form_pages/test_map_field.py

@@ -53,10 +53,19 @@ def test_form_map_field_back_and_submit(pub):
         ),
     ]
     formdef.store()
-    resp = get_app(pub).get('/test/')
     formdef.data_class().wipe()
+    resp = get_app(pub).get('/test/')
     assert 'qommon.map.js' in resp.text
     assert 'qommon.geolocation.js' in resp.text
+    assert (
+        resp.pyquery('.qommon-map')[0].attrib['data-tile-urltemplate']
+        == 'https://tiles.entrouvert.org/hdm/{z}/{x}/{y}.png'
+    )
+    assert (
+        resp.pyquery('.qommon-map')[0].attrib['data-map-attribution']
+        == 'Map data &copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a>'
+    )
+
     # with a real user interaction this would get set by javascript
     resp.forms[0]['f0$latlng'].value = '1.234;-1.234'
     assert 'data-geolocation="road"' in resp.text

tests/form_pages/test_prefill.py

@@ -173,19 +173,10 @@ def test_form_draft_from_prefill(pub, field_type, logged_in):
     assert formdef.data_class().count() == 0
     formdef.data_class().wipe()
 
-    # draft created if there's been some prefilled fields
+    # make sure no draft is created on prefilled fields
     formdef.fields[0].prefill = {'type': 'string', 'value': '{{request.GET.test|default:""}}'}
     formdef.store()
     app.get('/test/?test=hello')
-    assert formdef.data_class().count() == 1
-    formdef.data_class().wipe()
-
-    # unless the call was made from an application
-    app.get('/test/?test=hello', headers={'User-agent': 'python-requests/0'})
-    assert formdef.data_class().count() == 0
-
-    # or a bot
-    app.get('/test/?test=hello', headers={'User-agent': 'Googlebot'})
     assert formdef.data_class().count() == 0
 
     # check there's no leftover draft after submission
@@ -1683,3 +1674,78 @@ def test_form_page_prefill_and_tablerows_field(pub):
     resp = resp.form.submit('submit')
     assert resp.forms[0]['f1'].value == 'HELLO WORLD'
     assert not resp.pyquery('.widget-with-error')
+
+
+def test_form_page_user_data_source(pub):
+    user = create_user(pub)
+
+    NamedDataSource.wipe()
+    data_source = NamedDataSource(name='foobar')
+    data_source.data_source = {'type': 'wcs:users'}
+    data_source.store()
+
+    formdef = create_formdef()
+    formdef.data_class().wipe()
+
+    for prefill_value in ('{{ session_user }}', '{{ session_user_id }}'):
+        formdef.fields = [
+            fields.ItemField(
+                id='1',
+                label='item',
+                varname='item',
+                hint='help text',
+                required=False,
+                data_source={'type': data_source.slug},
+                prefill={'type': 'string', 'value': prefill_value},
+            )
+        ]
+        formdef.store()
+
+        resp = get_app(pub).get('/test/')
+        assert resp.form['f1'].value == ''
+        assert 'invalid value selected' not in resp.text
+
+        resp = login(get_app(pub), username='foo', password='foo').get('/test/')
+        assert resp.form['f1'].value == str(user.id)
+        assert 'invalid value selected' not in resp.text
+
+
+def test_form_page_template_block_rows_prefilled_with_form_data(pub):
+    BlockDef.wipe()
+    create_user(pub)
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [
+        fields.StringField(
+            id='123',
+            required=True,
+            label='Test',
+            varname='test',
+            prefill={'type': 'string', 'value': '{{ form_var_foo }}'},
+        ),
+    ]
+    block.store()
+
+    formdef = create_formdef()
+    formdef.data_class().wipe()
+    formdef.fields = [
+        fields.PageField(id='1', label='page1'),
+        fields.StringField(id='2', label='text', varname='foo'),
+        fields.PageField(id='3', label='page2'),
+        fields.BlockField(id='4', label='test', block_slug='foobar', max_items=3),
+    ]
+    formdef.store()
+
+    resp = get_app(pub).get('/test/')
+    resp.form['f2'] = 'foo'
+    resp = resp.form.submit('submit')  # -> second page
+    assert resp.form['f4$element0$f123'].value == 'foo'
+    resp = resp.form.submit('f4$add_element')
+    assert resp.form['f4$element1$f123'].value == 'foo'
+    resp.form['f4$element0$f123'] = 'bar'
+    resp = resp.form.submit('previous')  # -> first page
+    resp.form['f2'] = 'baz'
+    resp = resp.form.submit('submit')  # -> second page
+    assert resp.form['f4$element0$f123'].value == 'bar'  # not changed
+    assert resp.form['f4$element1$f123'].value == 'baz'  # updated

tests/test_comment_template.py

@@ -508,3 +508,40 @@ def test_comment_template_with_category(pub):
     export = ET.tostring(comment_template.export_to_xml(include_id=True))
     comment_template3 = CommentTemplate.import_from_xml_tree(ET.fromstring(export), include_id=True)
     assert comment_template3.category_id is None
+
+
+def test_comment_template_migration(pub):
+    comment_template = CommentTemplate(name='test template')
+    comment_template.description = 'hello'
+    assert comment_template.migrate() is True
+    assert not comment_template.description
+    assert comment_template.documentation == 'hello'
+
+
+def test_comment_template_legacy_xml(pub):
+    comment_template = CommentTemplate(name='test template')
+    comment_template.documentation = 'hello'
+    export = ET.tostring(export_to_indented_xml(comment_template))
+    export = export.replace(b'documentation>', b'description>')
+
+    comment_template2 = CommentTemplate.import_from_xml_tree(ET.fromstring(export))
+    comment_template2.store()
+    comment_template2.refresh_from_storage()
+    assert comment_template2.documentation
+
+
+def test_comment_template_documentation(pub, superuser):
+    CommentTemplate.wipe()
+    comment_template = CommentTemplate(name='foobar')
+    comment_template.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get(comment_template.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(comment_template.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    comment_template.refresh_from_storage()
+    assert comment_template.documentation == '<p>doc</p>'
+    resp = app.get(comment_template.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')

tests/test_fields.py

@@ -48,7 +48,7 @@ def test_get_admin_attributes():
         klass().get_admin_attributes()
 
 
-def test_add_to_form():
+def test_add_to_form(pub):
     for klass in fields.base.field_classes:
         form = Form(use_tokens=False)
         if klass is fields.PageField:
@@ -194,7 +194,7 @@ def test_bool():
     assert fields.BoolField().get_view_value(False) == 'No'
 
 
-def test_bool_stats():
+def test_bool_stats(pub):
     formdef = FormDef()
     formdef.name = 'title'
     formdef.url_name = 'title'
@@ -324,7 +324,7 @@ def test_file():
     assert fields.FileField().get_csv_value(upload) == ['/foo/bar']
 
 
-def test_page():
+def test_page(pub):
     formdef = FormDef()
     formdef.fields = []
     page = fields.PageField()
@@ -539,7 +539,7 @@ def test_map_set_value(pub):
     assert 'form_var_map_lon' not in keys
 
 
-def test_item_render():
+def test_item_render(pub):
     items_kwargs = []
     items_kwargs.append({'items': ['aa', 'ab', 'ac']})
     items_kwargs.append(
@@ -610,7 +610,7 @@ def test_item_render():
         assert str(form.render()).count('<option') == 1
 
 
-def test_item_render_as_autocomplete():
+def test_item_render_as_autocomplete(pub):
     field = fields.ItemField(id='1', label='Foobar', items=['aa', 'ab', 'ac'], display_mode='autocomplete')
     form = Form(use_tokens=False)
     field.add_to_form(form)
@@ -693,7 +693,7 @@ def test_item_render_as_list_with_hint(pub):
     assert len(PyQuery(str(form.render())).find('option')) == 3
 
 
-def test_item_render_as_radio():
+def test_item_render_as_radio(pub):
     items_kwargs = []
     items_kwargs.append({'items': ['aa', 'ab', 'ac']})
     items_kwargs.append(
@@ -760,7 +760,7 @@ def test_item_render_as_radio():
         assert str(form.render()).count('"radio"') == 1
 
 
-def test_item_radio_lengths():
+def test_item_radio_lengths(pub):
     field = fields.ItemField(id='1', label='Foobar', display_mode='radio', items=['aa', 'ab', 'ac'])
     form = Form(use_tokens=False)
     field.add_to_form(form)
@@ -786,7 +786,7 @@ def test_item_radio_lengths():
     assert 'widget-inline-radio' not in str(form.widgets[-1].render())
 
 
-def test_items_render():
+def test_items_render(pub):
     items_kwargs = []
     items_kwargs.append({'items': ['aa', 'ab', 'ac']})
     items_kwargs.append(
@@ -851,7 +851,7 @@ def test_table_rows():
     assert '<td>30.00</td>' in html_table
 
 
-def test_date():
+def test_date(pub):
     assert fields.DateField().convert_value_from_str('2015-01-04') is not None
     assert fields.DateField().convert_value_from_str('04/01/2015') is not None
     assert fields.DateField().convert_value_from_str('') is None
@@ -879,7 +879,7 @@ def test_date_anonymise(pub):
     assert formdata.data.get('0') == time.strptime('2023-03-28', '%Y-%m-%d')
 
 
-def test_file_convert_from_anything():
+def test_file_convert_from_anything(pub):
     assert fields.FileField().convert_value_from_anything(None) is None
 
     value = fields.FileField().convert_value_from_anything({'content': 'hello', 'filename': 'test.txt'})

tests/test_formdef.py

@@ -479,8 +479,8 @@ def test_unused_file_removal_job(pub):
         display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
         display_form.formdef.fields = []
 
-        data = {'blah_1': PicklableUpload('test.txt', 'text/plain')}
-        data['blah_1'].receive([b'hello world wf form'])
+        data = {'1': PicklableUpload('test.txt', 'text/plain')}
+        data['1'].receive([b'hello world wf form'])
         formdata.evolution[-1].parts = [
             WorkflowFormEvolutionPart(display_form, data),
         ]

tests/test_mail_templates.py

@@ -542,3 +542,40 @@ def test_mail_template_with_category(pub):
     export = ET.tostring(mail_template.export_to_xml(include_id=True))
     mail_template3 = MailTemplate.import_from_xml_tree(ET.fromstring(export), include_id=True)
     assert mail_template3.category_id is None
+
+
+def test_mail_template_migration(pub):
+    mail_template = MailTemplate(name='test template')
+    mail_template.description = 'hello'
+    assert mail_template.migrate() is True
+    assert not mail_template.description
+    assert mail_template.documentation == 'hello'
+
+
+def test_mail_template_legacy_xml(pub):
+    mail_template = MailTemplate(name='test template')
+    mail_template.documentation = 'hello'
+    export = ET.tostring(export_to_indented_xml(mail_template))
+    export = export.replace(b'documentation>', b'description>')
+
+    mail_template2 = MailTemplate.import_from_xml_tree(ET.fromstring(export))
+    mail_template2.store()
+    mail_template2.refresh_from_storage()
+    assert mail_template2.documentation
+
+
+def test_mail_template_documentation(pub, superuser):
+    MailTemplate.wipe()
+    mail_template = MailTemplate(name='foobar')
+    mail_template.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get(mail_template.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(mail_template.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    mail_template.refresh_from_storage()
+    assert mail_template.documentation == '<p>doc</p>'
+    resp = app.get(mail_template.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')

tests/test_misc.py

@@ -27,6 +27,7 @@ from wcs.qommon.misc import (
     ellipsize,
     format_time,
     get_as_datetime,
+    mark_spaces,
     normalize_geolocation,
     parse_decimal,
     parse_isotime,
@@ -515,7 +516,7 @@ def test_criteria_repr():
 
 
 def test_related_field_repr():
-    from wcs.backoffice.management import RelatedField
+    from wcs.backoffice.filter_fields import RelatedField
 
     related_field = RelatedField(None, field=StringField(label='foo'), parent_field=StringField(label='bar'))
     assert 'foo' in repr(related_field)
@@ -759,3 +760,19 @@ def test_parse_decimal_keep_none(value, expected):
 def test_parse_decimal_do_raise(value, exception):
     with pytest.raises(exception):
         parse_decimal(value, do_raise=True)
+
+
+def test_mark_spaces():
+    assert mark_spaces('test') == 'test'
+    assert str(mark_spaces('<b>test</b>')) == '&lt;b&gt;test&lt;/b&gt;'
+
+    button_code = (
+        '<button class="toggle-escape-button" role="button" '
+        'title="This line contains invisible characters."></button>'
+    )
+    space = '<span class="escaped-code-point" data-escaped="[U+0020]"><span class="char">&nbsp;</span></span>'
+    tab = '<span class="escaped-code-point" data-escaped="[U+0009]"><span class="char">&nbsp;</span></span>'
+    assert str(mark_spaces(' test ')) == button_code + space + 'test' + space
+    assert str(mark_spaces(' test  ')) == button_code + space + 'test' + space + space
+    assert str(mark_spaces('test\t ')) == button_code + 'test' + tab + space
+    assert str(mark_spaces(' <b>test</b>')) == button_code + space + '&lt;b&gt;test&lt;/b&gt;'

tests/test_testdef.py

@@ -78,6 +78,7 @@ def test_testdef_export_to_xml(pub):
     WebserviceResponse.wipe()
 
     testdef2 = TestDef.import_from_xml(io.BytesIO(testdef_xml), formdef)
+    testdef2.store()
     assert testdef2.name == 'test'
     assert testdef2.object_type == 'formdefs'
     assert testdef2.object_id == str(formdef.id)

tests/test_workflow_import.py

@@ -1101,3 +1101,38 @@ def test_import_root_node_error():
         excinfo.value.msg
         == 'Provided XML file is invalid, it starts with a <wrong_root_node> tag instead of <workflow>'
     )
+
+
+def test_documentation_attributes(pub):
+    Workflow.wipe()
+    workflow = Workflow(name='Workflow One')
+    workflow.documentation = 'doc1'
+    status = workflow.add_status(name='New status')
+    status.documentation = 'doc2'
+    action = status.add_action('anonymise')
+    action.documentation = 'doc3'
+    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
+    workflow.backoffice_fields_formdef.documentation = 'doc4'
+    workflow.backoffice_fields_formdef.fields = [
+        StringField(id='bo234', label='bo field 1'),
+    ]
+    workflow.backoffice_fields_formdef.fields[0].documentation = 'doc5'
+    workflow.variables_formdef = WorkflowVariablesFieldsFormDef(workflow=workflow)
+    workflow.variables_formdef.documentation = 'doc6'
+    workflow.variables_formdef.fields = [
+        StringField(id='va123', label='var field 1'),
+    ]
+    workflow.variables_formdef.fields[0].documentation = 'doc7'
+    global_action = workflow.add_global_action('action1')
+    global_action.documentation = 'doc8'
+    workflow.store()
+
+    wf2 = assert_import_export_works(workflow)
+    assert wf2.documentation == 'doc1'
+    assert wf2.possible_status[0].documentation == 'doc2'
+    assert wf2.possible_status[0].items[0].documentation == 'doc3'
+    assert wf2.backoffice_fields_formdef.documentation == 'doc4'
+    assert wf2.backoffice_fields_formdef.fields[0].documentation == 'doc5'
+    assert wf2.variables_formdef.documentation == 'doc6'
+    assert wf2.variables_formdef.fields[0].documentation == 'doc7'
+    assert wf2.global_actions[0].documentation == 'doc8'

tests/test_workflow_tests.py

@@ -1010,6 +1010,79 @@ def test_workflow_tests_backoffice_fields(pub):
     assert str(excinfo.value) == 'Field bo2 not found (expected value "abc").'
 
 
+def test_workflow_tests_dispatch(pub):
+    role = pub.role_class(name='test role')
+    role.store()
+    user = pub.user_class(name='test user')
+    user.test_uuid = '42'
+    user.roles = [role.id]
+    user.store()
+
+    other_role = pub.role_class(name='test role')
+    other_role.store()
+    other_user = pub.user_class(name='test user')
+    other_user.test_uuid = '43'
+    other_user.roles = [other_role.id]
+    other_user.store()
+
+    workflow = Workflow(name='Workflow One')
+    new_status = workflow.add_status(name='New status')
+    middle_status = workflow.add_status(name='Middle status')
+    end_status = workflow.add_status(name='End status')
+
+    dispatch = new_status.add_action('dispatch')
+    dispatch.dispatch_type = 'manual'
+    dispatch.role_key = '_receiver'
+    dispatch.role_id = role.id
+
+    choice = new_status.add_action('choice')
+    choice.label = 'Go to middle status'
+    choice.status = middle_status.id
+    choice.by = ['_receiver']
+
+    dispatch = middle_status.add_action('dispatch')
+    dispatch.dispatch_type = 'manual'
+    dispatch.role_key = '_receiver'
+    dispatch.role_id = other_role.id
+
+    choice = middle_status.add_action('choice')
+    choice.label = 'Go to end status'
+    choice.status = end_status.id
+    choice.by = ['_receiver']
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertStatus(status_name='New status'),
+        workflow_tests.ButtonClick(button_name='Go to middle status', who='other', who_id=user.test_uuid),
+        workflow_tests.AssertStatus(status_name='Middle status'),
+        workflow_tests.ButtonClick(button_name='Go to end status', who='other', who_id=other_user.test_uuid),
+        workflow_tests.AssertStatus(status_name='End status'),
+    ]
+
+    testdef.run(formdef)
+
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertStatus(status_name='New status'),
+        workflow_tests.ButtonClick(
+            button_name='Go to middle status', who='other', who_id=other_user.test_uuid
+        ),
+    ]
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Button "Go to middle status" is not displayed.'
+
+
 def test_workflow_tests_webservice(pub):
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')

tests/test_wscall.py

@@ -289,6 +289,47 @@ def test_webservice_empty_param_values(http_requests, pub):
     assert http_requests.get_last('body') == '{"toto": ""}'
 
 
+def test_webservice_with_unflattened_payload_keys(http_requests, pub):
+    NamedWsCall.wipe()
+
+    wscall = NamedWsCall()
+    wscall.name = 'Hello world'
+    wscall.request = {
+        'method': 'POST',
+        'url': 'http://remote.example.net/json',
+        'post_data': {'foo/0': 'first', 'foo/1': 'second', 'bar': 'example', 'foo/2': ''},
+    }
+    wscall.store()
+
+    wscall.call()
+    assert http_requests.get_last('url') == 'http://remote.example.net/json'
+    assert http_requests.get_last('body') == '{"bar": "example", "foo": ["first", "second", ""]}'
+    assert http_requests.count() == 1
+
+    wscall.request = {
+        'method': 'POST',
+        'url': 'http://remote.example.net/json',
+        'post_data': {'foo/0': 'first', 'foo/1': 'second', 'foo/bar': 'example'},
+    }
+    wscall.store()
+
+    pub.loggederror_class.wipe()
+    http_requests.empty()
+    wscall.call()
+    assert http_requests.count() == 0
+    assert pub.loggederror_class.count() == 0
+
+    wscall.record_on_errors = True
+    wscall.store()
+    pub.loggederror_class.wipe()
+    wscall.call()
+    assert pub.loggederror_class.count() == 1
+    assert (
+        pub.loggederror_class.select()[0].summary
+        == '[WSCALL] Webservice call failure because unable to unflatten payload keys (there is a mix between lists and dicts)'
+    )
+
+
 def test_webservice_timeout(http_requests, pub):
     NamedWsCall.wipe()
 

tests/workflow/test_backoffice_field.py

@@ -1468,7 +1468,7 @@ def test_set_backoffice_field_invalid_block_value(pub):
     logged_error = pub.loggederror_class.select()[0]
     assert (
         logged_error.summary
-        == 'Failed to set Field Block (foobar) field (bo1), error: invalid value for block (field id: bo1)'
+        == 'Failed to set Block of fields (foobar) field (bo1), error: invalid value for block (field id: bo1)'
     )
 
     formdata = formdef.data_class().get(formdata.id)

tests/workflow/test_carddata.py

@@ -1374,7 +1374,7 @@ def test_edit_carddata_partial_block_field(pub, admin_user):
     assert resp.form['mappings$element1$field_id'].options == [
         ('', False, '---'),
         ('0', False, 'foo - Text (line)'),
-        ('1', False, 'block field - Field Block (foobar)'),
+        ('1', False, 'block field - Block of fields (foobar)'),
         ('1$123', True, 'block field - Test - Text (line)'),
         ('1$234', False, 'block field - Test2 - Text (line)'),
     ]

tests/workflow/test_wscall.py

@@ -488,6 +488,26 @@ def test_webservice_call(http_requests, pub):
     assert isinstance(attachment, AttachmentEvolutionPart)
     assert attachment.base_filename == 'xxx.xml'
     assert attachment.content_type == 'text/xml'
+    assert attachment.display_in_history is True
+    attachment.fp.seek(0)
+    assert attachment.fp.read(5) == b'<?xml'
+    formdata.workflow_data = None
+
+    # check storing response as attachment, not displayed in history
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net/xml'
+    item.varname = 'xxx'
+    item.response_type = 'attachment'
+    item.record_errors = True
+    item.attach_file_to_history = False
+    item.perform(formdata)
+    assert formdata.workflow_data.get('xxx_status') == 200
+    assert formdata.workflow_data.get('xxx_content_type') == 'text/xml'
+    attachment = formdata.evolution[-1].parts[-1]
+    assert isinstance(attachment, AttachmentEvolutionPart)
+    assert attachment.base_filename == 'xxx.xml'
+    assert attachment.content_type == 'text/xml'
+    assert attachment.display_in_history is False
     attachment.fp.seek(0)
     assert attachment.fp.read(5) == b'<?xml'
     formdata.workflow_data = None
@@ -574,6 +594,82 @@ def test_webservice_call(http_requests, pub):
     assert payload == {'one': 1, 'str': 'abcd', 'evalme': formdata.get_display_id()}
 
 
+def test_webservice_with_unflattened_payload_keys(http_requests, pub):
+    wf = Workflow(name='wf1')
+    wf.add_status('Status1', 'st1')
+    wf.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.workflow_id = wf.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.post_data = {
+        'foo/0': 'first',
+        'foo/1': 'second',
+        'foo/2': '{{ form_name }}',
+        'bar': 'example',
+        'form//name': '{{ form_name }}',
+    }
+    pub.substitutions.feed(formdata)
+    item.perform(formdata)
+    assert http_requests.count() == 1
+    assert http_requests.get_last('url') == 'http://remote.example.net/'
+    assert http_requests.get_last('method') == 'POST'
+    payload = json.loads(http_requests.get_last('body'))
+    assert payload == {'foo': ['first', 'second', 'baz'], 'bar': 'example', 'form/name': 'baz'}
+
+    http_requests.empty()
+    pub.loggederror_class.wipe()
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.record_on_errors = True
+    item.post_data = {'foo/1': 'first', 'foo/2': 'second'}
+
+    item.perform(formdata)
+    assert http_requests.count() == 0
+    assert pub.loggederror_class.count() == 1
+    assert (
+        pub.loggederror_class.select()[0].summary
+        == '[WSCALL] Webservice call failure because unable to unflatten payload keys (incomplete array before key "foo/1")'
+    )
+
+    http_requests.empty()
+    pub.loggederror_class.wipe()
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.record_on_errors = True
+    item.post_data = {'0/foo': 'value', '1/bar': 'value', 'name': '{{ form_name }}'}
+
+    item.perform(formdata)
+    assert (
+        pub.loggederror_class.select()[0].summary
+        == '[WSCALL] Webservice call failure because unable to unflatten payload keys (there is a mix between lists and dicts)'
+    )
+
+    http_requests.empty()
+    pub.loggederror_class.wipe()
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.record_on_errors = True
+    item.post_data = {'0/foo': 'value', '1/bar': 'value'}
+
+    pub.substitutions.feed(formdata)
+    item.perform(formdata)
+
+    assert http_requests.count() == 1
+    payload = json.loads(http_requests.get_last('body'))
+    assert payload == [{'foo': 'value'}, {'bar': 'value'}]
+
+
 def test_webservice_waitpoint(pub):
     item = WebserviceCallStatusItem()
     assert item.waitpoint

wcs/admin/blocks.py

@@ -55,6 +55,7 @@ class BlockDirectory(FieldsDirectory):
         'duplicate',
         ('history', 'snapshots_dir'),
         'overwrite',
+        ('update-documentation', 'update_documentation'),
     ]
     field_def_page_class = BlockFieldDefPage
     blacklisted_types = ['page', 'table', 'table-select', 'tablerows', 'ranked-items', 'blocks', 'computed']
@@ -113,11 +114,13 @@ class BlockDirectory(FieldsDirectory):
         r += htmltext('<li><a href="export">%s</a></li>') % _('Export')
         r += htmltext('<li><a href="delete" rel="popup">%s</a></li>') % _('Delete')
         r += htmltext('</ul>')
+        r += self.get_documentable_button()
         r += htmltext('<a href="settings" rel="popup" role="button">%s</a>') % _('Settings')
         r += htmltext('</span>')
         r += htmltext('</div>')
         r += utils.last_modification_block(obj=self.objectdef)
         r += get_session().display_message()
+        r += self.get_documentable_zone()
 
         if not self.objectdef.fields:
             r += htmltext('<div class="infonotice">%s</div>') % _('There are not yet any fields defined.')

wcs/admin/comment_templates.py

@@ -20,6 +20,7 @@ from quixote.html import TemplateIO, htmltext
 
 from wcs.admin import utils
 from wcs.admin.categories import CommentTemplateCategoriesDirectory, get_categories
+from wcs.admin.documentable import DocumentableMixin
 from wcs.backoffice.applications import ApplicationsDirectory
 from wcs.backoffice.snapshots import SnapshotsDirectory
 from wcs.categories import CommentTemplateCategory
@@ -169,7 +170,7 @@ class CommentTemplatesDirectory(Directory):
         return redirect('%s/' % comment_template.id)
 
 
-class CommentTemplatePage(Directory):
+class CommentTemplatePage(Directory, DocumentableMixin):
     _q_exports = [
         '',
         'edit',
@@ -177,6 +178,7 @@ class CommentTemplatePage(Directory):
         'duplicate',
         'export',
         ('history', 'snapshots_dir'),
+        ('update-documentation', 'update_documentation'),
     ]
     do_not_call_in_templates = True
 
@@ -187,6 +189,8 @@ class CommentTemplatePage(Directory):
             raise errors.TraversalError()
         get_response().breadcrumb.append((component + '/', self.comment_template.name))
         self.snapshots_dir = SnapshotsDirectory(self.comment_template)
+        self.documented_object = self.comment_template
+        self.documented_element = self.comment_template
 
     def get_sidebar(self):
         r = TemplateIO(html=True)
@@ -247,14 +251,6 @@ class CommentTemplatePage(Directory):
                 value=self.comment_template.category_id,
             )
 
-        form.add(
-            TextWidget,
-            'description',
-            title=_('Description'),
-            cols=80,
-            rows=3,
-            value=self.comment_template.description,
-        )
         form.add(
             TextWidget,
             'comment',
@@ -307,7 +303,6 @@ class CommentTemplatePage(Directory):
         self.comment_template.name = name
         if form.get_widget('category_id'):
             self.comment_template.category_id = form.get_widget('category_id').parse()
-        self.comment_template.description = form.get_widget('description').parse()
         self.comment_template.comment = form.get_widget('comment').parse()
         self.comment_template.attachments = form.get_widget('attachments').parse()
         if slug_widget:

wcs/admin/data_sources.py

@@ -20,6 +20,7 @@ from quixote.html import TemplateIO, htmltext
 
 from wcs.admin import utils
 from wcs.admin.categories import DataSourceCategoriesDirectory, get_categories
+from wcs.admin.documentable import DocumentableMixin
 from wcs.backoffice.applications import ApplicationsDirectory
 from wcs.backoffice.snapshots import SnapshotsDirectory
 from wcs.carddef import CardDef
@@ -45,7 +46,6 @@ from wcs.qommon.form import (
     SingleSelectWidget,
     SlugWidget,
     StringWidget,
-    TextWidget,
     WidgetDict,
     WidgetList,
     get_response,
@@ -73,14 +73,6 @@ class NamedDataSourceUI:
                 options=category_options,
                 value=self.datasource.category_id,
             )
-        form.add(
-            TextWidget,
-            'description',
-            title=_('Description'),
-            cols=40,
-            rows=5,
-            value=self.datasource.description,
-        )
         if not self.datasource or (
             self.datasource.type != 'wcs:users' and self.datasource.external != 'agenda_manual'
         ):
@@ -297,7 +289,7 @@ class NamedDataSourceUI:
         self.datasource.store()
 
 
-class NamedDataSourcePage(Directory):
+class NamedDataSourcePage(Directory, DocumentableMixin):
     _q_exports = [
         '',
         'edit',
@@ -306,6 +298,7 @@ class NamedDataSourcePage(Directory):
         'duplicate',
         ('history', 'snapshots_dir'),
         ('preview-block', 'preview_block'),
+        ('update-documentation', 'update_documentation'),
     ]
     do_not_call_in_templates = True
 
@@ -319,6 +312,8 @@ class NamedDataSourcePage(Directory):
         self.datasource_ui = NamedDataSourceUI(self.datasource)
         get_response().breadcrumb.append((component + '/', self.datasource.name))
         self.snapshots_dir = SnapshotsDirectory(self.datasource)
+        self.documented_object = self.datasource
+        self.documented_element = self.datasource
 
     def get_sidebar(self):
         r = TemplateIO(html=True)

wcs/admin/documentable.py

@@ -0,0 +1,62 @@
+# w.c.s. - web application for online forms
+# Copyright (C) 2005-2024  Entr'ouvert
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+
+import json
+
+from quixote import get_request, get_response
+from quixote.html import htmltext
+
+from wcs.qommon import _, template
+from wcs.qommon.form import RichTextWidget
+
+
+class DocumentableMixin:
+    def get_documentable_button(self):
+        return htmltext(template.render('wcs/backoffice/includes/documentation-editor-link.html', {}))
+
+    def get_documentable_zone(self):
+        return htmltext('<span class="actions">%s</span>') % template.render(
+            'wcs/backoffice/includes/documentation.html',
+            {'element': self.documented_element, 'object': self.documented_object},
+        )
+
+    def update_documentation(self):
+        get_request().ignore_session = True
+        get_response().set_content_type('application/json')
+        try:
+            content = get_request().json['content']
+        except (KeyError, TypeError):
+            return json.dumps({'err': 1})
+        content = RichTextWidget('').clean_html(content) or None
+        changed = False
+        if content != self.documented_element.documentation:
+            changed = True
+            self.documented_element.documentation = content
+            self.documented_object.store(_('Documentation update'))
+        return json.dumps(
+            {'err': 0, 'empty': not bool(self.documented_element.documentation), 'changed': changed}
+        )
+
+
+class DocumentableFieldMixin:
+    def documentation_part(self):
+        if not self.field.documentation:
+            get_response().filter['sidebar_attrs'] = 'hidden'
+        return template.render(
+            'wcs/backoffice/includes/documentation.html',
+            {'element': self.documented_element, 'object': self.documented_object},
+        )

wcs/admin/fields.py

@@ -26,18 +26,21 @@ from wcs.admin import utils
 from wcs.carddef import CardDef
 from wcs.fields import BlockField, get_field_options
 from wcs.formdef import FormDef, UpdateStatisticsDataAfterJob
-from wcs.qommon import _, errors, get_cfg, misc
+from wcs.qommon import _, errors, get_cfg, misc, template
 from wcs.qommon.admin.menu import command_icon
 from wcs.qommon.form import CheckboxWidget, Form, HtmlWidget, OptGroup, SingleSelectWidget, StringWidget
 from wcs.qommon.substitution import CompatibilityNamesDict
 
+from .documentable import DocumentableFieldMixin, DocumentableMixin
 
-class FieldDefPage(Directory):
-    _q_exports = ['', 'delete', 'duplicate']
+
+class FieldDefPage(Directory, DocumentableMixin, DocumentableFieldMixin):
+    _q_exports = ['', 'delete', 'duplicate', ('update-documentation', 'update_documentation')]
 
     large = False
     page_id = None
     blacklisted_attributes = []
+    is_documentable = True
 
     def __init__(self, objectdef, field_id):
         self.objectdef = objectdef
@@ -47,6 +50,8 @@ class FieldDefPage(Directory):
             raise errors.TraversalError()
         if not self.field.label:
             self.field.label = str(_('None'))
+        self.documented_object = objectdef
+        self.documented_element = self.field
         label = misc.ellipsize(self.field.unhtmled_label, 40)
         last_breadcrumb_url_part, last_breadcrumb_label = get_response().breadcrumb[-1]
         get_response().breadcrumb = get_response().breadcrumb[:-1]
@@ -67,7 +72,11 @@ class FieldDefPage(Directory):
         return form
 
     def get_sidebar(self):
-        return None
+        if not self.is_documentable:
+            return None
+        r = TemplateIO(html=True)
+        r += self.documentation_part()
+        return r.getvalue()
 
     def _q_index(self):
         form = self.form()
@@ -94,15 +103,26 @@ class FieldDefPage(Directory):
             get_response().set_title(self.objectdef.name)
             get_response().filter['sidebar'] = self.get_sidebar()  # noqa pylint: disable=assignment-from-none
             r = TemplateIO(html=True)
+            r += htmltext('<div id="appbar" class="field-edit">')
             r += htmltext('<h2 class="field-edit--title">%s</h2>') % misc.ellipsize(
                 self.field.unhtmled_label, 80
             )
-            if isinstance(self.field, BlockField):
-                r += htmltext('<h3 class="field-edit--subtitle">%s - <a href="%s">%s</a></h3>') % (
-                    _('Block of fields'),
-                    self.field.block.get_admin_url(),
-                    self.field.block.name,
+            if self.is_documentable:
+                r += htmltext('<span class="actions">%s</span>') % template.render(
+                    'wcs/backoffice/includes/documentation-editor-link.html', {}
                 )
+            r += htmltext('</div>')
+            if isinstance(self.field, BlockField):
+                try:
+                    block_field = self.field.block
+                except KeyError:
+                    r += htmltext('<h3 class="field-edit--subtitle">%s</h3>') % self.field.get_type_label()
+                else:
+                    r += htmltext('<h3 class="field-edit--subtitle">%s - <a href="%s">%s</a></h3>') % (
+                        _('Block of fields'),
+                        block_field.get_admin_url(),
+                        block_field.name,
+                    )
             else:
                 r += htmltext('<h3 class="field-edit--subtitle">%s</h3>') % self.field.description
             existing_varnames = {
@@ -155,7 +175,7 @@ class FieldDefPage(Directory):
         self.objectdef.store(comment=_('Modification of field "%s"') % self.field.ellipsized_label)
 
     def get_deletion_extra_warning(self):
-        return _('Warning: this field data will be permanently deleted.')
+        return {'level': 'warning', 'message': _('Warning: this field data will be permanently deleted.')}
 
     def redirect_field_anchor(self, field):
         anchor = '#fieldId_%s' % field.id if field else ''
@@ -182,7 +202,7 @@ class FieldDefPage(Directory):
         if self.field.key not in ('page', 'subtitle', 'title', 'comment'):
             warning = self.get_deletion_extra_warning()
             if warning:
-                form.widgets.append(HtmlWidget('<div class="warningnotice">%s</div>' % warning))
+                form.widgets.append(HtmlWidget('<div class="%(level)snotice">%(message)s</div>' % warning))
         current_field_index = self.objectdef.fields.index(self.field)
         to_be_deleted = []
         if self.field.key == 'page':
@@ -324,8 +344,15 @@ class FieldsPagesDirectory(Directory):
         return directory
 
 
-class FieldsDirectory(Directory):
-    _q_exports = ['', 'update_order', 'move_page_fields', 'new', 'pages']
+class FieldsDirectory(Directory, DocumentableMixin):
+    _q_exports = [
+        '',
+        'update_order',
+        'move_page_fields',
+        'new',
+        'pages',
+        ('update-documentation', 'update_documentation'),
+    ]
     field_def_page_class = FieldDefPage
     blacklisted_types = []
     page_id = None
@@ -340,6 +367,8 @@ class FieldsDirectory(Directory):
 
     def __init__(self, objectdef):
         self.objectdef = objectdef
+        self.documented_object = self.objectdef
+        self.documented_element = self.objectdef
         self.pages = FieldsPagesDirectory(self)
 
     def _q_traverse(self, path):

wcs/admin/forms.py

@@ -69,6 +69,7 @@ from . import utils
 from .blocks import BlocksDirectory
 from .categories import CategoriesDirectory, get_categories
 from .data_sources import NamedDataSourcesDirectory
+from .documentable import DocumentableMixin
 from .fields import FieldDefPage, FieldsDirectory
 from .logged_errors import LoggedErrorsDirectory
 
@@ -214,7 +215,7 @@ class FormFieldDefPage(FieldDefPage):
     def get_deletion_extra_warning(self):
         if not self.objectdef.data_class().count():
             return None
-        return self.deletion_extra_warning_message
+        return {'level': 'warning', 'message': self.deletion_extra_warning_message}
 
 
 class FormFieldsDirectory(FieldsDirectory):
@@ -621,7 +622,7 @@ class WorkflowRoleDirectory(Directory):
             return redirect('..')
 
 
-class FormDefPage(Directory, TempfileDirectoryMixin):
+class FormDefPage(Directory, TempfileDirectoryMixin, DocumentableMixin):
     do_not_call_in_templates = True
     _q_exports = [
         '',
@@ -648,6 +649,7 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
         ('backoffice-submission-roles', 'backoffice_submission_roles'),
         ('logged-errors', 'logged_errors_dir'),
         ('history', 'snapshots_dir'),
+        ('update-documentation', 'update_documentation'),
     ]
 
     formdef_class = FormDef
@@ -691,6 +693,8 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
             parent_dir=self, formdef_class=self.formdef_class, formdef_id=self.formdef.id
         )
         self.snapshots_dir = SnapshotsDirectory(self.formdef)
+        self.documented_object = self.formdef
+        self.documented_element = self.formdef
 
     def add_option_line(self, link, label, current_value, popup=True):
         return htmltext(

wcs/admin/mail_templates.py

@@ -20,6 +20,7 @@ from quixote.html import TemplateIO, htmltext
 
 from wcs.admin import utils
 from wcs.admin.categories import MailTemplateCategoriesDirectory, get_categories
+from wcs.admin.documentable import DocumentableMixin
 from wcs.backoffice.applications import ApplicationsDirectory
 from wcs.backoffice.snapshots import SnapshotsDirectory
 from wcs.categories import MailTemplateCategory
@@ -167,7 +168,7 @@ class MailTemplatesDirectory(Directory):
         return redirect('%s/' % mail_template.id)
 
 
-class MailTemplatePage(Directory):
+class MailTemplatePage(Directory, DocumentableMixin):
     _q_exports = [
         '',
         'edit',
@@ -175,6 +176,7 @@ class MailTemplatePage(Directory):
         'duplicate',
         'export',
         ('history', 'snapshots_dir'),
+        ('update-documentation', 'update_documentation'),
     ]
     do_not_call_in_templates = True
 
@@ -185,6 +187,8 @@ class MailTemplatePage(Directory):
             raise errors.TraversalError()
         get_response().breadcrumb.append((component + '/', self.mail_template.name))
         self.snapshots_dir = SnapshotsDirectory(self.mail_template)
+        self.documented_object = self.mail_template
+        self.documented_element = self.mail_template
 
     def get_sidebar(self):
         r = TemplateIO(html=True)
@@ -242,15 +246,6 @@ class MailTemplatePage(Directory):
                 options=category_options,
                 value=self.mail_template.category_id,
             )
-
-        form.add(
-            TextWidget,
-            'description',
-            title=_('Description'),
-            cols=80,
-            rows=3,
-            value=self.mail_template.description,
-        )
         form.add(
             StringWidget,
             'subject',
@@ -312,7 +307,6 @@ class MailTemplatePage(Directory):
         self.mail_template.name = name
         if form.get_widget('category_id'):
             self.mail_template.category_id = form.get_widget('category_id').parse()
-        self.mail_template.description = form.get_widget('description').parse()
         self.mail_template.subject = form.get_widget('subject').parse()
         self.mail_template.body = form.get_widget('body').parse()
         self.mail_template.attachments = form.get_widget('attachments').parse()

wcs/admin/settings.py

@@ -131,6 +131,7 @@ authentication is unavailable. Lasso must be installed to use it.'
 
 class UserFieldDefPage(FieldDefPage):
     blacklisted_attributes = ['condition']
+    is_documentable = False
 
 
 class UserFieldsDirectory(FieldsDirectory):

wcs/admin/tests.py

@@ -25,10 +25,14 @@ from quixote import get_publisher, get_request, get_response, get_session, redir
 from quixote.directory import Directory
 from quixote.html import TemplateIO, htmltext
 
+from wcs.admin import utils
 from wcs.admin.workflow_tests import WorkflowTestsDirectory
 from wcs.api import posted_json_data_to_formdata_data
 from wcs.backoffice.management import FormBackofficeEditPage, FormBackOfficeStatusPage
 from wcs.backoffice.pagination import pagination_links
+from wcs.backoffice.snapshots import SnapshotsDirectory
+from wcs.carddef import CardDef
+from wcs.formdef import FormDef
 from wcs.forms.common import FormStatusPage
 from wcs.qommon import _, misc, template
 from wcs.qommon.afterjobs import AfterJob
@@ -130,12 +134,12 @@ class TestEditPage(FormBackofficeEditPage):
         self.testdef.data = testdef.data
 
         self.testdef.expected_error = get_request().form.get('error')
-        self.testdef.store()
+        self.testdef.store(comment=_('Mark test as failing'))
         return redirect('..')
 
     def change_submission_mode(self):
         self.testdef.is_in_backoffice = not self.testdef.is_in_backoffice
-        self.testdef.store()
+        self.testdef.store(comment=_('Change submission mode'))
         return redirect('.')
 
 
@@ -148,13 +152,19 @@ class TestPage(FormBackOfficeStatusPage):
         'duplicate',
         ('workflow', 'workflow_tests'),
         ('webservice-responses', 'webservice_responses'),
+        ('history', 'snapshots_dir'),
     ]
 
-    def __init__(self, component, objectdef):
+    def __init__(self, component, objectdef=None, instance=None):
         try:
-            self.testdef = TestDef.get(component)
+            self.testdef = instance or TestDef.get(component)
         except KeyError:
             raise TraversalError()
+
+        if not objectdef:
+            klass = FormDef if self.testdef.object_type == 'formdefs' else CardDef
+            objectdef = klass.get(self.testdef.object_id)
+
         self.testdef.formdef = objectdef
 
         filled = self.testdef.build_formdata(objectdef, include_fields=True)
@@ -162,6 +172,7 @@ class TestPage(FormBackOfficeStatusPage):
 
         self.workflow_tests = WorkflowTestsDirectory(self.testdef, self.formdef)
         self.webservice_responses = WebserviceResponseDirectory(self.testdef)
+        self.snapshots_dir = SnapshotsDirectory(self.testdef)
 
     @property
     def edit_data(self):
@@ -179,16 +190,28 @@ class TestPage(FormBackOfficeStatusPage):
         return False
 
     def get_extra_context_bar(self, parent=None):
-        return render_to_string('wcs/backoffice/test_sidebar.html', context={})
+        if self.testdef.is_readonly():
+            r = TemplateIO(html=True)
+            r += htmltext('<div class="infonotice"><p>%s</p></div>') % _('This test is readonly.')
+            r += utils.snapshot_info_block(self.testdef.snapshot_object)
+            r += htmltext('<h3>%s</h3>') % _('Navigation')
+            r += htmltext(
+                '<li><a class="button button-paragraph" href="webservice-responses/">%s</a></li>'
+            ) % _('Webservice responses')
+            r += htmltext('<li><a class="button button-paragraph" href="inspect">%s</a></li>') % _('Inspect')
+            r += htmltext('</h3>')
+            return r.getvalue()
+        else:
+            return render_to_string('wcs/backoffice/test_sidebar.html', context={})
 
     def status(self):
         r = TemplateIO(html=True)
         r += htmltext('<div id="appbar">')
         r += htmltext('<h2>%s</h2>') % self.testdef
         r += htmltext('<span class="actions">')
-        r += htmltext('<a href="edit-data/">%s</a>') % _('Edit data')
-        if get_publisher().has_site_option('enable-workflow-tests'):
-            r += htmltext('<a href="workflow/">%s</a>') % _('Workflow tests')
+        if not self.testdef.is_readonly():
+            r += htmltext('<a href="edit-data/">%s</a>') % _('Edit data')
+        r += htmltext('<a href="workflow/">%s</a>') % _('Workflow tests')
         r += htmltext('</span>')
         r += htmltext('</div>')
         if self.testdef.expected_error:
@@ -255,7 +278,7 @@ class TestPage(FormBackOfficeStatusPage):
             self.testdef.name = form.get_widget('name').parse()
             self.testdef.user_uuid = form.get_widget('user').parse()
 
-            self.testdef.store()
+            self.testdef.store(comment=_('Change in options'))
             return redirect('.')
 
     def duplicate(self):
@@ -286,7 +309,7 @@ class TestPage(FormBackOfficeStatusPage):
 
         self.testdef.name = form.get_widget('name').parse()
         self.testdef = TestDef.import_from_xml_tree(self.testdef.export_to_xml(), self.formdef)
-        self.testdef.store()
+        self.testdef.store(comment=_('Creation (from duplication)'))
 
         return redirect(self.testdef.get_admin_url())
 
@@ -343,11 +366,8 @@ class TestsDirectory(Directory):
             creation_options = [
                 ('empty', _('Fill data manually'), 'empty'),
                 ('formdata', _('Import data from form'), 'formdata'),
+                ('formdata-wf', _('Import data from form (and initialise workflow tests)'), 'formdata-wf'),
             ]
-            if get_publisher().has_site_option('enable-workflow-tests'):
-                creation_options.append(
-                    ('formdata-wf', _('Import data from form (and initialise workflow tests)'), 'formdata-wf')
-                )
             form.add(
                 RadiobuttonsWidget,
                 'creation_mode',
@@ -390,7 +410,7 @@ class TestsDirectory(Directory):
             testdef = TestDef.create_from_formdata(self.objectdef, self.objectdef.data_class()())
             testdef.name = form.get_widget('name').parse()
             testdef.agent_id = test_agent_user.test_uuid
-            testdef.store()
+            testdef.store(comment=_('Creation (empty)'))
             return redirect(testdef.get_admin_url() + 'edit-data/')
         else:
             formdata_id = form.get_widget('formdata').parse()
@@ -403,7 +423,7 @@ class TestsDirectory(Directory):
             )
             testdef.name = form.get_widget('name').parse()
             testdef.agent_id = test_agent_user.test_uuid
-            testdef.store()
+            testdef.store(comment=_('Creation (from formdata)'))
             return redirect(testdef.get_admin_url())
 
     def p_import(self):
@@ -426,9 +446,6 @@ class TestsDirectory(Directory):
         get_response().set_title(_('Import Test'))
         r = TemplateIO(html=True)
         r += htmltext('<h2>%s</h2>') % _('Import Test')
-        r += htmltext('<p>%s</p>') % _(
-            'You can add a new test or update an existing one by importing a JSON file.'
-        )
         r += form.render()
         return r.getvalue()
 
@@ -441,6 +458,7 @@ class TestsDirectory(Directory):
             form.set_error('file', _('Invalid File'))
             raise e
 
+        testdef.store(comment=_('Creation (from import)'))
         get_session().message = ('info', _('Test "%s" has been successfully imported.') % testdef.name)
         return redirect('.')
 
@@ -825,7 +843,8 @@ class WebserviceResponsePage(Directory):
             },
         )
 
-        form.add_submit('submit', _('Submit'))
+        if not self.testdef.is_readonly():
+            form.add_submit('submit', _('Submit'))
         form.add_submit('cancel', _('Cancel'))
         form.add_media()
 
@@ -847,6 +866,7 @@ class WebserviceResponsePage(Directory):
         self.webservice_response.method = form.get_widget('method').parse()
         self.webservice_response.post_data = form.get_widget('post_data').parse()
         self.webservice_response.store()
+        self.testdef.store(comment=_('Change webservice response "%s"') % self.webservice_response.name)
 
         return redirect('..')
 
@@ -872,11 +892,14 @@ class WebserviceResponsePage(Directory):
         new_webservice_response.id = None
         new_webservice_response.name = '%s %s' % (new_webservice_response.name, _('(copy)'))
         new_webservice_response.store()
+        self.testdef.store(
+            comment=_('Duplication of webservice response "%s"') % self.webservice_response.name
+        )
         return redirect('..')
 
 
 class WebserviceResponseDirectory(Directory):
-    _q_exports = ['', 'new']
+    _q_exports = ['', 'new', ('import', 'p_import')]
 
     def __init__(self, testdef):
         self.testdef = testdef
@@ -891,7 +914,8 @@ class WebserviceResponseDirectory(Directory):
     def _q_index(self):
         context = {
             'webservice_responses': self.testdef.get_webservice_responses(),
-            'has_sidebar': True,
+            'has_sidebar': bool(not self.testdef.is_readonly()),
+            'testdef': self.testdef,
         }
         get_response().add_javascript(['popup.js'])
         get_response().set_title(_('Webservice responses'))
@@ -923,12 +947,53 @@ class WebserviceResponseDirectory(Directory):
         webservice_response.testdef_id = self.testdef.id
         webservice_response.name = form.get_widget('name').parse()
         webservice_response.store()
+        self.testdef.store(comment=_('New webservice response "%s"') % webservice_response.name)
 
         return redirect(self.testdef.get_admin_url() + 'webservice-responses/%s/' % webservice_response.id)
 
+    def p_import(self):
+        form = Form(enctype='multipart/form-data')
+
+        testdef_options = [
+            (x.id, x, x.id)
+            for x in TestDef.select_for_objectdef(self.testdef.formdef)
+            if x.id != self.testdef.id
+        ]
+        form.add(
+            SingleSelectWidget,
+            'testdef_id',
+            required=False,
+            options=testdef_options,
+            **{'data-autocomplete': 'true'},
+        )
+
+        form.add_submit('submit', _('Import'))
+        form.add_submit('cancel', _('Cancel'))
+
+        if form.get_submit() == 'cancel':
+            return redirect('.')
+
+        if not form.is_submitted() or form.has_errors():
+            get_response().breadcrumb.append(('import', _('Import')))
+            get_response().set_title(_('Import webservice responses'))
+            r = TemplateIO(html=True)
+            r += htmltext('<h2>%s</h2>') % _('Import webservice responses')
+            r += form.render()
+            return r.getvalue()
+
+        testdef_id = form.get_widget('testdef_id').parse()
+        testdef = TestDef.get(testdef_id)
+
+        for response in testdef.get_webservice_responses():
+            response.id = None
+            response.testdef_id = self.testdef.id
+            response.store()
+
+        return redirect('.')
+
 
 class TestUserPage(Directory):
-    _q_exports = ['', 'delete']
+    _q_exports = ['', 'delete', 'export']
 
     def __init__(self, component):
         try:
@@ -988,7 +1053,12 @@ class TestUserPage(Directory):
 
         get_response().breadcrumb.append(('edit', _('Edit test user')))
         r = TemplateIO(html=True)
+        r += htmltext('<div id="appbar">')
         r += htmltext('<h2>%s</h2>') % (_('Edit test user'))
+        r += htmltext('<span class="actions">')
+        r += htmltext('<a href="export">%s</a>') % _('Export')
+        r += htmltext('</span>')
+        r += htmltext('</div>')
         r += form.render()
         return r.getvalue()
 
@@ -1009,9 +1079,16 @@ class TestUserPage(Directory):
         self.user.remove_object(self.user.id)
         return redirect('..')
 
+    def export(self):
+        get_response().set_content_type('application/json')
+        get_response().set_header(
+            'content-disposition', 'attachment; filename=wcs_test_user_%s.json' % self.user.name
+        )
+        return json.dumps({'test-users': [self.user.get_json_export_dict(full=True, include_roles=True)]})
+
 
 class TestUsersDirectory(Directory):
-    _q_exports = ['', 'new']
+    _q_exports = ['', 'new', 'export', ('import', 'p_import')]
 
     def _q_traverse(self, path):
         get_response().breadcrumb.append(('test-users/', _('Test users')))
@@ -1086,3 +1163,71 @@ class TestUsersDirectory(Directory):
         r += htmltext('<h2>%s</h2>') % _('New test user')
         r += form.render()
         return r.getvalue()
+
+    def export(self):
+        get_response().set_content_type('application/json')
+        get_response().set_header('content-disposition', 'attachment; filename=wcs_test_users.json')
+        users = get_publisher().user_class.select([NotNull('test_uuid')])
+        return json.dumps(
+            {'test-users': [x.get_json_export_dict(full=True, include_roles=True) for x in users]}
+        )
+
+    def p_import(self):
+        form = Form(enctype='multipart/form-data')
+
+        form.add(FileWidget, 'file', title=_('File'), required=True)
+        form.add_submit('submit', _('Import'))
+        form.add_submit('cancel', _('Cancel'))
+
+        if form.get_submit() == 'cancel':
+            return redirect('.')
+
+        if form.is_submitted() and not form.has_errors():
+            try:
+                return self.import_submit(form)
+            except ValueError:
+                pass
+
+        get_response().breadcrumb.append(('import', _('Import')))
+        get_response().set_title(_('Import test users'))
+        r = TemplateIO(html=True)
+        r += htmltext('<h2>%s</h2>') % _('Import test users')
+        r += form.render()
+        return r.getvalue()
+
+    def import_submit(self, form):
+        fp = form.get_widget('file').parse().fp
+        try:
+            data = json.loads(fp.read())
+        except ValueError:
+            form.set_error('file', _('Invalid JSON file'))
+            raise ValueError
+
+        existing_users = get_publisher().user_class.select([NotNull('test_uuid')])
+        existing_uuids = {x.test_uuid for x in existing_users}
+        existing_emails = {x.email for x in existing_users}
+
+        users = []
+        users_were_ignored = False
+        for user_dict in data.get('test-users', []):
+            try:
+                user = get_publisher().user_class.import_from_json(user_dict)
+            except KeyError:
+                form.set_error('file', _('Invalid File'))
+                raise ValueError
+
+            if user.test_uuid in existing_uuids or user.email in existing_emails:
+                users_were_ignored = True
+                continue
+
+            users.append(user)
+
+        for user in users:
+            user.store()
+
+        if users_were_ignored:
+            get_session().message = ('warning', _('Some already existing users were not imported.'))
+        else:
+            get_session().message = ('success', _('Test users have been successfully imported.'))
+
+        return redirect('.')

wcs/admin/workflow_tests.py

@@ -47,7 +47,8 @@ class WorkflowTestActionPage(Directory):
         if not form.widgets:
             form.add_global_errors([htmltext(self.action.empty_form_error)])
         else:
-            form.add_submit('submit', _('Submit'))
+            if not self.testdef.is_readonly():
+                form.add_submit('submit', _('Submit'))
         form.add_submit('cancel', _('Cancel'))
 
         if form.get_widget('cancel').parse():
@@ -69,7 +70,7 @@ class WorkflowTestActionPage(Directory):
 
             setattr(self.action, widget.name, value)
 
-        self.testdef.store()
+        self.testdef.store(comment=_('Change in workflow test action "%s"') % self.action.label)
         return redirect('..')
 
     def delete(self):
@@ -90,7 +91,7 @@ class WorkflowTestActionPage(Directory):
         self.testdef.workflow_tests.actions = [
             x for x in self.testdef.workflow_tests.actions if x.id != self.action.id
         ]
-        self.testdef.store()
+        self.testdef.store(comment=_('Deletion of workflow test action "%s"') % self.action.label)
         return redirect('..')
 
     def duplicate(self):
@@ -98,7 +99,7 @@ class WorkflowTestActionPage(Directory):
         new_action.id = self.testdef.workflow_tests.get_new_action_id()
         action_position = self.testdef.workflow_tests.actions.index(self.action)
         self.testdef.workflow_tests.actions.insert(action_position + 1, new_action)
-        self.testdef.store()
+        self.testdef.store(comment=_('Duplication of workflow test action "%s"') % self.action.label)
         return redirect('..')
 
 
@@ -120,7 +121,7 @@ class WorkflowTestsDirectory(Directory):
     def _q_index(self):
         context = {
             'testdef': self.testdef,
-            'has_sidebar': True,
+            'has_sidebar': bool(not self.testdef.is_readonly()),
             'sidebar_form': self.get_sidebar_form(),
         }
 
@@ -161,7 +162,8 @@ class WorkflowTestsDirectory(Directory):
             **{'data-autocomplete': 'true'},
         )
 
-        form.add_submit('submit', _('Submit'))
+        if not self.testdef.is_readonly():
+            form.add_submit('submit', _('Submit'))
         form.add_submit('cancel', _('Cancel'))
 
         if form.get_widget('cancel').parse():
@@ -176,7 +178,7 @@ class WorkflowTestsDirectory(Directory):
             return r.getvalue()
 
         self.testdef.agent_id = form.get_widget('agent').parse()
-        self.testdef.store()
+        self.testdef.store(comment=_('Change in workflow test options'))
         return redirect('.')
 
     def new(self):
@@ -190,7 +192,7 @@ class WorkflowTestsDirectory(Directory):
         action_type = form.get_widget('type').parse()
         action_class = get_test_action_class_by_type(action_type)
         self.testdef.workflow_tests.add_action(action_class)
-        self.testdef.store()
+        self.testdef.store(comment=_('New test action "%s"') % action_class.label)
 
         return redirect('.')
 
@@ -219,7 +221,7 @@ class WorkflowTestsDirectory(Directory):
             return json.dumps({'success': 'ko'})
 
         self.testdef.workflow_tests.actions = new_actions
-        self.testdef.store()
+        self.testdef.store(comment=_('Change in workflow test actions order'))
 
         return json.dumps(
             {

wcs/admin/workflows.py

@@ -67,6 +67,7 @@ from wcs.workflows import (
 from . import utils
 from .comment_templates import CommentTemplatesDirectory
 from .data_sources import NamedDataSourcesDirectory
+from .documentable import DocumentableFieldMixin, DocumentableMixin
 from .fields import FieldDefPage, FieldsDirectory
 from .logged_errors import LoggedErrorsDirectory
 from .mail_templates import MailTemplatesDirectory
@@ -440,8 +441,14 @@ class WorkflowUI:
         return workflow
 
 
-class WorkflowItemPage(Directory):
-    _q_exports = ['', 'delete', 'copy']
+class WorkflowItemPage(Directory, DocumentableMixin):
+    _q_exports = [
+        '',
+        'delete',
+        'copy',
+        ('update-documentation', 'update_documentation'),
+    ]
+    do_not_call_in_templates = True
 
     def __init__(self, workflow, parent, component):
         try:
@@ -450,6 +457,8 @@ class WorkflowItemPage(Directory):
             raise errors.TraversalError()
         self.workflow = workflow
         self.parent = parent
+        self.documented_object = self.workflow
+        self.documented_element = self.item
         get_response().breadcrumb.append(('items/%s/' % component, self.item.description))
 
     def _q_index(self):
@@ -491,12 +500,20 @@ class WorkflowItemPage(Directory):
                 return redirect('..')
 
         get_response().set_title('%s - %s' % (_('Workflow'), self.workflow.name))
-        r = TemplateIO(html=True)
-        r += htmltext('<h2>%s</h2>') % self.item.description
-        r += form.render()
-        if self.item.support_substitution_variables:
-            r += get_publisher().substitutions.get_substitution_html_table()
-        return r.getvalue()
+        get_response().add_javascript(['jquery-ui.js'])
+        context = {
+            'view': self,
+            'html_form': form,
+            'workflow': self.workflow,
+            'has_sidebar': True,
+            'action': self.item,
+            'get_substitution_html_table': get_publisher().substitutions.get_substitution_html_table,
+        }
+        return template.QommonTemplateResponse(
+            templates=['wcs/backoffice/workflow-action.html'],
+            context=context,
+            is_django_native=True,
+        )
 
     def delete(self):
         form = Form(enctype='multipart/form-data')
@@ -667,7 +684,7 @@ class GlobalActionItemsDir(ToChildDirectory):
     klass = WorkflowItemPage
 
 
-class WorkflowStatusPage(Directory):
+class WorkflowStatusPage(Directory, DocumentableMixin):
     _q_exports = [
         '',
         'delete',
@@ -681,6 +698,7 @@ class WorkflowStatusPage(Directory):
         'fullscreen',
         ('schema.svg', 'svg'),
         'svg',
+        ('update-documentation', 'update_documentation'),
     ]
     do_not_call_in_templates = True
 
@@ -692,6 +710,8 @@ class WorkflowStatusPage(Directory):
             raise errors.TraversalError()
 
         self.items_dir = WorkflowItemsDir(workflow, self.status)
+        self.documented_object = self.workflow
+        self.documented_element = self.status
         get_response().breadcrumb.append(('status/%s/' % status_id, self.status.name))
 
     def _q_index(self):
@@ -1081,9 +1101,16 @@ class WorkflowStatusDirectory(Directory):
         return r.getvalue()
 
 
-class WorkflowVariablesFieldDefPage(FieldDefPage):
+class WorkflowVariablesFieldDefPage(FieldDefPage, DocumentableFieldMixin):
     section = 'workflows'
     blacklisted_attributes = ['condition', 'prefill', 'display_locations', 'anonymise']
+    has_documentation = True
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.workflow = self.objectdef.workflow
+        self.documented_object = self.workflow
+        self.documented_element = self.field
 
     def form(self):
         form = super().form()
@@ -1109,7 +1136,7 @@ class WorkflowVariablesFieldDefPage(FieldDefPage):
         super().submit(form)
 
 
-class WorkflowBackofficeFieldDefPage(FieldDefPage):
+class WorkflowBackofficeFieldDefPage(FieldDefPage, DocumentableFieldMixin):
     section = 'workflows'
     blacklisted_attributes = ['condition']
 
@@ -1122,21 +1149,23 @@ class WorkflowBackofficeFieldDefPage(FieldDefPage):
                 continue
             if any(x.get('field_id') == self.field.id for x in action.fields or []):
                 usage_actions.append(action)
-        if not usage_actions:
-            return
+
         r = TemplateIO(html=True)
-        r += htmltext('<div class="actions-using-this-field">')
-        r += htmltext('<h3>%s</h3>') % _('Actions using this field')
-        r += htmltext('<ul>')
-        for action in usage_actions:
-            label = _('"%s" action') % action.label if action.label else _('Action')
-            if isinstance(action.parent, WorkflowGlobalAction):
-                location = _('in global action "%s"') % action.parent.name
-            else:
-                location = _('in status "%s"') % action.parent.name
-            r += htmltext(f'<li><a href="{action.get_admin_url()}">%s %s</a></li>') % (label, location)
-        r += htmltext('</ul>')
-        r += htmltext('<div>')
+        r += self.documentation_part()
+        if usage_actions:
+            get_response().filter['sidebar_attrs'] = ''
+            r += htmltext('<div class="actions-using-this-field">')
+            r += htmltext('<h3>%s</h3>') % _('Actions using this field')
+            r += htmltext('<ul>')
+            for action in usage_actions:
+                label = _('"%s" action') % action.label if action.label else _('Action')
+                if isinstance(action.parent, WorkflowGlobalAction):
+                    location = _('in global action "%s"') % action.parent.name
+                else:
+                    location = _('in status "%s"') % action.parent.name
+                r += htmltext(f'<li><a href="{action.get_admin_url()}">%s %s</a></li>') % (label, location)
+            r += htmltext('</ul>')
+            r += htmltext('<div>')
         return r.getvalue()
 
     def form(self):
@@ -1158,7 +1187,7 @@ class WorkflowBackofficeFieldDefPage(FieldDefPage):
 
 
 class WorkflowVariablesFieldsDirectory(FieldsDirectory):
-    _q_exports = ['', 'update_order', 'new']
+    _q_exports = ['', 'update_order', 'new', ('update-documentation', 'update_documentation')]
 
     section = 'workflows'
     field_def_page_class = WorkflowVariablesFieldDefPage
@@ -1175,8 +1204,12 @@ class WorkflowVariablesFieldsDirectory(FieldsDirectory):
 
     def index_top(self):
         r = TemplateIO(html=True)
+        r += htmltext('<div id="appbar">')
         r += htmltext('<h2>%s - %s - %s</h2>') % (_('Workflow'), self.objectdef.name, _('Variables'))
+        r += htmltext('<span class="actions">%s</span>') % self.get_documentable_button()
+        r += htmltext('</div>')
         r += get_session().display_message()
+        r += self.get_documentable_zone()
         if not self.objectdef.fields:
             r += htmltext('<p>%s</p>') % _('There are not yet any variables.')
         return r.getvalue()
@@ -1186,7 +1219,7 @@ class WorkflowVariablesFieldsDirectory(FieldsDirectory):
 
 
 class WorkflowBackofficeFieldsDirectory(FieldsDirectory):
-    _q_exports = ['', 'update_order', 'new']
+    _q_exports = ['', 'update_order', 'new', ('update-documentation', 'update_documentation')]
 
     section = 'workflows'
     field_def_page_class = WorkflowBackofficeFieldDefPage
@@ -1201,10 +1234,19 @@ class WorkflowBackofficeFieldsDirectory(FieldsDirectory):
     fields_count_total_soft_limit = 40
     fields_count_total_hard_limit = 80
 
+    def __init__(self, objectdef):
+        super().__init__(objectdef)
+        self.documented_object = objectdef
+        self.documented_element = objectdef
+
     def index_top(self):
         r = TemplateIO(html=True)
+        r += htmltext('<div id="appbar">')
         r += htmltext('<h2>%s - %s - %s</h2>') % (_('Workflow'), self.objectdef.name, _('Backoffice Fields'))
+        r += htmltext('<span class="actions">%s</span>') % self.get_documentable_button()
+        r += htmltext('</div>')
         r += get_session().display_message()
+        r += self.get_documentable_zone()
         if not self.objectdef.fields:
             r += htmltext('<p>%s</p>') % _('There are not yet any backoffice fields.')
         return r.getvalue()
@@ -1425,6 +1467,7 @@ class GlobalActionPage(WorkflowStatusPage):
         ('triggers', 'triggers_dir'),
         'update_triggers_order',
         'options',
+        ('update-documentation', 'update_documentation'),
     ]
 
     def __init__(self, workflow, action_id):
@@ -1436,6 +1479,8 @@ class GlobalActionPage(WorkflowStatusPage):
         self.status = self.action
         self.items_dir = GlobalActionItemsDir(workflow, self.action)
         self.triggers_dir = GlobalActionTriggersDir(workflow, self.action)
+        self.documented_object = self.workflow
+        self.documented_element = self.action
 
     def _q_traverse(self, path):
         get_response().breadcrumb.append(
@@ -1613,7 +1658,7 @@ class GlobalActionsDirectory(Directory):
         return r.getvalue()
 
 
-class WorkflowPage(Directory):
+class WorkflowPage(Directory, DocumentableMixin):
     _q_exports = [
         '',
         'edit',
@@ -1637,6 +1682,7 @@ class WorkflowPage(Directory):
         ('logged-errors', 'logged_errors_dir'),
         ('history', 'snapshots_dir'),
         ('fullscreen'),
+        ('update-documentation', 'update_documentation'),
     ]
     do_not_call_in_templates = True
 
@@ -1659,6 +1705,8 @@ class WorkflowPage(Directory):
         self.criticality_levels_dir = CriticalityLevelsDirectory(self.workflow)
         self.logged_errors_dir = LoggedErrorsDirectory(parent_dir=self, workflow_id=self.workflow.id)
         self.snapshots_dir = SnapshotsDirectory(self.workflow)
+        self.documented_object = self.workflow
+        self.documented_element = self.workflow
         if component:
             get_response().breadcrumb.append((component + '/', self.workflow.name))
 

wcs/admin/wscalls.py

@@ -21,10 +21,11 @@ from quixote.directory import Directory
 from quixote.html import TemplateIO, htmltext
 
 from wcs.admin import utils
+from wcs.admin.documentable import DocumentableMixin
 from wcs.backoffice.applications import ApplicationsDirectory
 from wcs.backoffice.snapshots import SnapshotsDirectory
 from wcs.qommon import _, errors, misc, template
-from wcs.qommon.form import CheckboxWidget, FileWidget, Form, HtmlWidget, SlugWidget, StringWidget, TextWidget
+from wcs.qommon.form import CheckboxWidget, FileWidget, Form, HtmlWidget, SlugWidget, StringWidget
 from wcs.utils import grep_strings
 from wcs.wscalls import NamedWsCall, NamedWsCallImportError, WsCallRequestWidget
 
@@ -38,9 +39,6 @@ class NamedWsCallUI:
     def get_form(self):
         form = Form(enctype='multipart/form-data', use_tabs=True)
         form.add(StringWidget, 'name', title=_('Name'), required=True, size=30, value=self.wscall.name)
-        form.add(
-            TextWidget, 'description', title=_('Description'), cols=40, rows=5, value=self.wscall.description
-        )
         if self.wscall.slug:
             form.add(
                 SlugWidget,
@@ -100,7 +98,6 @@ class NamedWsCallUI:
             raise ValueError()
 
         self.wscall.name = name
-        self.wscall.description = form.get_widget('description').parse()
         self.wscall.notify_on_errors = form.get_widget('notify_on_errors').parse()
         self.wscall.record_on_errors = form.get_widget('record_on_errors').parse()
         self.wscall.request = form.get_widget('request').parse()
@@ -109,7 +106,7 @@ class NamedWsCallUI:
         self.wscall.store()
 
 
-class NamedWsCallPage(Directory):
+class NamedWsCallPage(Directory, DocumentableMixin):
     do_not_call_in_templates = True
     _q_exports = [
         '',
@@ -118,6 +115,7 @@ class NamedWsCallPage(Directory):
         'export',
         ('history', 'snapshots_dir'),
         'usage',
+        ('update-documentation', 'update_documentation'),
     ]
 
     def __init__(self, component, instance=None):
@@ -128,6 +126,8 @@ class NamedWsCallPage(Directory):
         self.wscall_ui = NamedWsCallUI(self.wscall)
         get_response().breadcrumb.append((component + '/', self.wscall.name))
         self.snapshots_dir = SnapshotsDirectory(self.wscall)
+        self.documented_object = self.wscall
+        self.documented_element = self.wscall
 
     def get_sidebar(self):
         r = TemplateIO(html=True)
@@ -203,6 +203,7 @@ class NamedWsCallPage(Directory):
                 return redirect('../%s/' % self.wscall.id)
 
         get_response().breadcrumb.append(('edit', _('Edit')))
+        get_response().add_javascript(['jquery-ui.js'])
         get_response().set_title(_('Edit webservice call'))
         r = TemplateIO(html=True)
         r += htmltext('<h2>%s</h2>') % _('Edit webservice call')

wcs/api.py

@@ -18,6 +18,7 @@ import base64
 import copy
 import datetime
 import json
+import re
 import urllib.parse
 
 from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse
@@ -26,6 +27,7 @@ from django.utils.timezone import localtime, make_naive
 from quixote import get_publisher, get_request, get_response, get_session, redirect
 from quixote.directory import Directory
 from quixote.errors import MethodNotAllowedError, RequestError
+from quixote.html import TemplateIO, htmltext
 
 import wcs.qommon.storage as st
 from wcs.admin.settings import UserFieldsFormDef
@@ -56,6 +58,7 @@ from wcs.sql_criterias import (
     StrictNotEqual,
 )
 from wcs.workflows import ContentSnapshotPart
+from wcs.wscalls import UnflattenKeysException, unflatten_keys
 
 from .backoffice.data_management import CardPage as BackofficeCardPage
 from .backoffice.management import FormPage as BackofficeFormPage
@@ -1406,6 +1409,7 @@ class ApiDirectory(Directory):
         'geojson',
         'jobs',
         ('card-file-by-token', 'card_file_by_token'),
+        ('preview-payload-structure', 'preview_payload_structure'),
         ('sign-url-token', 'sign_url_token'),
     ]
 
@@ -1433,6 +1437,81 @@ class ApiDirectory(Directory):
         get_response().set_content_type('application/json')
         return json.dumps({'err': 0, 'data': list_roles})
 
+    def preview_payload_structure(self):
+        if not (get_request().user and get_request().user.can_go_in_admin()):
+            raise AccessForbiddenError('user has no access to backoffice')
+
+        def parse_payload():
+            payload = {}
+            for param, value in get_request().form.items():
+                # skip elements which are not part of payload
+                if 'post_data$element' not in param or param.endswith('value_python'):
+                    continue
+                prefix, order, field = re.split(r'(\d+)(?!\d)', param)  # noqa pylint: disable=unused-variable
+                # skip elements that aren't ordered
+                if not order:
+                    continue
+
+                if order not in payload:
+                    payload[order] = []
+
+                if field == 'key':
+                    # skip empty keys
+                    if not value:
+                        continue
+                    # insert key on first position
+                    payload[order].insert(0, value)
+                else:
+                    payload[order].append(value)
+            return dict([v for v in payload.values() if len(v) > 1])
+
+        def format_payload(o, html=htmltext(''), last_element=True):
+            if isinstance(o, (list, tuple)):
+                html += htmltext('[<span class="payload-preview--obj">')
+                while True:
+                    try:
+                        head, tail = o[0], o[1:]
+                    except IndexError:
+                        break
+                    html = format_payload(head, html=html, last_element=len(tail) < 1)
+                    o = tail
+                html += htmltext('</span>]')
+            elif isinstance(o, dict):
+                html += htmltext('{<span class="payload-preview--obj">')
+                for i, (k, v) in enumerate(o.items()):
+                    html += htmltext('<span class="payload-preview--key">"%s"</span>: ' % k)
+                    html = format_payload(v, html=html, last_element=i == len(o) - 1)
+                html += htmltext('</span>}')
+            else:
+                # check if it's empty string, a template with text around or just text
+                if not o or re.sub('^({[{|%]).+([%|}]})$', '', o):
+                    # and add double quotes
+                    html += htmltext('<span class="payload-preview--value">"%s"</span>' % o)
+                else:
+                    html += htmltext('<span class="payload-preview--template-value">%s</span>' % o)
+            # last element doesn't need separator
+            if not last_element:
+                html += htmltext('<span class="payload-preview--item-separator">,</span>')
+            return html
+
+        payload = parse_payload()
+        r = TemplateIO(html=True)
+        r += htmltext('<h2>%s</h2>') % _('Payload structure preview')
+        r += htmltext('<div class="payload-preview">')
+        try:
+            unflattened_payload = unflatten_keys(payload)
+            r += htmltext('<div class="payload-preview--structure">')
+            r += format_payload(unflattened_payload)
+            r += htmltext('</div>')
+        except UnflattenKeysException as e:
+            r += htmltext('<div class="errornotice"><p>%s</p><p>%s %s</p></div>') % (
+                _('Unable to preview payload.'),
+                _('Following error occured: '),
+                e,
+            )
+        r += htmltext('</div>')
+        return r.getvalue()
+
     def _q_traverse(self, path):
         get_request().is_json_marker = True
         return super()._q_traverse(path)
@@ -1476,6 +1555,15 @@ def validate_condition(request, *args, **kwargs):
         Condition(condition).validate()
     except ValidationError as e:
         hint['msg'] = str(e)
+    else:
+        if request.GET.get('warn-on-datetime') == 'true' and condition['type'] == 'django':
+            variables = re.compile(r'\b(today|now)\b')
+            filters = re.compile(r'\|age_in_(years|months|days|hours)')
+            if variables.search(condition['value']) or filters.search(condition['value']):
+                hint['msg'] = _(
+                    'Warning: conditions are only evaluated when entering the action, '
+                    'you may need to set a timeout if you want it to be evaluated regularly.'
+                )
     return JsonResponse(hint)
 
 

wcs/backoffice/cards.py

@@ -98,6 +98,21 @@ class CardFieldDefPage(FormFieldDefPage):
         'Warning: this field data will be permanently deleted from existing cards.'
     )
 
+    def get_deletion_extra_warning(self):
+        warning = super().get_deletion_extra_warning()
+        if warning and self.field.varname and self.objectdef.id_template:
+            varnames = self.field.get_referenced_varnames(self.objectdef, self.objectdef.id_template)
+            if self.field.varname in varnames:
+                warning['level'] = 'error'
+                warning['message'] = htmltext('%s<br>%s') % (
+                    warning['message'],
+                    _(
+                        'This field may be used in the card custom identifiers, '
+                        'its removal may render cards unreachable.'
+                    ),
+                )
+        return warning
+
 
 class CardFieldsDirectory(FormFieldsDirectory):
     field_def_page_class = CardFieldDefPage

wcs/backoffice/filter_fields.py

@@ -0,0 +1,519 @@
+# w.c.s. - web application for online forms
+# Copyright (C) 2005-2024  Entr'ouvert
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+import datetime
+
+from django.utils.encoding import force_str
+from quixote import get_publisher
+from quixote.html import TemplateIO, htmltext
+
+from wcs.qommon import _, misc, pgettext_lazy
+from wcs.qommon.form import DateWidget, SingleSelectWidget, StringWidget
+from wcs.sql_criterias import ArrayContains, Or
+
+
+def render_filter_widget(filter_widget, operators, filter_field_operator_key, filter_field_operator):
+    result = htmltext('<div class="widget operator-and-value-widget">')
+    result += htmltext('<div class="title-and-operator">')
+    result += filter_widget.render_title(filter_widget.get_title())
+    if operators:
+        result += htmltext('<div class="operator">')
+        operator_widget = SingleSelectWidget(
+            filter_field_operator_key,
+            options=[(o[0], o[1], o[0]) for o in operators],
+            value=filter_field_operator,
+            render_br=False,
+        )
+        result += operator_widget.render_content()
+        result += htmltext('</div>')
+    result += htmltext('</div>')
+    result += htmltext('<div class="value">')
+    result += filter_widget.render_content()
+    result += htmltext('</div>')
+    result += htmltext('</div>')
+    return result
+
+
+class FilterField:
+    can_include_in_listing = True
+    id = None
+    key = None
+    label = None
+    available_for_filter = False
+    include_in_statistics = False
+    geojson_label = None
+    store_display_value = None
+    store_structured_value = None
+
+    def __init__(self, formdef):
+        self.formdef = formdef
+        self.varname = self.id.replace('-', '_')
+        self.contextual_id = self.id
+        self.contextual_varname = self.varname
+        self.label = force_str(self.label)  # so it can be pickled
+        self.geojson_label = force_str(self.geojson_label or self.label)
+        self.filter_field_key = 'filter-%s-value' % self.contextual_id
+        self.filter_field_operator_key = '%s-operator' % self.filter_field_key.replace('-value', '')
+        self.filters_dict = {}
+
+    def get_allowed_operators(self):
+        from wcs.variables import LazyFormDefObjectsManager
+
+        lazy_manager = LazyFormDefObjectsManager(formdef=self.formdef)
+        return lazy_manager.get_field_allowed_operators(self) or []
+
+    def get_view_value(self, value):
+        # just here to quack like a duck
+        return None
+
+    def get_csv_heading(self):
+        return [self.label]
+
+    def get_csv_value(self, element, **kwargs):
+        return [element]
+
+    @property
+    def has_relations(self):
+        return bool(self.id == 'user-label')
+
+    def get_filter_field_value(self):
+        return self.filters_dict.get(self.filter_field_key)
+
+    def get_filter_field_operator(self):
+        return self.filters_dict.get(self.filter_field_operator_key) or 'eq'
+
+    def render_filter_widget(self, widget):
+        return render_filter_widget(
+            widget,
+            operators=self.get_allowed_operators(),
+            filter_field_operator_key=self.filter_field_operator_key,
+            filter_field_operator=self.get_filter_field_operator(),
+        )
+
+
+class RelatedField:
+    is_related_field = True
+    key = 'related-field'
+    varname = None
+    related_field = None
+    can_include_in_listing = True
+    available_for_filter = False
+
+    def __init__(self, carddef, field, parent_field):
+        self.carddef = carddef
+        self.related_field = field
+        self.parent_field = parent_field
+        self.parent_field_id = parent_field.id
+
+    @property
+    def id(self):
+        return '%s$%s' % (self.parent_field_id, self.related_field.id)
+
+    @property
+    def contextual_id(self):
+        return self.id
+
+    @property
+    def label(self):
+        return '%s - %s' % (self.parent_field.label, self.related_field.label)
+
+    def __repr__(self):
+        return '<%s (card: %r, parent: %r, related: %r)>' % (
+            self.__class__.__name__,
+            self.carddef,
+            self.parent_field.label,
+            self.related_field.label,
+        )
+
+    @property
+    def store_display_value(self):
+        return self.related_field.store_display_value
+
+    @property
+    def store_structured_value(self):
+        return self.related_field.store_structured_value
+
+    def get_view_value(self, value, **kwargs):
+        if value is None:
+            return ''
+        if isinstance(value, bool):
+            return _('Yes') if value else _('No')
+        if isinstance(value, datetime.date):
+            return misc.strftime(misc.date_format(), value)
+        return value
+
+    def get_view_short_value(self, value, max_len=30, **kwargs):
+        return self.get_view_value(value)
+
+    def get_csv_heading(self):
+        if self.related_field:
+            return self.related_field.get_csv_heading()
+        return [self.label]
+
+    def get_csv_value(self, value, **kwargs):
+        if self.related_field:
+            return self.related_field.get_csv_value(value, **kwargs)
+        return [self.get_view_value(value)]
+
+    def get_column_field_id(self):
+        from wcs.sql import get_field_id
+
+        return get_field_id(self.related_field)
+
+
+class UserRelatedField(RelatedField):
+    # it is named 'user-label' and not 'user' for compatibility with existing
+    # listings, as the 'classic' user column is named 'user-label'.
+    parent_field_id = 'user-label'
+    store_display_value = None
+    store_structured_value = None
+
+    def __init__(self, field):
+        self.related_field = field
+
+    def __repr__(self):
+        return '<%s (field: %r)>' % (
+            self.__class__.__name__,
+            self.related_field.label,
+        )
+
+    @property
+    def label(self):
+        return _('%s of User') % self.related_field.label
+
+
+class UserLabelRelatedField(UserRelatedField):
+    # custom user-label column, targetting the "name" (= full name) column
+    # of the users table
+    id = 'user-label'
+    key = 'user-label'
+    varname = 'user_label'
+    has_relations = True
+
+    def __init__(self):
+        pass
+
+    def __repr__(self):
+        return '<UserLabelRelatedField>'
+
+    def get_column_field_id(self):
+        return 'name'
+
+    @property
+    def label(self):
+        return _('User Label')
+
+
+class DisplayNameFilterField(FilterField):
+    id = 'name'
+    key = 'display_name'
+    label = _('Name')
+
+
+class StatusFilterField(FilterField):
+    id = 'status'
+    key = 'status'
+    label = _('Status')
+    include_in_statistics = True
+
+    def __init__(self, formdef):
+        super().__init__(formdef=formdef)
+        if self.formdef:
+            self.waitpoint_status = self.formdef.workflow.get_waitpoint_status()
+
+    @property
+    def available_for_filter(self):
+        return bool(self.formdef is None or self.waitpoint_status)
+
+    def get_filter_widget(self, mode=None):
+        filter_field_value = self.get_filter_field_value()
+        r = TemplateIO(html=True)
+        operators = [
+            ('eq', '='),
+            ('ne', '!='),
+        ]
+        r += htmltext('<div class="widget operator-and-value-widget">')
+        r += htmltext('<div class="title-and-operator">')
+        r += htmltext('<div class="title">%s</div>') % _('Status to display')
+        if mode != 'stats':
+            r += htmltext('<div class="operator">')
+            operator_widget = SingleSelectWidget(
+                'filter-operator',
+                options=[(o[0], o[1], o[0]) for o in operators],
+                value=self.get_filter_field_operator(),
+                render_br=False,
+            )
+            r += operator_widget.render_content()
+            r += htmltext('</div>')
+        r += htmltext('</div>')
+        r += htmltext('<div class="value content">')
+        r += htmltext('<select name="filter">')
+        filters = [
+            ('waiting', _('Waiting for an action'), None),
+            ('all', _('All'), None),
+            ('pending', pgettext_lazy('formdata', 'Open'), None),
+            ('done', _('Done'), None),
+        ]
+        for status in self.waitpoint_status:
+            filters.append((status.id, status.name, status.colour))
+        for filter_id, filter_label, filter_colour in filters:
+            if filter_id == filter_field_value:
+                selected = ' selected="selected"'
+            else:
+                selected = ''
+            style = ''
+            if filter_colour and filter_colour != '#FFFFFF':
+                fg_colour = misc.get_foreground_colour(filter_colour)
+                style = 'style="background: %s; color: %s;"' % (filter_colour, fg_colour)
+            r += htmltext('<option value="%s"%s %s>' % (filter_id, selected, style))
+            r += htmltext('%s</option>') % filter_label
+        r += htmltext('</select>')
+        r += htmltext('</div>')
+        r += htmltext('</div>')
+        return r.getvalue()
+
+
+class UserVisibleStatusField(FilterField):
+    id = 'user-visible-status'
+    key = 'user-visible-status'
+    label = _('Status (for user)')
+    geolabel_status = _('Status')
+
+
+class InternalIdFilterField(FilterField):
+    id = 'internal-id'
+    key = 'internal-id'
+    label = _('Identifier')
+    available_for_filter = True
+
+    def get_filter_widget(self, **kwargs):
+        widget = StringWidget(
+            self.filter_field_key,
+            title=self.label,
+            value=self.get_filter_field_value(),
+            render_br=False,
+        )
+        return self.render_filter_widget(widget)
+
+
+class AbstractPeriodFilterField(FilterField):
+    available_for_filter = True
+
+    def get_filter_widget(self, **kwargs):
+        return DateWidget(
+            self.filter_field_key, title=self.label, value=self.get_filter_field_value(), render_br=False
+        ).render()
+
+
+class PeriodStartFilterField(AbstractPeriodFilterField):
+    id = 'start'
+    key = 'period-date'
+    label = _('Start')
+
+
+class PeriodEndFilterField(AbstractPeriodFilterField):
+    id = 'end'
+    key = 'period-date'
+    label = _('End')
+
+
+class PeriodStartUpdateTimeFilterField(AbstractPeriodFilterField):
+    id = 'start-mtime'
+    key = 'period-date'
+    label = _('Start (modification time)')
+
+
+class PeriodEndUpdateTimeFilterField(AbstractPeriodFilterField):
+    id = 'end-mtime'
+    key = 'period-date'
+    label = _('End (modification time)')
+
+
+class UserIdFilterField(FilterField):
+    id = 'user'
+    key = 'user-id'
+    label = _('User')
+    available_for_filter = True
+
+    def get_allowed_operators(self):
+        return []
+
+    def get_filter_widget(self, **kwargs):
+        filter_field_value = self.get_filter_field_value()
+        options = [
+            ('', _('None'), ''),
+            ('__current__', _('Current user'), '__current__'),
+        ]
+        if filter_field_value and filter_field_value != '__current__':
+            try:
+                filtered_user = get_publisher().user_class.get(filter_field_value)
+            except KeyError:
+                filtered_user = None
+            filtered_user_value = filtered_user.display_name if filtered_user else _('Unknown')
+            options += [(filter_field_value, filtered_user_value, filter_field_value)]
+        widget = SingleSelectWidget(
+            self.filter_field_key,
+            title=self.label,
+            options=options,
+            value=filter_field_value,
+            render_br=False,
+        )
+        return self.render_filter_widget(widget)
+
+
+class UserFunctionFilterField(FilterField):
+    id = 'user-function'
+    key = 'user-function'
+    label = _('Current User Function')
+    available_for_filter = True
+
+    def get_allowed_operators(self):
+        return []
+
+    def get_filter_widget(self, **kwargs):
+        options = [('', '', '')] + [(x[0], x[1], x[0]) for x in self.formdef.workflow.get_sorted_functions()]
+        widget = SingleSelectWidget(
+            self.filter_field_key,
+            title=self.label,
+            options=options,
+            value=self.get_filter_field_value(),
+            render_br=False,
+        )
+        return self.render_filter_widget(widget)
+
+
+class SubmissionAgentFilterField(FilterField):
+    id = 'submission-agent'
+    key = 'submission-agent'
+    label = _('Submission Agent')
+
+    @property
+    def available_for_filter(self):
+        return bool(self.formdef.backoffice_submission_roles)
+
+    def get_filter_widget(self, **kwargs):
+        filter_field_value = self.get_filter_field_value()
+        options = [
+            ('', '', ''),
+            ('__current__', _('Current user'), '__current__'),
+        ]
+        if filter_field_value == '-1':
+            # this happens when ?filter-submission-agent-uuid is given with an unknown uuid,
+            # an option for "invalid user" is added so refreshs or new filters won't reset
+            # this filter.
+            options.append(('-1', _('Invalid user'), '-1'))
+        options.extend(
+            [
+                (str(x.id), x.display_name, str(x.id))
+                for x in get_publisher().user_class.select(
+                    [Or([ArrayContains('roles', [str(y)]) for y in self.formdef.backoffice_submission_roles])]
+                )
+            ]
+        )
+        widget = SingleSelectWidget(
+            self.filter_field_key,
+            title=self.label,
+            options=options,
+            value=filter_field_value,
+            render_br=False,
+        )
+        return self.render_filter_widget(widget)
+
+
+class SubmissionChannelFilterField(FilterField):
+    id = 'submission_channel'
+    key = 'submission_channel'
+    label = _('Channel')
+
+
+class CriticalityLevelFilterFiled(FilterField):
+    id = 'criticality-level'
+    key = 'criticality-level'
+    label = _('Criticality Level')
+
+    @property
+    def available_for_filter(self):
+        return bool(self.formdef.workflow.criticality_levels)
+
+    def get_allowed_operators(self):
+        return []
+
+    def get_filter_widget(self, **kwargs):
+        options = [('', pgettext_lazy('criticality-level', 'All'), '')] + [
+            (str(i), x.name, str(i)) for i, x in enumerate(self.formdef.workflow.criticality_levels)
+        ]
+        widget = SingleSelectWidget(
+            self.filter_field_key,
+            title=self.label,
+            options=options,
+            value=self.get_filter_field_value(),
+            render_br=False,
+        )
+        return self.render_filter_widget(widget)
+
+
+class DigestFilterField(FilterField):
+    id = 'digest'
+    key = 'digest'
+    label = _('Digest')
+
+
+class IdFilterField(FilterField):
+    id = 'id'
+    key = 'id'
+
+    def __init__(self, formdef):
+        super().__init__(formdef=formdef)
+        self.label = force_str(_('Identifier') if self.formdef.id_template else _('Number'))
+
+
+class TimeFilterField(FilterField):
+    id = 'time'
+    key = 'time'
+    label = _('Created')
+
+
+class LastUpdateFilterField(FilterField):
+    id = 'last_update_time'
+    key = 'last_update_time'
+    label = _('Last Modified')
+
+
+class AnonymisedFilterField(FilterField):
+    id = 'anonymised'
+    key = 'anonymised'
+    label = _('Anonymised')
+
+
+class NumberFilterField(FilterField):
+    id = 'number'
+    key = 'number'
+    label = _('Number')
+    available_for_filter = True
+
+
+class IdentifierFilterField(FilterField):
+    id = 'identifier'
+    key = 'identifier'
+    label = _('Identifier')
+    available_for_filter = True
+
+
+class DistanceFilterField(FilterField):
+    id = 'distance'
+    key = 'distance'
+    label = _('Distance')
+    available_for_filter = True

wcs/backoffice/management.py

@@ -34,6 +34,7 @@ from quixote.http_request import parse_query
 
 from wcs.api_access import ApiAccess
 from wcs.api_utils import get_query_flag, get_user_from_api_query_string
+from wcs.backoffice import filter_fields
 from wcs.backoffice.pagination import pagination_links
 from wcs.carddef import CardDef
 from wcs.categories import Category
@@ -761,8 +762,8 @@ class ManagementDirectory(Directory):
         criterias = self.get_global_listing_criterias()
         formdatas = sql.AnyFormData.select(criterias + [NotNull('geoloc_base_x'), Null('anonymised')])
         fields = [
-            FakeField('name', 'display_name', _('Name')),
-            FakeField('status', 'status', _('Status')),
+            filter_fields.DisplayNameFilterField(formdef=None),
+            filter_fields.StatusFilterField(formdef=None),
         ]
         get_response().set_content_type('application/json')
         return json.dumps(geojson_formdatas(formdatas, fields=fields), cls=misc.JSONEncoder)
@@ -1132,28 +1133,6 @@ class FormPage(Directory, TempfileDirectoryMixin):
             {'err': 0, 'data': [{'id': x[0], 'text': x[1]} for x in options]}, cls=misc.JSONEncoder
         )
 
-    def get_filterable_field_types(self):
-        types = [
-            'string',
-            'text',
-            'email',
-            'item',
-            'bool',
-            'numeric',
-            'items',
-            'internal-id',
-            'identifier',
-            'number',
-            'period-date',
-            'user-id',
-            'user-function',
-            'submission-agent-id',
-            'date',
-            'distance',
-            'criticality-level',
-        ]
-        return types
-
     def get_filter_sidebar(
         self,
         selected_filter=None,
@@ -1164,27 +1143,26 @@ class FormPage(Directory, TempfileDirectoryMixin):
     ):
         r = TemplateIO(html=True)
 
-        waitpoint_status = self.formdef.workflow.get_waitpoint_status()
         fake_fields = [
-            FakeField('internal-id', 'internal-id', _('Identifier')),
-            FakeField('start', 'period-date', _('Start')),
-            FakeField('end', 'period-date', _('End')),
-            FakeField('user', 'user-id', _('User')),
-            FakeField('user-function', 'user-function', _('Current User Function')),
-            FakeField('submission-agent', 'submission-agent-id', _('Submission Agent'), addable=False),
+            klass(formdef=self.formdef)
+            for klass in (
+                filter_fields.InternalIdFilterField,
+                filter_fields.PeriodStartFilterField,
+                filter_fields.PeriodEndFilterField,
+                filter_fields.UserIdFilterField,
+                filter_fields.UserFunctionFilterField,
+                filter_fields.CriticalityLevelFilterFiled,
+            )
         ]
-        if self.formdef.workflow.criticality_levels:
-            fake_fields.append(FakeField('criticality-level', 'criticality-level', _('Criticality Level')))
         default_filters = self.get_default_filters(mode)
 
-        filter_fields = []
+        available_fields = []
         for field in fake_fields + list(self.get_formdef_fields()):
             field.enabled = False
-            if field.key not in self.get_filterable_field_types() + ['status']:
+            field.formdef = self.formdef
+            if not field.available_for_filter:
                 continue
-            if field.key == 'status' and not waitpoint_status:
-                continue
-            filter_fields.append(field)
+            available_fields.append(field)
 
             if getattr(field, 'block_field', None):
                 field.label = '%s / %s' % (field.block_field.label, field.label)
@@ -1242,31 +1220,21 @@ class FormPage(Directory, TempfileDirectoryMixin):
             filters_dict.update(self.view.get_filters_dict())
         filters_dict.update(get_request().form)
 
-        def render_widget(filter_widget, operators):
-            result = htmltext('<div class="widget operator-and-value-widget">')
-            result += htmltext('<div class="title-and-operator">')
-            result += filter_widget.render_title(filter_widget.get_title())
-            if operators:
-                result += htmltext('<div class="operator">')
-                operator_widget = SingleSelectWidget(
-                    filter_field_operator_key,
-                    options=[(o[0], o[1], o[0]) for o in operators],
-                    value=filter_field_operator,
-                    render_br=False,
-                )
-                result += operator_widget.render_content()
-                result += htmltext('</div>')
-            result += htmltext('</div>')
-            result += htmltext('<div class="value">')
-            result += filter_widget.render_content()
-            result += htmltext('</div>')
-            result += htmltext('</div>')
-            return result
+        if selected_filter:
+            filters_dict['filter-status-value'] = selected_filter
+            filters_dict['filter-status-operator'] = selected_filter_operator
 
-        for filter_field in filter_fields:
+        def render_widget(filter_widget, operators):
+            return filter_fields.render_filter_widget(
+                filter_widget, operators, filter_field_operator_key, filter_field_operator
+            )
+
+        for filter_field in available_fields:
             if not filter_field.enabled:
                 continue
 
+            filter_field.filters_dict = filters_dict
+
             filter_field_key = 'filter-%s-value' % filter_field.contextual_id
             filter_field_value = filters_dict.get(filter_field_key)
 
@@ -1276,124 +1244,8 @@ class FormPage(Directory, TempfileDirectoryMixin):
             lazy_manager = LazyFormDefObjectsManager(formdef=self.formdef)
             operators = lazy_manager.get_field_allowed_operators(filter_field) or []
 
-            if filter_field.key == 'status':
-                operators = [
-                    ('eq', '='),
-                    ('ne', '!='),
-                ]
-                r += htmltext('<div class="widget operator-and-value-widget">')
-                r += htmltext('<div class="title-and-operator">')
-                r += htmltext('<div class="title">%s</div>') % _('Status to display')
-                if mode != 'stats':
-                    r += htmltext('<div class="operator">')
-                    operator_widget = SingleSelectWidget(
-                        'filter-operator',
-                        options=[(o[0], o[1], o[0]) for o in operators],
-                        value=selected_filter_operator,
-                        render_br=False,
-                    )
-                    r += operator_widget.render_content()
-                    r += htmltext('</div>')
-                r += htmltext('</div>')
-                r += htmltext('<div class="value content">')
-                r += htmltext('<select name="filter">')
-                filters = [
-                    ('waiting', _('Waiting for an action'), None),
-                    ('all', _('All'), None),
-                    ('pending', pgettext_lazy('formdata', 'Open'), None),
-                    ('done', _('Done'), None),
-                ]
-                for status in waitpoint_status:
-                    filters.append((status.id, status.name, status.colour))
-                for filter_id, filter_label, filter_colour in filters:
-                    if filter_id == selected_filter:
-                        selected = ' selected="selected"'
-                    else:
-                        selected = ''
-                    style = ''
-                    if filter_colour and filter_colour != '#FFFFFF':
-                        fg_colour = misc.get_foreground_colour(filter_colour)
-                        style = 'style="background: %s; color: %s;"' % (filter_colour, fg_colour)
-                    r += htmltext('<option value="%s"%s %s>' % (filter_id, selected, style))
-                    r += htmltext('%s</option>') % filter_label
-                r += htmltext('</select>')
-                r += htmltext('</div>')
-                r += htmltext('</div>')
-
-            elif filter_field.key == 'period-date':
-                r += DateWidget(
-                    filter_field_key, title=filter_field.label, value=filter_field_value, render_br=False
-                ).render()
-
-            elif filter_field.key == 'user-id':
-                options = [
-                    ('', _('None'), ''),
-                    ('__current__', _('Current user'), '__current__'),
-                ]
-                if filter_field_value and filter_field_value != '__current__':
-                    try:
-                        filtered_user = get_publisher().user_class.get(filter_field_value)
-                    except KeyError:
-                        filtered_user = None
-                    filtered_user_value = filtered_user.display_name if filtered_user else _('Unknown')
-                    options += [(filter_field_value, filtered_user_value, filter_field_value)]
-                widget = SingleSelectWidget(
-                    filter_field_key,
-                    title=filter_field.label,
-                    options=options,
-                    value=filter_field_value,
-                    render_br=False,
-                )
-                r += render_widget(widget, operators=[])
-
-            elif filter_field.key == 'submission-agent-id':
-                r += HiddenWidget(filter_field_key, value=filter_field_value).render()
-                if filter_field_value:
-                    filtered_user = get_publisher().user_class.get(filter_field_value, ignore_errors=True)
-                    widget = StringWidget(
-                        '_' + filter_field_key,
-                        title=filter_field.label,
-                        value=filtered_user.display_name if filtered_user else _('Unknown'),
-                        readonly=True,
-                        render_br=False,
-                    )
-                    widget._parsed = True  # make sure value is not replaced by request query
-                    r += widget.render()
-
-            elif filter_field.key == 'user-function':
-                options = [('', '', '')] + [
-                    (x[0], x[1], x[0]) for x in self.formdef.workflow.get_sorted_functions()
-                ]
-                widget = SingleSelectWidget(
-                    filter_field_key,
-                    title=filter_field.label,
-                    options=options,
-                    value=filter_field_value,
-                    render_br=False,
-                )
-                r += render_widget(widget, operators=[])
-
-            elif filter_field.key == 'internal-id':
-                widget = StringWidget(
-                    filter_field_key,
-                    title=filter_field.label,
-                    value=filter_field_value,
-                    render_br=False,
-                )
-                r += render_widget(widget, operators)
-
-            elif filter_field.key == 'criticality-level':
-                options = [('', pgettext_lazy('criticality-level', 'All'), '')] + [
-                    (str(i), x.name, str(i)) for i, x in enumerate(self.formdef.workflow.criticality_levels)
-                ]
-                widget = SingleSelectWidget(
-                    filter_field_key,
-                    title=filter_field.label,
-                    options=options,
-                    value=filter_field_value,
-                    render_br=False,
-                )
-                r += render_widget(widget, operators=[])
+            if hasattr(filter_field, 'get_filter_widget'):
+                r += filter_field.get_filter_widget(mode=mode)
 
             elif filter_field.key in ('item', 'items'):
                 filter_field.required = False
@@ -1485,9 +1337,8 @@ class FormPage(Directory, TempfileDirectoryMixin):
         # field filter dialog content
         r += htmltext('<div style="display: none;">')
         r += htmltext('<ul id="field-filter" class="objects-list">')
-        for field in filter_fields:
-            addable = getattr(field, 'addable', True)
-            r += htmltext('<li %s>') % ('' if addable else 'hidden')
+        for field in available_fields:
+            r += htmltext('<li>')
             r += htmltext('<label for="fields-filter-%s">') % field.contextual_id
             r += htmltext('<input type="checkbox" name="filter-%s"') % field.contextual_id
             if field.enabled:
@@ -1591,7 +1442,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
                     classnames = 'has-relations-field'
                     attrs = 'data-field-id="%s"' % field.id
                     seen_parents.add(field.id)
-                elif isinstance(field, RelatedField):
+                elif isinstance(field, filter_fields.RelatedField):
                     classnames = 'related-field'
                     if field.parent_field_id in seen_parents:
                         classnames += ' collapsed'
@@ -1874,24 +1725,24 @@ class FormPage(Directory, TempfileDirectoryMixin):
             return redirect('..')
 
     def get_formdef_fields(self, include_block_items_fields=False):
-        yield FakeField('id', 'id', _('Identifier') if self.formdef.id_template else _('Number'))
+        yield filter_fields.IdFilterField(formdef=self.formdef)
         if self.formdef.default_digest_template:
-            yield FakeField('digest', 'digest', _('Digest'))
-        yield FakeField('submission_channel', 'submission_channel', _('Channel'))
+            yield filter_fields.DigestFilterField(formdef=self.formdef)
+        yield filter_fields.SubmissionChannelFilterField(formdef=self.formdef)
         if self.formdef.backoffice_submission_roles:
-            yield FakeField('submission_agent', 'submission_agent', _('Submission By'))
-        yield FakeField('time', 'time', _('Created'))
-        yield FakeField('last_update_time', 'last_update_time', _('Last Modified'))
+            yield filter_fields.SubmissionAgentFilterField(formdef=self.formdef)
+        yield filter_fields.TimeFilterField(formdef=self.formdef)
+        yield filter_fields.LastUpdateFilterField(formdef=self.formdef)
 
         # user fields
         # user-label field but as a custom field, to get full name of user
         # using a sql join clause.
-        yield UserLabelRelatedField()
+        yield filter_fields.UserLabelRelatedField()
         for field in get_publisher().user_class.get_fields():
             if not field.can_include_in_listing:
                 continue
             field.has_relations = True
-            yield UserRelatedField(field)
+            yield filter_fields.UserRelatedField(field)
 
         for field in self.formdef.iter_fields(include_block_fields=True):
             if getattr(field, 'block_field', None):
@@ -1917,17 +1768,12 @@ class FormPage(Directory, TempfileDirectoryMixin):
                 if not card_field.can_include_in_listing:
                     continue
                 field.has_relations = True
-                yield RelatedField(carddef, card_field, field)
+                yield filter_fields.RelatedField(carddef, card_field, field)
 
-        yield FakeField('status', 'status', _('Status'), include_in_statistics=True)
+        yield filter_fields.StatusFilterField(formdef=self.formdef)
         if any(x.get_visibility_mode() != 'all' for x in self.formdef.workflow.possible_status):
-            yield FakeField(
-                'user-visible-status',
-                'user-visible-status',
-                _('Status (for user)'),
-                geojson_label=_('Status'),
-            )
-        yield FakeField('anonymised', 'anonymised', _('Anonymised'))
+            yield filter_fields.UserVisibleStatusField(formdef=self.formdef)
+        yield filter_fields.AnonymisedFilterField(formdef=self.formdef)
 
     def get_default_columns(self):
         if self.view:
@@ -2004,18 +1850,21 @@ class FormPage(Directory, TempfileDirectoryMixin):
         statistics_fields_only=False,
     ):
         fake_fields = [
-            FakeField('internal-id', 'internal-id', _('Identifier')),
-            FakeField('number', 'number', _('Number')),
-            FakeField('identifier', 'identifier', _('Identifier')),
-            FakeField('start', 'period-date', _('Start')),
-            FakeField('end', 'period-date', _('End')),
-            FakeField('start-mtime', 'period-date', _('Start (modification time)')),
-            FakeField('end-mtime', 'period-date', _('End (modification time)')),
-            FakeField('user', 'user-id', _('User')),
-            FakeField('user-function', 'user-function', _('Current User Function')),
-            FakeField('submission-agent', 'submission-agent-id', _('Submission Agent')),
-            FakeField('distance', 'distance', _('Distance')),
-            FakeField('criticality-level', 'criticality-level', _('Criticality Level')),
+            klass(formdef=self.formdef)
+            for klass in (
+                filter_fields.InternalIdFilterField,
+                filter_fields.NumberFilterField,
+                filter_fields.IdentifierFilterField,
+                filter_fields.PeriodStartFilterField,
+                filter_fields.PeriodEndFilterField,
+                filter_fields.PeriodStartUpdateTimeFilterField,
+                filter_fields.PeriodEndUpdateTimeFilterField,
+                filter_fields.UserIdFilterField,
+                filter_fields.UserFunctionFilterField,
+                filter_fields.SubmissionAgentFilterField,
+                filter_fields.DistanceFilterField,
+                filter_fields.CriticalityLevelFilterFiled,
+            )
         ]
         criterias = []
 
@@ -2052,7 +1901,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
             criterias.append(Nothing())
 
         for filter_field in fake_fields + list(self.get_formdef_fields(include_block_items_fields=True)):
-            if filter_field.key not in self.get_filterable_field_types():
+            if not filter_field.available_for_filter:
                 continue
 
             if statistics_fields_only and not getattr(filter_field, 'include_in_statistics', False):
@@ -2108,7 +1957,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
                     # allow for short form, with a single query parameter
                     filters_dict['filter-user-function-value'] = filters_dict.get('filter-user-function')
 
-            if filter_field.key == 'submission-agent-id':
+            if filter_field.key == 'submission-agent':
                 # convert uuid based filter into local id filter
                 name_id = filters_dict.get('filter-submission-agent-uuid')
                 if name_id:
@@ -2284,7 +2133,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
                 elif filter_field.id == 'end-mtime':
                     criterias.append(LessOrEqual('last_update_time', filter_date_value))
                 criterias[-1]._label = '%s: %s' % (filter_field.label, filter_field_value)
-            elif filter_field.key == 'user-id':
+            elif filter_field.key in ('submission-agent', 'user-id'):
                 if filter_field_value == '__current__':
                     context_vars = get_publisher().substitutions.get_context_variables(mode='lazy')
                     if request and request.is_in_backoffice() and context_vars.get('form'):
@@ -2299,10 +2148,10 @@ class FormPage(Directory, TempfileDirectoryMixin):
                         filter_field_value = None
                 if filter_field_value in ('__current__', None):
                     criterias.append(Nothing())
-                else:
+                elif filter_field.key == 'user-id':
                     criterias.append(Equal('user_id', filter_field_value))
-            elif filter_field.key == 'submission-agent-id':
-                criterias.append(Equal('submission_agent_id', filter_field_value))
+                elif filter_field.key == 'submission-agent':
+                    criterias.append(Equal('submission_agent_id', filter_field_value))
             elif filter_field.key == 'user-function':
                 user_object = None
                 if ':' in filter_field_value:
@@ -4062,7 +3911,7 @@ class FormBackOfficeStatusPage(FormStatusPage):
                     if field_url:
                         r += htmltext(' <a title="%s" href="%s"></a>' % (v._field.label, field_url))
                 r += htmltext('</code>')
-                r += htmltext('  <div class="value"><span>%s</span>') % v
+                r += htmltext('  <div class="value"><span>%s</span>') % misc.mark_spaces(v)
                 if isinstance(v, NoneFieldVar):
                     r += htmltext(' <span class="type">(%s)</span>') % _('no value')
             elif isinstance(v, (types.FunctionType, types.MethodType)):
@@ -4103,7 +3952,13 @@ class FormBackOfficeStatusPage(FormStatusPage):
                     r += ', '.join(custom_repr(x) for x in v)
                     r += htmltext(']</span>')
                 else:
-                    r += htmltext('  <div class="value"><span>%s</span>') % ellipsize(safe(v), 10000)
+                    if k in ('form_details', 'form_evolution'):
+                        # do not mark spaces in those variables
+                        r += htmltext('  <div class="value"><span>%s</span>') % ellipsize(safe(v), 10000)
+                    else:
+                        r += htmltext('  <div class="value"><span>%s</span>') % misc.mark_spaces(
+                            ellipsize(safe(v), 10000)
+                        )
                 if not isinstance(v, str):
                     r += htmltext(' <span class="type">(%s)</span>') % get_type_name(v)
             r += htmltext('</div></li>')
@@ -4202,148 +4057,6 @@ class FormBackOfficeStatusPage(FormStatusPage):
         return self.test_tool_result()
 
 
-class FakeField:
-    can_include_in_listing = True
-
-    def __init__(self, id, type_key, label, addable=True, include_in_statistics=False, geojson_label=None):
-        self.id = id
-        self.contextual_id = self.id
-        self.key = type_key
-        self.label = force_str(label)
-        self.fake = True
-        self.varname = id.replace('-', '_')
-        self.contextual_varname = self.varname
-        self.store_display_value = None
-        self.store_structured_value = None
-        self.addable = addable
-        self.include_in_statistics = include_in_statistics
-        self.geojson_label = force_str(geojson_label or self.label)
-
-    def get_view_value(self, value):
-        # just here to quack like a duck
-        return None
-
-    def get_csv_heading(self):
-        return [self.label]
-
-    def get_csv_value(self, element, **kwargs):
-        return [element]
-
-    @property
-    def has_relations(self):
-        return bool(self.id == 'user-label')
-
-
-class RelatedField:
-    is_related_field = True
-    key = 'related-field'
-    varname = None
-    related_field = None
-    can_include_in_listing = True
-
-    def __init__(self, carddef, field, parent_field):
-        self.carddef = carddef
-        self.related_field = field
-        self.parent_field = parent_field
-        self.parent_field_id = parent_field.id
-
-    @property
-    def id(self):
-        return '%s$%s' % (self.parent_field_id, self.related_field.id)
-
-    @property
-    def contextual_id(self):
-        return self.id
-
-    @property
-    def label(self):
-        return '%s - %s' % (self.parent_field.label, self.related_field.label)
-
-    def __repr__(self):
-        return '<%s (card: %r, parent: %r, related: %r)>' % (
-            self.__class__.__name__,
-            self.carddef,
-            self.parent_field.label,
-            self.related_field.label,
-        )
-
-    @property
-    def store_display_value(self):
-        return self.related_field.store_display_value
-
-    @property
-    def store_structured_value(self):
-        return self.related_field.store_structured_value
-
-    def get_view_value(self, value, **kwargs):
-        if value is None:
-            return ''
-        if isinstance(value, bool):
-            return _('Yes') if value else _('No')
-        if isinstance(value, datetime.date):
-            return misc.strftime(misc.date_format(), value)
-        return value
-
-    def get_view_short_value(self, value, max_len=30, **kwargs):
-        return self.get_view_value(value)
-
-    def get_csv_heading(self):
-        if self.related_field:
-            return self.related_field.get_csv_heading()
-        return [self.label]
-
-    def get_csv_value(self, value, **kwargs):
-        if self.related_field:
-            return self.related_field.get_csv_value(value, **kwargs)
-        return [self.get_view_value(value)]
-
-    def get_column_field_id(self):
-        return get_field_id(self.related_field)
-
-
-class UserRelatedField(RelatedField):
-    # it is named 'user-label' and not 'user' for compatibility with existing
-    # listings, as the 'classic' user column is named 'user-label'.
-    parent_field_id = 'user-label'
-    store_display_value = None
-    store_structured_value = None
-
-    def __init__(self, field):
-        self.related_field = field
-
-    def __repr__(self):
-        return '<%s (field: %r)>' % (
-            self.__class__.__name__,
-            self.related_field.label,
-        )
-
-    @property
-    def label(self):
-        return _('%s of User') % self.related_field.label
-
-
-class UserLabelRelatedField(UserRelatedField):
-    # custom user-label column, targetting the "name" (= full name) column
-    # of the users table
-    id = 'user-label'
-    key = 'user-label'
-    varname = 'user_label'
-    has_relations = True
-
-    def __init__(self):
-        pass
-
-    def __repr__(self):
-        return '<UserLabelRelatedField>'
-
-    def get_column_field_id(self):
-        return 'name'
-
-    @property
-    def label(self):
-        return _('User Label')
-
-
 def do_graphs_section(period_start=None, period_end=None, criterias=None):
     from wcs import sql
 
@@ -4846,3 +4559,10 @@ class JsonFileExportAfterJob(CsvExportAfterJob):
         )
         self.content_type = 'application/json'
         self.store()
+
+
+class FakeField:
+    # 2024-04-12, legacy class, for transition from FakeField to filter_fields.*
+    # can be removed once all afterjobs referencing fake fields are removed.
+    # (= 2 days after update)
+    pass

wcs/backoffice/snapshots.py

@@ -52,7 +52,8 @@ class SnapshotsDirectory(Directory):
             templates=['wcs/backoffice/snapshots.html'],
             context={
                 'view': self,
-                'form_has_tests': bool(TestDef.select_for_objectdef(self.obj)),
+                'form_has_tests': self.object_type in ('formdef', 'carddef')
+                and bool(TestDef.select_for_objectdef(self.obj)),
             },
         )
 
@@ -144,6 +145,8 @@ class SnapshotsDirectory(Directory):
                 'snapshot2': snapshot2,
             }
         )
+        get_response().add_javascript(['gadjo.snapshotdiff.js'])
+        get_response().add_css_include('gadjo.snapshotdiff.css')
         return template.QommonTemplateResponse(
             templates=['wcs/backoffice/snapshots_compare.html'],
             context=context,
@@ -261,7 +264,7 @@ class SnapshotsDirectory(Directory):
         current_date = None
         snapshots = get_publisher().snapshot_class.select_object_history(self.obj)
         test_results = TestResult.select(
-            [Equal('object_type', self.obj.get_table_name()), Equal('object_id', self.obj.id)]
+            [Equal('object_type', self.obj.get_table_name()), Equal('object_id', str(self.obj.id))]
         )
         test_results_by_id = {x.id: x for x in test_results}
         day_snapshot = None

wcs/backoffice/studio.py

@@ -98,7 +98,7 @@ class StudioDirectory(Directory):
         backoffice_root = get_publisher().get_backoffice_root()
         object_types = []
         if backoffice_root.is_accessible('forms'):
-            extra_links.append(('../forms/blocks/', pgettext('studio', 'Field blocks')))
+            extra_links.append(('../forms/blocks/', pgettext('studio', 'Blocks of fields')))
         if backoffice_root.is_accessible('workflows'):
             extra_links.append(('../workflows/mail-templates/', pgettext('studio', 'Mail templates')))
             extra_links.append(('../workflows/comment-templates/', pgettext('studio', 'Comment templates')))

wcs/blocks.py

@@ -51,8 +51,8 @@ class BlockDef(StorableObject):
     _indexes = ['slug']
     backoffice_class = 'wcs.admin.blocks.BlockDirectory'
     xml_root_node = 'block'
-    verbose_name = _('Field block')
-    verbose_name_plural = _('Field blocks')
+    verbose_name = _('Block of fields')
+    verbose_name_plural = _('Blocks of fields')
     var_prefixes = ['block']
 
     name = None
@@ -60,11 +60,12 @@ class BlockDef(StorableObject):
     fields = None
     digest_template = None
     category_id = None
+    documentation = None
 
     SLUG_DASH = '_'
 
     # declarations for serialization
-    TEXT_ATTRIBUTES = ['name', 'slug', 'digest_template']
+    TEXT_ATTRIBUTES = ['name', 'slug', 'digest_template', 'documentation']
 
     def __init__(self, name=None, **kwargs):
         super().__init__(**kwargs)

wcs/comment_templates.py

@@ -34,7 +34,7 @@ class CommentTemplate(XmlStorableObject):
 
     name = None
     slug = None
-    description = None
+    documentation = None
     comment = None
     attachments = []
     category_id = None
@@ -43,7 +43,8 @@ class CommentTemplate(XmlStorableObject):
     XML_NODES = [
         ('name', 'str'),
         ('slug', 'str'),
-        ('description', 'str'),
+        ('description', 'str'),  # legacy
+        ('documentation', 'str'),
         ('comment', 'str'),
         ('attachments', 'str_list'),
     ]
@@ -52,6 +53,16 @@ class CommentTemplate(XmlStorableObject):
         XmlStorableObject.__init__(self)
         self.name = name
 
+    def migrate(self):
+        changed = False
+        if getattr(self, 'description', None):  # 2024-04-07
+            self.documentation = getattr(self, 'description')
+            self.description = None
+            changed = True
+        if changed:
+            self.store(comment=_('Automatic update'), snapshot_store_user=False)
+        return changed
+
     @property
     def category(self):
         return CommentTemplateCategory.get(self.category_id, ignore_errors=True)
@@ -67,14 +78,16 @@ class CommentTemplate(XmlStorableObject):
         base_url = get_publisher().get_backoffice_url()
         return '%s/workflows/comment-templates/%s/' % (base_url, self.id)
 
-    def store(self, comment=None, application=None, *args, **kwargs):
+    def store(self, comment=None, snapshot_store_user=True, application=None, *args, **kwargs):
         assert not self.is_readonly()
         if self.slug is None:
             # set slug if it's not yet there
             self.slug = self.get_new_slug()
         super().store(*args, **kwargs)
         if get_publisher().snapshot_class:
-            get_publisher().snapshot_class.snap(instance=self, comment=comment, application=application)
+            get_publisher().snapshot_class.snap(
+                instance=self, store_user=snapshot_store_user, comment=comment, application=application
+            )
 
     def get_places_of_use(self):
         from wcs.workflows import Workflow

wcs/custom_views.py

@@ -47,6 +47,17 @@ class CustomView(StorableObject):
 
     xml_root_node = 'custom_view'
 
+    def migrate(self):
+        changed = False
+        # 2024-04-10
+        if self.columns and 'submission_agent' in [x['id'] for x in self.columns['list']]:
+            self.columns['list'] = [
+                {'id': x['id'].replace('submission_agent', 'submission-agent')} for x in self.columns['list']
+            ]
+            changed = True
+        if changed:
+            self.store()
+
     @property
     def user(self):
         return get_publisher().user_class.get(self.user_id)

wcs/data_sources.py

@@ -676,7 +676,7 @@ class NamedDataSource(XmlStorableObject):
 
     name = None
     slug = None
-    description = None
+    documentation = None
     data_source = None
     cache_duration = None
     query_parameter = None
@@ -702,7 +702,8 @@ class NamedDataSource(XmlStorableObject):
     XML_NODES = [
         ('name', 'str'),
         ('slug', 'str'),
-        ('description', 'str'),
+        ('description', 'str'),  # legacy
+        ('documentation', 'str'),
         ('cache_duration', 'str'),
         ('query_parameter', 'str'),
         ('id_parameter', 'str'),
@@ -737,6 +738,11 @@ class NamedDataSource(XmlStorableObject):
                 self.data_source['value'] = translate_url(publisher, url)
                 changed = True
 
+        if getattr(self, 'description', None):  # 2024-04-07
+            self.documentation = getattr(self, 'description')
+            self.description = None
+            changed = True
+
         if changed:
             self.store(comment=_('Automatic update'), snapshot_store_user=False)
 
@@ -1117,6 +1123,8 @@ class NamedDataSource(XmlStorableObject):
         elif self.type == 'json' and self.id_parameter:
             value = self.get_value_by_id(self.id_parameter, option_id)
         elif self.type == 'wcs:users':
+            if isinstance(option_id, get_publisher().user_class):
+                option_id = option_id.id
             value = get_publisher().user_class.get_user_with_roles(
                 option_id,
                 included_roles=self.users_included_roles,

wcs/fields/base.py

@@ -222,6 +222,8 @@ class Field:
     condition = None
     is_no_data_field = False
     can_include_in_listing = False
+    available_for_filter = False
+    documentation = None
 
     # flag a field for removal by AnonymiseWorkflowStatusItem
     # possible values are final, intermediate, no.
@@ -231,7 +233,7 @@ class Field:
 
     # declarations for serialization, they are mostly for legacy files,
     # new exports directly include typing attributes.
-    TEXT_ATTRIBUTES = ['label', 'type', 'hint', 'varname', 'extra_css_class']
+    TEXT_ATTRIBUTES = ['label', 'type', 'hint', 'varname', 'extra_css_class', 'documentation']
 
     def __init__(self, **kwargs):
         for k, v in kwargs.items():
@@ -319,7 +321,7 @@ class Field:
 
     def export_to_xml(self, include_id=False):
         field = ET.Element('field')
-        extra_fields = ['default_value']  # specific to workflow variables
+        extra_fields = ['default_value', 'documentation']  # default_value is specific to workflow variables
         if include_id:
             extra_fields.append('id')
         ET.SubElement(field, 'type').text = self.key
@@ -358,7 +360,7 @@ class Field:
         return field
 
     def init_with_xml(self, elem, include_id=False, snapshot=False):
-        extra_fields = ['default_value']  # specific to workflow variables
+        extra_fields = ['documentation', 'default_value']  # default_value is specific to workflow variables
         for attribute in self.get_admin_attributes() + extra_fields:
             el = elem.find(attribute)
             if hasattr(self, '%s_init_with_xml' % attribute):

wcs/fields/block.py

@@ -136,9 +136,9 @@ class BlockField(WidgetField):
 
     def get_type_label(self):
         try:
-            return _('Field Block (%s)') % self.block.name
+            return _('Block of fields (%s)') % self.block.name
         except KeyError:
-            return _('Field Block (%s, missing)') % self.block_slug
+            return _('Block of fields (%s, missing)') % self.block_slug
 
     def get_dependencies(self):
         yield from super().get_dependencies()

wcs/fields/bool.py

@@ -32,6 +32,7 @@ class BoolField(WidgetField):
     description = _('Check Box (single choice)')
     allow_complex = True
     allow_statistics = True
+    available_for_filter = True
 
     widget_class = CheckboxWidget
     required = False

wcs/fields/date.py

@@ -29,6 +29,7 @@ from .base import WidgetField, register_field_class
 class DateField(WidgetField):
     key = 'date'
     description = _('Date')
+    available_for_filter = True
 
     widget_class = DateWidget
     minimum_date = None

wcs/fields/email.py

@@ -30,6 +30,7 @@ class EmailField(WidgetField):
     key = 'email'
     description = _('Email')
     use_live_server_validation = True
+    available_for_filter = True
 
     widget_class = EmailWidget
 

wcs/fields/item.py

@@ -257,6 +257,7 @@ class ItemField(WidgetField, MapOptionsMixin, ItemFieldMixin, ItemWithImageField
     description = _('List')
     allow_complex = True
     allow_statistics = True
+    available_for_filter = True
 
     items = []
     show_as_radio = None

wcs/fields/items.py

@@ -41,6 +41,7 @@ class ItemsField(WidgetField, ItemFieldMixin, ItemWithImageFieldMixin):
     description = _('Multiple choice list')
     allow_complex = True
     allow_statistics = True
+    available_for_filter = True
 
     items = []
     min_choices = 0

wcs/fields/numeric.py

@@ -33,6 +33,7 @@ class NumericField(WidgetField):
     allow_complex = True
     allow_statistics = False
     use_live_server_validation = True
+    available_for_filter = True
 
     widget_class = NumericWidget
     validation = None

wcs/fields/string.py

@@ -41,6 +41,7 @@ from .base import WidgetField, register_field_class
 class StringField(WidgetField):
     key = 'string'
     description = _('Text (line)')
+    available_for_filter = True
 
     widget_class = WcsExtraStringWidget
     size = None

wcs/fields/text.py

@@ -37,6 +37,7 @@ from .base import WidgetField, register_field_class
 class TextField(WidgetField):
     key = 'text'
     description = _('Long Text')
+    available_for_filter = True
 
     widget_class = TextWidget
     cols = None

wcs/formdata.py

@@ -45,7 +45,7 @@ class NoContentSnapshotAt(RequestError):
     pass
 
 
-def get_dict_with_varnames(fields, data, formdata=None, varnames_only=False):
+def get_dict_with_varnames(fields, data, formdata=None, varnames_only=False, include_files=True):
     new_data = {}
     for field in fields:
         if not hasattr(field, 'get_view_value'):
@@ -80,6 +80,8 @@ def get_dict_with_varnames(fields, data, formdata=None, varnames_only=False):
                     new_data['var_%s_raw' % field.varname] = value
                     new_data['var_%s_url' % field.varname] = None
                     if value and hasattr(value, 'base_filename'):
+                        if include_files is False:
+                            del new_data[f'var_{field.varname}_raw']
                         new_data['var_%s' % field.varname] = value.base_filename
                         if formdata is not None:
                             new_data['var_%s_url' % field.varname] = '%s?f=%s' % (
@@ -990,7 +992,7 @@ class FormData(StorableObject):
                 return StatusFieldValue(self.get_visible_status(user=None))
             if field.key == 'submission_channel':
                 return self.get_submission_channel_label()
-            if field.key == 'submission_agent':
+            if field.key == 'submission-agent':
                 try:
                     agent_user = self.submission_agent_id
                     return get_publisher().user_class.get(agent_user).display_name

wcs/formdef.py

@@ -186,6 +186,7 @@ class FormDef(StorableObject):
     drafts_lifespan = None
     drafts_max_per_user = None
     user_support = None
+    documentation = None
 
     geolocations = None
     history_pane_default_mode = 'expanded'
@@ -219,6 +220,7 @@ class FormDef(StorableObject):
         'drafts_lifespan',
         'drafts_max_per_user',
         'user_support',
+        'documentation',
     ]
     BOOLEAN_ATTRIBUTES = [
         'discussion',
@@ -597,9 +599,8 @@ class FormDef(StorableObject):
             if not field.key == 'block':
                 continue
             try:
-                count += (
-                    len([x for x in field.block.fields or [] if not x.is_no_data_field])
-                    * field.default_items_count
+                count += len([x for x in field.block.fields or [] if not x.is_no_data_field]) * (
+                    field.default_items_count or 1
                 )
             except KeyError:
                 continue
@@ -1767,7 +1768,8 @@ class FormDef(StorableObject):
         from .testdef import TestDef
 
         for testdef in self.xml_testdefs:
-            TestDef.import_from_xml_tree(testdef, self)
+            obj = TestDef.import_from_xml_tree(testdef, self)
+            obj.store()
 
     def get_detailed_email_form(self, formdata, url):
         r = ''
@@ -2380,7 +2382,7 @@ class UpdateDigestAfterJob(AfterJob):
     label = _('Updating digests')
 
     def __init__(self, formdefs):
-        super().__init__(formdefs=[(x.__class__, x.id) for x in formdefs])
+        super().__init__(formdefs=[(x.__class__, x.id) for x in formdefs if x.id])
 
     def execute(self):
         for formdef_class, formdef_id in self.kwargs['formdefs']:

wcs/forms/backoffice.py

@@ -19,6 +19,7 @@ import urllib.parse
 from quixote import get_publisher, get_request, get_session, redirect
 from quixote.html import TemplateIO, htmltext
 
+from wcs.backoffice.filter_fields import FilterField
 from wcs.backoffice.pagination import pagination_links
 from wcs.roles import logged_users_role
 from wcs.sql_criterias import Contains, FtsMatch, Intersects, Not, NotContains, Null, StrictNotEqual
@@ -145,11 +146,11 @@ class FormDefUI:
                 return htmltext('<span title="%s">%s</span>') % (label, misc.ellipsize(label, 20))
 
         for f in fields:
-            if getattr(f, 'fake', False):
+            if isinstance(f, FilterField):
                 field_sort_key = f.id
                 if f.id == 'time':
                     field_sort_key = 'receipt_time'
-                elif f.id in ('user-label', 'submission_agent'):
+                elif f.id in ('user-label', 'submission-agent'):
                     field_sort_key = None
             elif getattr(f, 'is_related_field', False):
                 field_sort_key = None
@@ -390,7 +391,7 @@ class FormDefUI:
                     'user-label': 'cell-user',
                     'status': 'cell-status',
                     'anonymised': 'cell-anonymised',
-                    'submission_agent': 'cell-submission-agent',
+                    'submission-agent': 'cell-submission-agent',
                 }.get(f.key)
                 if css_class:
                     r += htmltext('<td class="%s">' % css_class)

wcs/forms/common.py

@@ -561,6 +561,10 @@ class FormStatusPage(Directory, FormTemplateMixin):
                 'fields': self.display_fields(form_url=form_url),
                 'view': self,
                 'user': user,
+                'section_title': _('Summary'),
+                'section_id': 'sect-dataview',
+                'div_id': 'summary',
+                'enable_compact_dataview': get_publisher().has_site_option('enable-compact-dataview'),
             },
         )
 
@@ -628,18 +632,16 @@ class FormStatusPage(Directory, FormTemplateMixin):
         content = self.display_fields(backoffice_fields, include_unset_required_fields=True)
         if not content:
             return
-        r = TemplateIO(html=True)
-        r += htmltext('<div class="section foldable">')
-        r += htmltext(
-            '<h2><span role="button" aria-expanded="true" '
-            'aria-controls="sect-backoffice-data" '
-            'id="sect-backoffice-data-label">%s</span></h2>'
-        ) % _('Backoffice Data')
-        r += htmltext('<div class="dataview" id="sect-backoffice-data">')
-        r += content
-        r += htmltext('</div>')
-        r += htmltext('</div>')
-        return r.getvalue()
+        return template.render(
+            ['wcs/backoffice/backoffice_fields_section.html'],
+            {
+                'section_title': _('Backoffice Data'),
+                'section_id': 'sect-backoffice-data',
+                'should_fold_summary': False,
+                'fields': content,
+                'enable_compact_dataview': get_publisher().has_site_option('enable-compact-dataview'),
+            },
+        )
 
     def status(self):
         if get_request().get_query() == 'unlock':

wcs/forms/root.py

@@ -15,6 +15,7 @@
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
 import copy
+import hashlib
 import io
 import json
 import time
@@ -294,6 +295,7 @@ class TrackingCodesDirectory(Directory):
 
 
 class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
+    # noqa pylint: disable=too-many-public-methods
     _q_exports = [
         '',
         'tempfile',
@@ -470,6 +472,9 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
             prefilled = False
             locked = False
 
+            if block:
+                field_key = f'{block.id}${block_idx}${field_key}'
+
             if field.get_prefill_configuration():
                 prefill_user = get_request().user
                 if get_request().is_in_backoffice():
@@ -592,8 +597,18 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
                 ),
             )
 
+    def get_honeypot_value(self):
+        # simply an hash of the formdef slug
+        return hashlib.sha1(self.formdef.slug.encode()).hexdigest()
+
     def page(
-        self, page, page_change=True, page_error_messages=None, submit_button=None, transient_formdata=None
+        self,
+        page,
+        arrival=False,
+        page_change=True,
+        page_error_messages=None,
+        submit_button=None,
+        transient_formdata=None,
     ):
         displayed_fields = []
         self.current_page = page
@@ -698,7 +713,7 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
                 # include prefilled data
                 transient_formdata = self.get_transient_formdata(magictoken)
                 transient_formdata.data.update(self.formdef.get_data(form))
-                if self.has_draft_support() and not (req.is_from_application() or req.is_from_bot()):
+                if self.has_draft_support() and (not arrival or computed_fields_on_page):
                     # save to get prefilling data in database
                     self.save_draft(form_data)
                     # and make sure draft formdata id is tracked in session
@@ -762,11 +777,21 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
                 css_class = 'cancel form-discard'
             form.add_submit('cancel', cancel_label, css_class=css_class, attrs={'aria-label': aria_label})
 
-        # add fake field as honey pot
+        # add fake fields as honey pot
         honeypot = form.add(
             StringWidget, 'f00', value='', title=_('leave this field blank to prove your humanity'), size=25
         )
         honeypot.is_hidden = True
+        if 'level2' in get_publisher().get_site_option('honeypots'):
+            honeypot2 = form.add(
+                StringWidget,
+                'f002',
+                value='',
+                title=_('and leave this field as prefilled by javascript'),
+                size=25,
+            )
+            honeypot2.is_hidden = True
+            form.attrs['data-honey-pot-value'] = self.get_honeypot_value()
 
         context = {
             'view': self,
@@ -1279,7 +1304,7 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
             self.feed_current_data(None)
             if not self.pages:
                 return template.error_page(_('This form has no visible page.'))
-            return self.page(self.pages[0])
+            return self.page(self.pages[0], arrival=True)
 
         if form.get_submit() == 'cancel':
             if self.edit_mode:
@@ -1404,11 +1429,15 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
                         page_error_messages.append(get_publisher().translate(error_message))
 
             honeypot_error = False
-            if get_request().form.get('f00'):  # 🍯
+            # 🍯
+            if get_request().form.get('f00') or (
+                'level2' in get_publisher().get_site_option('honeypots')
+                and get_request().form.get('f002') != self.get_honeypot_value()
+            ):
                 honeypot_error = True
                 form.add(HiddenErrorWidget, 'honeypot')
                 form.set_error('honeypot', 'error')
-                page_error_messages.append(_('Honey pot should be left untouched.'))
+                page_error_messages.append(_('Honey pots should be left untouched.'))
 
             # form.get_submit() returns the name of the clicked button, and
             # it will return True if the form has been submitted, but not
@@ -1972,7 +2001,7 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
             testdef = TestDef.create_from_formdata(self.formdef, self.edited_data)
             self.testdef.data = testdef.data
             self.testdef.expected_error = None
-            self.testdef.store()
+            self.testdef.store(comment=_('Change in test data'))
             return redirect(self.formdef.get_admin_url() + 'tests/%s/' % self.testdef.id)
 
         evo = self.edited_data.evolution[-1]

wcs/locale/fr/LC_MESSAGES/django.po

@@ -4,8 +4,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: wcs 0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-04-09 11:26+0200\n"
-"PO-Revision-Date: 2024-04-09 11:26+0200\n"
+"POT-Creation-Date: 2024-04-15 17:07+0200\n"
+"PO-Revision-Date: 2024-04-15 17:06+0200\n"
 "Last-Translator: Thomas Noël <tnoel@entrouvert.com>\n"
 "Language-Team: french\n"
 "Language: fr\n"
@@ -16,16 +16,14 @@ msgstr ""
 
 #: admin/api_access.py admin/blocks.py admin/comment_templates.py
 #: admin/data_sources.py admin/forms.py admin/mail_templates.py admin/tests.py
-#: admin/users.py admin/workflows.py admin/wscalls.py backoffice/management.py
-#: fields/base.py qommon/ident/franceconnect.py
+#: admin/users.py admin/workflows.py admin/wscalls.py
+#: backoffice/filter_fields.py fields/base.py qommon/ident/franceconnect.py
 #: templates/wcs/backoffice/test-result.html wf/profile.py workflow_tests.py
 msgid "Name"
 msgstr "Nom"
 
-#: admin/api_access.py admin/categories.py admin/comment_templates.py
-#: admin/data_sources.py admin/forms.py admin/mail_templates.py
-#: admin/wscalls.py qommon/ident/franceconnect.py
-#: templates/wcs/backoffice/formdef-inspect.html
+#: admin/api_access.py admin/categories.py admin/forms.py
+#: qommon/ident/franceconnect.py templates/wcs/backoffice/formdef-inspect.html
 #: templates/wcs/backoffice/snapshots.html
 msgid "Description"
 msgstr "Description"
@@ -176,10 +174,11 @@ msgid "Applications"
 msgstr "Applications"
 
 #: admin/blocks.py admin/comment_templates.py admin/data_sources.py
-#: admin/mail_templates.py admin/settings.py admin/wscalls.py
+#: admin/mail_templates.py admin/settings.py admin/tests.py admin/wscalls.py
 #: backoffice/i18n.py backoffice/management.py
 #: templates/wcs/backoffice/category.html templates/wcs/backoffice/formdef.html
 #: templates/wcs/backoffice/i18n.html templates/wcs/backoffice/snapshots.html
+#: templates/wcs/backoffice/test-users.html
 #: templates/wcs/backoffice/test_sidebar.html
 #: templates/wcs/backoffice/workflow.html
 msgid "Export"
@@ -238,7 +237,7 @@ msgstr "Enregistrer une sauvegarde"
 msgid "Overwrite with new import"
 msgstr "Écraser avec un nouvel import"
 
-#: admin/blocks.py templates/wcs/backoffice/blocks.html
+#: admin/blocks.py admin/tests.py templates/wcs/backoffice/blocks.html
 #: templates/wcs/backoffice/cards.html templates/wcs/backoffice/category.html
 #: templates/wcs/backoffice/comment-templates.html
 #: templates/wcs/backoffice/data-sources.html
@@ -254,6 +253,7 @@ msgstr "Navigation"
 #: admin/mail_templates.py admin/wscalls.py backoffice/snapshots.py
 #: templates/wcs/backoffice/category.html templates/wcs/backoffice/formdef.html
 #: templates/wcs/backoffice/snapshots.html
+#: templates/wcs/backoffice/test_sidebar.html
 #: templates/wcs/backoffice/workflow.html
 msgid "History"
 msgstr "Historique"
@@ -379,7 +379,8 @@ msgstr "Importer un bloc de champs"
 #: templates/wcs/backoffice/data-sources.html
 #: templates/wcs/backoffice/forms.html templates/wcs/backoffice/i18n.html
 #: templates/wcs/backoffice/mail-templates.html
-#: templates/wcs/backoffice/tests.html templates/wcs/backoffice/workflows.html
+#: templates/wcs/backoffice/test-users.html templates/wcs/backoffice/tests.html
+#: templates/wcs/backoffice/workflows.html
 #: templates/wcs/backoffice/wscalls.html
 msgid "Import"
 msgstr "Importer"
@@ -870,13 +871,17 @@ msgid "Agendas will be updated in the background."
 msgstr ""
 "Les sources basées sur les agendas vont être actualisées en arrière-plan."
 
-#: admin/fields.py admin/settings.py admin/users.py backoffice/management.py
+#: admin/documentable.py
+msgid "Documentation update"
+msgstr "Mise à jour de la documentation"
+
+#: admin/fields.py admin/settings.py admin/users.py backoffice/filter_fields.py
 #: data_sources.py fields/base.py qommon/form.py qommon/ident/password.py
 #: statistics/views.py wf/create_formdata.py workflow_tests.py
 msgid "None"
 msgstr "Aucun"
 
-#: admin/fields.py api_export_import.py
+#: admin/fields.py api_export_import.py blocks.py
 msgid "Block of fields"
 msgstr "Bloc de champs"
 
@@ -1296,7 +1301,7 @@ msgstr ""
 msgid "Appearance"
 msgstr "Apparence"
 
-#: admin/forms.py backoffice/management.py
+#: admin/forms.py backoffice/filter_fields.py backoffice/management.py
 #: templates/wcs/backoffice/formdef-inspect.html
 msgid "Digest"
 msgstr "Résumé"
@@ -1633,8 +1638,8 @@ msgstr "Un changement de workflow est déjà en cours."
 msgid "Invalid target workflow."
 msgstr "Workflow cible invalide."
 
-#: admin/forms.py backoffice/management.py formdef.py
-#: templates/wcs/backoffice/test-results.html workflows.py
+#: admin/forms.py backoffice/filter_fields.py backoffice/management.py
+#: formdef.py templates/wcs/backoffice/test-results.html workflows.py
 msgid "Status"
 msgstr "Statut"
 
@@ -1813,8 +1818,8 @@ msgstr "Gabarit"
 msgid "Text"
 msgstr "Texte"
 
-#: admin/logged_errors.py backoffice/management.py formdata.py
-#: templates/wcs/backoffice/includes/inspect-draft-by-page.html
+#: admin/logged_errors.py backoffice/filter_fields.py backoffice/management.py
+#: formdata.py templates/wcs/backoffice/includes/inspect-draft-by-page.html
 #: wf/create_formdata.py workflows.py
 msgid "Unknown"
 msgstr "Inconnu"
@@ -2113,6 +2118,7 @@ msgstr ""
 "converties vers le type correspondant."
 
 #: admin/settings.py admin/workflows.py backoffice/management.py
+#: templates/wcs/backoffice/includes/documentation.html
 msgid "Save"
 msgstr "Enregistrer"
 
@@ -2401,8 +2407,8 @@ msgstr "URL"
 msgid "Database Name"
 msgstr "Nom de la base de données"
 
-#: admin/settings.py admin/tests.py admin/users.py backoffice/journal.py
-#: backoffice/management.py templates/wcs/backoffice/journal.html
+#: admin/settings.py admin/tests.py admin/users.py backoffice/filter_fields.py
+#: backoffice/journal.py templates/wcs/backoffice/journal.html
 #: templates/wcs/backoffice/snapshots.html users.py
 msgid "User"
 msgstr "Utilisateur"
@@ -2594,6 +2600,29 @@ msgstr "Modifier les données"
 msgid "Mark as failing"
 msgstr "Marquer comme devant échouer"
 
+#: admin/tests.py templates/wcs/backoffice/test_edit_sidebar.html
+msgid "Mark test as failing"
+msgstr "Marquer le test comme devant échouer"
+
+#: admin/tests.py
+msgid "Change submission mode"
+msgstr "Modification du mode de soumission"
+
+#: admin/tests.py
+msgid "This test is readonly."
+msgstr "Ce test est en lecture seule."
+
+#: admin/tests.py templates/wcs/backoffice/test-webservice-responses.html
+#: templates/wcs/backoffice/test_sidebar.html
+msgid "Webservice responses"
+msgstr "Réponses webservice"
+
+#: admin/tests.py templates/wcs/backoffice/snapshots_compare.html
+#: templates/wcs/backoffice/test-result.html
+#: templates/wcs/backoffice/test_sidebar.html
+msgid "Inspect"
+msgstr "Inspecteur"
+
 #: admin/tests.py admin/workflow_tests.py
 #: templates/wcs/backoffice/workflow-tests.html
 msgid "Workflow tests"
@@ -2616,10 +2645,18 @@ msgstr "Suppression du test :"
 msgid "Edit test"
 msgstr "Modifier le test"
 
+#: admin/tests.py
+msgid "Change in options"
+msgstr "Changement dans les options"
+
 #: admin/tests.py
 msgid "Duplicate test"
 msgstr "Dupliquer le test"
 
+#: admin/tests.py
+msgid "Creation (from duplication)"
+msgstr "Création (via duplication)"
+
 #: admin/tests.py templates/wcs/backoffice/formdef.html
 #: templates/wcs/backoffice/snapshots.html templates/wcs/backoffice/tests.html
 msgid "Tests"
@@ -2655,15 +2692,21 @@ msgstr ""
 msgid "New test"
 msgstr "Nouveau test"
 
+#: admin/tests.py
+msgid "Creation (empty)"
+msgstr "Création (vide)"
+
+#: admin/tests.py
+msgid "Creation (from formdata)"
+msgstr "Création (depuis une demande)"
+
 #: admin/tests.py
 msgid "Import Test"
 msgstr "Importer un test"
 
 #: admin/tests.py
-msgid ""
-"You can add a new test or update an existing one by importing a JSON file."
-msgstr ""
-"Vous pouvez créer ou mettre à jour un test en téléchargeant un fichier JSON."
+msgid "Creation (from import)"
+msgstr "Création (via importation)"
 
 #: admin/tests.py
 #, python-format
@@ -2747,19 +2790,33 @@ msgstr "Limiter au données contenues dans le corps de la requête"
 msgid "Edit webservice response"
 msgstr "Modifier la réponse webservice"
 
+#: admin/tests.py
+#, python-format
+msgid "Change webservice response \"%s\""
+msgstr "Modification de la réponse webservice « %s »"
+
 #: admin/tests.py
 msgid "Deleting:"
 msgstr "Suppression :"
 
-#: admin/tests.py templates/wcs/backoffice/test-webservice-responses.html
-#: templates/wcs/backoffice/test_sidebar.html
-msgid "Webservice responses"
-msgstr "Réponses webservice"
+#: admin/tests.py
+#, python-format
+msgid "Duplication of webservice response \"%s\""
+msgstr "Duplication de la réponse webservice « %s »"
 
 #: admin/tests.py
 msgid "New webservice response"
 msgstr "Nouvelle réponse webservice"
 
+#: admin/tests.py
+#, python-format
+msgid "New webservice response \"%s\""
+msgstr "Nouvelle réponse webservice « %s »"
+
+#: admin/tests.py
+msgid "Import webservice responses"
+msgstr "Importer des réponses webservice"
+
 #: admin/tests.py
 msgid "Test user label"
 msgstr "Libellé de l’utilisateur de test"
@@ -2789,6 +2846,22 @@ msgstr "Copier un utilisateur existant"
 msgid "New test user"
 msgstr "Nouvel utilisateur de test"
 
+#: admin/tests.py
+msgid "Import test users"
+msgstr "Importer des utilisateurs de test"
+
+#: admin/tests.py backoffice/data_management.py
+msgid "Invalid JSON file"
+msgstr "Fichier JSON invalide"
+
+#: admin/tests.py
+msgid "Some already existing users were not imported."
+msgstr "Certains utilisateurs existaient déjà et n’ont pas été importés."
+
+#: admin/tests.py
+msgid "Test users have been successfully imported."
+msgstr "Les utilisateurs de test ont été importés correctement."
+
 #: admin/users.py fields/base.py fields/email.py formdata.py formdef.py
 #: forms/root.py qommon/admin/emails.py qommon/ident/franceconnect.py
 #: qommon/ident/idp.py qommon/ident/password.py wf/profile.py wf/sendmail.py
@@ -2920,19 +2993,47 @@ msgstr "Inspecter cette version"
 msgid "Edit action"
 msgstr "Modifier l’action"
 
+#: admin/workflow_tests.py
+#, python-format
+msgid "Change in workflow test action \"%s\""
+msgstr "Modification de l’action de test « %s »"
+
 #: admin/workflow_tests.py
 msgid "Deleting action:"
 msgstr "Suppression de l’action :"
 
+#: admin/workflow_tests.py
+#, python-format
+msgid "Deletion of workflow test action \"%s\""
+msgstr "Suppression de l’action de test « %s »"
+
+#: admin/workflow_tests.py
+#, python-format
+msgid "Duplication of workflow test action \"%s\""
+msgstr "Duplication de l’action de test « %s »"
+
 #: admin/workflow_tests.py workflow_tests.py
 msgid "Backoffice user"
 msgstr "Utilisateur agent"
 
+#: admin/workflow_tests.py
+msgid "Change in workflow test options"
+msgstr "Changement dans les options de test de workflow"
+
+#: admin/workflow_tests.py
+#, python-format
+msgid "New test action \"%s\""
+msgstr "Nouvelle action de test « %s »"
+
+#: admin/workflow_tests.py
+msgid "Change in workflow test actions order"
+msgstr "Changement de l’ordre des actions de test"
+
 #: admin/workflows.py
 msgid "Workflow Name"
 msgstr "Nom du workflow"
 
-#: admin/workflows.py backoffice/management.py fields/base.py
+#: admin/workflows.py backoffice/filter_fields.py fields/base.py
 #: fields/computed.py fields/page.py qommon/form.py
 #: templates/wcs/backoffice/snapshots.html
 #: templates/wcs/backoffice/test-results.html wf/attachment.py wf/choice.py
@@ -3647,6 +3748,27 @@ msgstr "%(name)s - n°%(id)s (%(status)s)"
 msgid "unknown"
 msgstr "inconnu"
 
+#: api.py
+msgid "Payload structure preview"
+msgstr "Prévisualisation de la structure de la requête"
+
+#: api.py
+msgid "Unable to preview payload."
+msgstr "Impossible d’afficher la prévisualisation."
+
+#: api.py
+msgid "Following error occured: "
+msgstr "L’erreur suivante a été eu lieu : "
+
+#: api.py
+msgid ""
+"Warning: conditions are only evaluated when entering the action, you may "
+"need to set a timeout if you want it to be evaluated regularly."
+msgstr ""
+"Attention : les conditions sont évaluées uniquement à l’arrivée dans le "
+"statut, il faut mettre en place un saut sur expiration si une évaluation "
+"régulière est souhaitée."
+
 #: api_export_import.py backoffice/data_management.py backoffice/root.py
 #: data_sources.py templates/wcs/backoffice/data-management.html
 msgid "Cards"
@@ -3844,6 +3966,14 @@ msgstr ""
 "Attention : l’information contenue dans ce champ sera perdue de façon "
 "irréversible."
 
+#: backoffice/cards.py
+msgid ""
+"This field may be used in the card custom identifiers, its removal may "
+"render cards unreachable."
+msgstr ""
+"Ce champ semble utilisé pour l’identifiant personnalisé, sa suppression "
+"pourrait rendre des fiches inaccessibles."
+
 #: backoffice/cards.py
 #, python-format
 msgid "This card model contains %d fields."
@@ -3986,9 +4116,9 @@ msgstr "Importer des fiches depuis un fichier"
 msgid "will be ignored - type %s not supported"
 msgstr "sera ignoré - type %s pas pris en charge"
 
-#: backoffice/data_management.py backoffice/management.py fields/base.py
-#: fields/bool.py formdata.py statistics/views.py
-#: templates/wcs/backoffice/data-source.html workflows.py
+#: backoffice/data_management.py backoffice/filter_fields.py
+#: backoffice/management.py fields/base.py fields/bool.py formdata.py
+#: statistics/views.py templates/wcs/backoffice/data-source.html workflows.py
 msgid "Yes"
 msgstr "Oui"
 
@@ -4061,10 +4191,6 @@ msgstr "(lignes %s)"
 msgid "(line numbers %s and more)"
 msgstr "(lignes %s et plus)"
 
-#: backoffice/data_management.py
-msgid "Invalid JSON file"
-msgstr "Fichier JSON invalide"
-
 #: backoffice/data_management.py
 msgid "This card has already been submitted."
 msgstr "Cette fiche a déjà été enregistrée."
@@ -4221,6 +4347,113 @@ msgstr "Webservice"
 msgid "Python expression detected"
 msgstr "Expression Python détectée"
 
+#: backoffice/filter_fields.py backoffice/management.py fields/base.py
+#: fields/bool.py formdata.py statistics/views.py
+#: templates/wcs/backoffice/data-source.html workflows.py
+msgid "No"
+msgstr "Non"
+
+#: backoffice/filter_fields.py
+#, python-format
+msgid "%s of User"
+msgstr "%s de l’usager"
+
+#: backoffice/filter_fields.py
+msgid "User Label"
+msgstr "Nom de l’usager"
+
+#: backoffice/filter_fields.py
+msgid "Status to display"
+msgstr "Statuts à afficher"
+
+#: backoffice/filter_fields.py backoffice/management.py
+msgid "Waiting for an action"
+msgstr "En attente de votre part"
+
+#: backoffice/filter_fields.py backoffice/management.py statistics/views.py
+msgid "All"
+msgstr "Tous"
+
+#: backoffice/filter_fields.py backoffice/management.py
+msgctxt "formdata"
+msgid "Open"
+msgstr "En attente"
+
+#: backoffice/filter_fields.py backoffice/management.py statistics/views.py
+msgid "Done"
+msgstr "Terminé"
+
+#: backoffice/filter_fields.py
+msgid "Status (for user)"
+msgstr "Statut (visible à l’usager)"
+
+#: backoffice/filter_fields.py
+msgid "Start"
+msgstr "Début"
+
+#: backoffice/filter_fields.py
+msgid "End"
+msgstr "Fin"
+
+#: backoffice/filter_fields.py
+msgid "Start (modification time)"
+msgstr "Début (date de modification)"
+
+#: backoffice/filter_fields.py
+msgid "End (modification time)"
+msgstr "Fin (date de modification)"
+
+#: backoffice/filter_fields.py
+msgid "Current user"
+msgstr "Utilisateur connecté"
+
+#: backoffice/filter_fields.py
+msgid "Current User Function"
+msgstr "Fonction de l’utilisateur connecté"
+
+#: backoffice/filter_fields.py backoffice/submission.py
+msgid "Submission Agent"
+msgstr "Agent à la saisie"
+
+#: backoffice/filter_fields.py
+msgid "Invalid user"
+msgstr "Utilisateur invalide"
+
+#: backoffice/filter_fields.py backoffice/management.py
+#: backoffice/submission.py statistics/views.py
+msgid "Channel"
+msgstr "Canal"
+
+#: backoffice/filter_fields.py
+msgid "Criticality Level"
+msgstr "Niveau de criticité"
+
+#: backoffice/filter_fields.py
+msgctxt "criticality-level"
+msgid "All"
+msgstr "Tous"
+
+#: backoffice/filter_fields.py fields/base.py
+msgid "Number"
+msgstr "Numéro"
+
+#: backoffice/filter_fields.py backoffice/management.py
+#: backoffice/submission.py
+msgid "Created"
+msgstr "Date de création"
+
+#: backoffice/filter_fields.py backoffice/management.py
+msgid "Last Modified"
+msgstr "Dernière modification"
+
+#: backoffice/filter_fields.py
+msgid "Anonymised"
+msgstr "Anonymisé"
+
+#: backoffice/filter_fields.py
+msgid "Distance"
+msgstr "Distance"
+
 #: backoffice/i18n.py backoffice/root.py templates/wcs/backoffice/i18n.html
 msgid "Multilinguism"
 msgstr "Multilinguisme"
@@ -4283,23 +4516,6 @@ msgstr "Formulaire / Modèle de fiche"
 msgid "Form/Card Identifier"
 msgstr "Identifiant de la demande/fiche"
 
-#: backoffice/management.py
-msgid "Waiting for an action"
-msgstr "En attente de votre part"
-
-#: backoffice/management.py
-msgctxt "formdata"
-msgid "Open"
-msgstr "En attente"
-
-#: backoffice/management.py statistics/views.py
-msgid "Done"
-msgstr "Terminé"
-
-#: backoffice/management.py statistics/views.py
-msgid "All"
-msgstr "Tous"
-
 #: backoffice/management.py qommon/admin/menu.py
 #: templates/wcs/backoffice/snapshots.html
 msgid "View"
@@ -4369,10 +4585,6 @@ msgstr "Toutes"
 msgid "Add Category"
 msgstr "Ajouter une catégorie"
 
-#: backoffice/management.py backoffice/submission.py statistics/views.py
-msgid "Channel"
-msgstr "Canal"
-
 #: backoffice/management.py statistics/views.py
 msgctxt "channel"
 msgid "All"
@@ -4441,14 +4653,6 @@ msgstr[1] "%(total)s éléments"
 msgid "Reference"
 msgstr "Référence"
 
-#: backoffice/management.py backoffice/submission.py
-msgid "Created"
-msgstr "Date de création"
-
-#: backoffice/management.py
-msgid "Last Modified"
-msgstr "Dernière modification"
-
 #: backoffice/management.py
 msgctxt "frontoffice"
 msgid "User"
@@ -4502,26 +4706,6 @@ msgstr "défaut"
 msgid "custom value"
 msgstr "valeur personnalisée"
 
-#: backoffice/management.py
-msgid "Start"
-msgstr "Début"
-
-#: backoffice/management.py
-msgid "End"
-msgstr "Fin"
-
-#: backoffice/management.py
-msgid "Current User Function"
-msgstr "Fonction de l’utilisateur connecté"
-
-#: backoffice/management.py backoffice/submission.py
-msgid "Submission Agent"
-msgstr "Agent à la saisie"
-
-#: backoffice/management.py
-msgid "Criticality Level"
-msgstr "Niveau de criticité"
-
 #: backoffice/management.py
 msgid "Current view"
 msgstr "Vue actuelle"
@@ -4546,24 +4730,6 @@ msgstr "Paramétrage des marqueurs"
 msgid "markers"
 msgstr "marqueurs"
 
-#: backoffice/management.py
-msgid "Status to display"
-msgstr "Statuts à afficher"
-
-#: backoffice/management.py
-msgid "Current user"
-msgstr "Utilisateur connecté"
-
-#: backoffice/management.py
-msgctxt "criticality-level"
-msgid "All"
-msgstr "Tous"
-
-#: backoffice/management.py fields/base.py fields/bool.py formdata.py
-#: statistics/views.py templates/wcs/backoffice/data-source.html workflows.py
-msgid "No"
-msgstr "Non"
-
 #: backoffice/management.py
 msgid "When nothing is checked the default settings will apply."
 msgstr "En l’absence de sélection le paramétrage par défaut s’applique."
@@ -4629,34 +4795,6 @@ msgstr ""
 msgid "Delete Custom View"
 msgstr "Supprimer la vue personnalisée"
 
-#: backoffice/management.py fields/base.py
-msgid "Number"
-msgstr "Numéro"
-
-#: backoffice/management.py
-msgid "Submission By"
-msgstr "Saisie par"
-
-#: backoffice/management.py
-msgid "Status (for user)"
-msgstr "Statut (visible à l’usager)"
-
-#: backoffice/management.py
-msgid "Anonymised"
-msgstr "Anonymisé"
-
-#: backoffice/management.py
-msgid "Start (modification time)"
-msgstr "Début (date de modification)"
-
-#: backoffice/management.py
-msgid "End (modification time)"
-msgstr "Fin (date de modification)"
-
-#: backoffice/management.py
-msgid "Distance"
-msgstr "Distance"
-
 #: backoffice/management.py
 #, python-format
 msgid ""
@@ -4950,15 +5088,6 @@ msgstr "supprimé"
 msgid "unset"
 msgstr "non définie"
 
-#: backoffice/management.py
-#, python-format
-msgid "%s of User"
-msgstr "%s de l’usager"
-
-#: backoffice/management.py
-msgid "User Label"
-msgstr "Nom de l’usager"
-
 #: backoffice/management.py
 msgid "Submissions by year"
 msgstr "Transmissions par année"
@@ -5134,7 +5263,7 @@ msgstr "Tous les changements"
 
 #: backoffice/studio.py
 msgctxt "studio"
-msgid "Field blocks"
+msgid "Blocks of fields"
 msgstr "Blocs de champs"
 
 #: backoffice/studio.py
@@ -5197,14 +5326,11 @@ msgid "Pending submissions"
 msgstr "Saisies entamées"
 
 #: blocks.py
-msgid "Field block"
-msgstr "Bloc de champs"
-
-#: blocks.py
-msgid "Field blocks"
+msgid "Blocks of fields"
 msgstr "Blocs de champs"
 
-#: blocks.py data_sources.py formdef.py sql.py workflows.py
+#: blocks.py comment_templates.py data_sources.py formdef.py mail_templates.py
+#: sql.py workflows.py wscalls.py
 msgid "Automatic update"
 msgstr "Mise à jour automatique"
 
@@ -5609,13 +5735,13 @@ msgstr "valeur invalide pour la création du bloc : %s"
 
 #: fields/block.py
 #, python-format
-msgid "Field Block (%s)"
-msgstr "Bloc de champ (%s)"
+msgid "Block of fields (%s)"
+msgstr "Bloc de champs (%s)"
 
 #: fields/block.py
 #, python-format
-msgid "Field Block (%s, missing)"
-msgstr "Bloc de champ (%s, manquant)"
+msgid "Block of fields (%s, missing)"
+msgstr "Bloc de champs (%s, manquant)"
 
 #: fields/block.py
 msgid "Number of items to display by default"
@@ -6605,6 +6731,10 @@ msgstr "Votre dossier a été pris en charge par :"
 msgid "Your case is handled by:"
 msgstr "Votre dossier est pris en charge par :"
 
+#: forms/common.py
+msgid "Summary"
+msgstr "Résumé"
+
 #: forms/common.py wf/backoffice_fields.py
 msgid "Backoffice Data"
 msgstr "Données de traitement"
@@ -6718,6 +6848,10 @@ msgstr "Abandonner la saisie"
 msgid "leave this field blank to prove your humanity"
 msgstr "Laissez ce champ vide pour prouver votre humanité"
 
+#: forms/root.py
+msgid "and leave this field as prefilled by javascript"
+msgstr "et laissez ce champ prérempli automatiquement en l’était"
+
 #: forms/root.py
 #, python-format
 msgid "Value too long for field %(field)s: %(value)s (truncated)"
@@ -6762,8 +6896,8 @@ msgid "Technical error, please try again."
 msgstr "Erreur technique, veuillez réessayer."
 
 #: forms/root.py
-msgid "Honey pot should be left untouched."
-msgstr "Le pot de miel ne doit pas être touché."
+msgid "Honey pots should be left untouched."
+msgstr "Les pots de miel ne doivent pas être touchés."
 
 #: forms/root.py
 msgid "Technical error saving draft, please try again."
@@ -6773,6 +6907,10 @@ msgstr "Erreur technique à l’enregistrement du brouillon, veuillez réessayer
 msgid "Unexpected field error, please check."
 msgstr "Erreur inattendue sur un champ, veuillez vérifier ceux-ci."
 
+#: forms/root.py
+msgid "Change in test data"
+msgstr "Modification des données du formulaire"
+
 #: forms/root.py templates/wcs/formdata_sidebox.html
 msgid "Tracking code"
 msgstr "Code de suivi"
@@ -8941,6 +9079,10 @@ msgstr "fichier"
 msgid "Unauthorized Python Usage"
 msgstr "Utilisation de Python interdite"
 
+#: qommon/misc.py
+msgid "This line contains invisible characters."
+msgstr "Cette ligne contient des caractères invisibles."
+
 #: qommon/myspace.py
 msgid "My Space"
 msgstr "Mon espace"
@@ -9999,6 +10141,10 @@ msgstr "non trouvé"
 msgid "There are no agendas."
 msgstr "Il n’y a pas d’agendas."
 
+#: templates/wcs/backoffice/includes/documentation-editor-link.html
+msgid "Edit documentation"
+msgstr "Modifier la documentation"
+
 #: templates/wcs/backoffice/includes/forms.html
 #, python-format
 msgid "Published from %(date1)s until %(date2)s"
@@ -10265,12 +10411,6 @@ msgstr "Comparer les sauvegardes"
 msgid "XML"
 msgstr "XML"
 
-#: templates/wcs/backoffice/snapshots_compare.html
-#: templates/wcs/backoffice/test-result.html
-#: templates/wcs/backoffice/test_sidebar.html
-msgid "Inspect"
-msgstr "Inspecteur"
-
 #: templates/wcs/backoffice/snapshots_compare.html
 msgid "Compare inspect"
 msgstr "Comparer les inspecteurs"
@@ -10404,6 +10544,10 @@ msgstr "Pas encore de résultats des tests."
 msgid "There are no test users yet."
 msgstr "Il n’y a pas encore d’utilisateurs de tests."
 
+#: templates/wcs/backoffice/test-webservice-responses.html
+msgid "Import from other test"
+msgstr "Importer depuis un autre test"
+
 #: templates/wcs/backoffice/test-webservice-responses.html
 #: wf/assign_carddata.py wf/create_formdata.py wf/redirect_to_url.py
 #: wf/roles.py workflow_tests.py
@@ -10432,10 +10576,6 @@ msgstr "frontoffice,backoffice"
 msgid "Switch to %(mode)s mode."
 msgstr "Passer en mode %(mode)s."
 
-#: templates/wcs/backoffice/test_edit_sidebar.html
-msgid "Mark test as failing"
-msgstr "Marquer le test comme devant échouer"
-
 #: templates/wcs/backoffice/test_edit_sidebar.html
 #, python-format
 msgid ""
@@ -10750,14 +10890,14 @@ msgstr "Étape %(page_no)s sur %(page_count)s"
 msgid "current step"
 msgstr "étape courante"
 
-#: templates/wcs/formdata_summary.html
-msgid "Summary"
-msgstr "Résumé"
-
 #: templates/wcs/formdata_summary.html
 msgid "display form details"
 msgstr "afficher le détail de la demande"
 
+#: templates/wcs/formdata_summary.html
+msgid "Compact table view"
+msgstr "Vue compacte"
+
 #: templates/wcs/includes/drafts-recall.html
 msgid ""
 "You already started to fill this form. You can continue it or submit a new "
@@ -11081,6 +11221,10 @@ msgstr "Vouliez-vous écrire"
 msgid "Apply fix"
 msgstr "Corriger"
 
+#: views.py
+msgid "Preview payload structure"
+msgstr "Prévisualiser la structure de la requête"
+
 #: wf/aggregation_email.py
 msgid "Daily Summary Email"
 msgstr "Courriel récapitulatif quotidien"
@@ -11193,7 +11337,7 @@ msgstr "Enregistrer dans une donnée de traitement"
 msgid "This is used to get attachment in expressions."
 msgstr "Utilisé pour obtenir le fichier attaché dans des expressions."
 
-#: wf/attachment.py
+#: wf/attachment.py wf/wscall.py
 msgid "Include in form history"
 msgstr "Inclure dans l’historique du formulaire"
 
@@ -12246,6 +12390,26 @@ msgstr ""
 msgid "POST data"
 msgstr "Données à envoyer dans le corps de la requête"
 
+#: wf/wscall.py wscalls.py
+msgid ""
+"The / in parameter name allows to generate complex objects. Thus a parameter "
+"named \"element/child\" containing \"value\" will generate the payload "
+"\"element\": {\"child\": \"value\"}. If the subkey, i.e. \"child\", is an "
+"integer it will become a list index and two elements \"element/0\", "
+"\"element/1\" (indexes should  start from zero) containing \"value1\" and "
+"\"value2\" will generate the payload \"element\": [\"value1\", \"value2\"]. "
+"It is possible to combine the two types, for example \"element/0/key1\" to "
+"generate a list of objects."
+msgstr ""
+"Un caractère « / » dans un nom de paramètre permet de générer des structures "
+"complexes. Un paramètre nommé « element/child » avec comme valeur « value » "
+"produira \"element\": {\"child\": \"value\"}. La sous-clé (ici « child ») "
+"peut être un nombre entier, elle désigne alors un index dans une liste et "
+"les deux éléments « element/0 » et « element/1 » avec respectivement comme "
+"valeurs « value1 » et « value2 » produiront \"element\": [\"value1\", "
+"\"value2\"]. Les deux types peuvent se combiner, par exemple « element/0/"
+"key1 », pour générer une liste d’objets."
+
 #: wf/wscall.py
 msgid "Response Type"
 msgstr "Type de réponse"
@@ -13161,6 +13325,26 @@ msgstr "(vers le dernier marqueur)"
 msgid "(conditional)"
 msgstr "(conditionné)"
 
+#: wscalls.py
+#, python-format
+msgid "Webservice call failure because unable to unflatten payload keys (%s)"
+msgstr ""
+"Erreur à l’appel webservice, impossible de créer une requête structurée (%s)"
+
+#: wscalls.py
+msgid "there is a mix between lists and dicts"
+msgstr "il existe un mélange de listes et de dictionnaires"
+
+#: wscalls.py
+#, python-format
+msgid "incomplete array before key \"%s\""
+msgstr "liste incomplète devant la clé « %s »"
+
+#: wscalls.py
+#, python-format
+msgid "incomplete array before %s in %s"
+msgstr "liste incomplèete devant %s dans %s"
+
 #: wscalls.py
 msgid "Webservice call"
 msgstr "Appel de webservice"

wcs/mail_templates.py

@@ -34,7 +34,7 @@ class MailTemplate(XmlStorableObject):
 
     name = None
     slug = None
-    description = None
+    documentation = None
     subject = None
     body = None
     attachments = []
@@ -44,7 +44,8 @@ class MailTemplate(XmlStorableObject):
     XML_NODES = [
         ('name', 'str'),
         ('slug', 'str'),
-        ('description', 'str'),
+        ('description', 'str'),  # legacy
+        ('documentation', 'str'),
         ('subject', 'str'),
         ('body', 'str'),
         ('attachments', 'str_list'),
@@ -54,6 +55,16 @@ class MailTemplate(XmlStorableObject):
         XmlStorableObject.__init__(self)
         self.name = name
 
+    def migrate(self):
+        changed = False
+        if getattr(self, 'description', None):  # 2024-04-07
+            self.documentation = getattr(self, 'description')
+            self.description = None
+            changed = True
+        if changed:
+            self.store(comment=_('Automatic update'), snapshot_store_user=False)
+        return changed
+
     @property
     def category(self):
         return MailTemplateCategory.get(self.category_id, ignore_errors=True)
@@ -69,14 +80,16 @@ class MailTemplate(XmlStorableObject):
         base_url = get_publisher().get_backoffice_url()
         return '%s/workflows/mail-templates/%s/' % (base_url, self.id)
 
-    def store(self, comment=None, application=None, *args, **kwargs):
+    def store(self, comment=None, snapshot_store_user=True, application=None, *args, **kwargs):
         assert not self.is_readonly()
         if self.slug is None:
             # set slug if it's not yet there
             self.slug = self.get_new_slug()
         super().store(*args, **kwargs)
         if get_publisher().snapshot_class:
-            get_publisher().snapshot_class.snap(instance=self, comment=comment, application=application)
+            get_publisher().snapshot_class.snap(
+                instance=self, store_user=snapshot_store_user, comment=comment, application=application
+            )
 
     def get_places_of_use(self):
         from wcs.workflows import Workflow

wcs/publisher.py

@@ -82,6 +82,10 @@ class UnpicklerClass(pickle.Unpickler):
             ('wcs.qommon.storage', 'ElementIntersects'): 'wcs.sql',
             ('wcs.qommon.storage', 'Nothing'): 'wcs.sql',
             ('wcs.qommon.storage', 'Distance'): 'wcs.sql',
+            # filter field classes moved to their own file (2024-04-12)
+            ('wcs.backoffice.management', 'RelatedField'): 'wcs.backoffice.filter_fields',
+            ('wcs.backoffice.management', 'UserRelatedField'): 'wcs.backoffice.filter_fields',
+            ('wcs.backoffice.management', 'UserLabelRelatedField'): 'wcs.backoffice.filter_fields',
             # removed actions
             ('wcs.wf.redirect_to_status', 'RedirectToStatusWorkflowStatusItem'): 'NoLongerAvailableAction',
             ('wcs.workflows', 'RedirectToStatusWorkflowStatusItem'): 'NoLongerAvailableAction',
@@ -587,6 +591,7 @@ class WcsPublisher(QommonPublisher):
         from wcs.data_sources import NamedDataSource
         from wcs.formdef import FormDef
         from wcs.mail_templates import MailTemplate
+        from wcs.testdef import TestDef
         from wcs.workflows import Workflow
         from wcs.wscalls import NamedWsCall
 
@@ -606,6 +611,7 @@ class WcsPublisher(QommonPublisher):
             MailTemplateCategory,
             CommentTemplateCategory,
             DataSourceCategory,
+            TestDef,
         ):
             if klass.xml_root_node == object_type:
                 return klass

wcs/qommon/form.py

@@ -2681,49 +2681,49 @@ class WysiwygTextWidget(TextWidget):
     def get_plain_text_value(self):
         return misc.html2text(self.value)
 
+    def clean_html(self, value):
+        try:
+            from bleach.css_sanitizer import CSSSanitizer
+
+            css_sanitizer = CSSSanitizer(allowed_css_properties=self.ALL_STYLES)
+            kwargs = {
+                'css_sanitizer': css_sanitizer,
+            }
+        except ModuleNotFoundError:
+            # bleach < 5
+            kwargs = {'styles': self.ALL_STYLES}
+
+        cleaner = Cleaner(
+            tags=getattr(self, 'allowed_tags', None) or self.ALL_TAGS,
+            attributes=self.ALL_ATTRS,
+            strip=True,
+            strip_comments=False,
+            filters=[
+                partial(
+                    linkifier.LinkifyFilter,
+                    skip_tags=['pre'],
+                    parse_email=True,
+                    url_re=self.URL_RE,
+                    email_re=self.EMAIL_RE,
+                )
+            ],
+            **kwargs,
+        )
+        value = cleaner.clean(value).removeprefix('<br />').removesuffix('<br />')
+        if not strip_tags(value).strip() and not ('<img' in value or '<hr' in value):
+            value = ''
+        return value
+
     def _parse(self, request):
         TextWidget._parse(self, request, use_validation_function=False)
         if self.value:
-            all_tags = self.ALL_TAGS[:]
+            self.allowed_tags = self.ALL_TAGS[:]
             if get_publisher().get_site_option('ckeditor-allow-style-tag'):
-                all_tags.append('style')
+                self.allowed_tags.append('style')
             if get_publisher().get_site_option('ckeditor-allow-script-tag'):
-                all_tags.append('script')
+                self.allowed_tags.append('script')
 
-            try:
-                from bleach.css_sanitizer import CSSSanitizer
-
-                css_sanitizer = CSSSanitizer(allowed_css_properties=self.ALL_STYLES)
-                kwargs = {
-                    'css_sanitizer': css_sanitizer,
-                }
-            except ModuleNotFoundError:
-                # bleach < 5
-                kwargs = {'styles': self.ALL_STYLES}
-
-            cleaner = Cleaner(
-                tags=all_tags,
-                attributes=self.ALL_ATTRS,
-                strip=True,
-                strip_comments=False,
-                filters=[
-                    partial(
-                        linkifier.LinkifyFilter,
-                        skip_tags=['pre'],
-                        parse_email=True,
-                        url_re=self.URL_RE,
-                        email_re=self.EMAIL_RE,
-                    )
-                ],
-                **kwargs,
-            )
-            self.value = cleaner.clean(self.value)
-            if self.value.startswith('<br />'):
-                self.value = self.value[6:]
-            if self.value.endswith('<br />'):
-                self.value = self.value[:-6]
-            if not strip_tags(self.value).strip() and not ('<img' in self.value or '<hr' in self.value):
-                self.value = ''
+            self.value = self.clean_html(self.value)
 
             # unescape Django template tags
             def unquote_django(matchobj):
@@ -3659,6 +3659,7 @@ class MapWidget(CompositeWidget):
     def init_map_attributes(self, value, **kwargs):
         self.map_attributes = {}
         self.map_attributes.update(get_publisher().get_map_attributes())
+        self.map_attributes['data-map-attribution'] = mark_safe(self.map_attributes['data-map-attribution'])
         self.sync_map_and_address_fields = get_publisher().has_site_option('sync-map-and-address-fields')
         if kwargs.get('initial_zoom') is None:
             kwargs['initial_zoom'] = get_publisher().get_default_zoom_level()

wcs/qommon/http_request.py

@@ -214,16 +214,6 @@ class HTTPRequest(quixote.http_request.HTTPRequest):
         user_agent = self.get_environ('HTTP_USER_AGENT', '').lower()
         return bool('bot' in user_agent or 'crawl' in user_agent)
 
-    def is_from_application(self):
-        # detect calls made from other applications or debug tools
-        # this is not to detect bots (is_from_bot above)
-        user_agent = self.get_environ('HTTP_USER_AGENT', '')
-        return (
-            user_agent.startswith('python-requests')
-            or user_agent.startswith('curl')
-            or user_agent.startswith('Wget')
-        )
-
     def is_from_mobile(self):
         user_agent = self.get_environ('HTTP_USER_AGENT', '')
         try:

wcs/qommon/misc.py

@@ -48,7 +48,7 @@ from django.utils.timezone import is_aware, make_naive
 from PIL import Image
 from quixote import get_publisher, get_request, get_response, redirect
 from quixote.errors import RequestError
-from quixote.html import htmltext
+from quixote.html import htmlescape, htmltext
 from requests.adapters import HTTPAdapter
 
 from . import _, ezt, force_str, get_cfg, get_logger
@@ -1365,3 +1365,26 @@ def parse_decimal(value, do_raise=False, keep_none=False):
         if do_raise:
             raise
         return decimal.Decimal(0)
+
+
+def mark_spaces(s):
+    s = str(htmlescape(str(s)))
+    got_sub = False
+
+    def get_sub(match):
+        nonlocal got_sub
+        got_sub = True
+        return ''.join(
+            f'<span class="escaped-code-point" data-escaped="[U+{ord(x):04X}]"><span class="char">&nbsp;</span></span>'
+            for x in match.group()
+        )
+
+    s = re.sub(r'^(\s+)', get_sub, s)
+    s = re.sub(r'(\s+)$', get_sub, s)
+    s = htmltext(s)
+    if got_sub:
+        s = htmltext(
+            '<button class="toggle-escape-button" role="button" title="%s"></button>%s'
+            % (_('This line contains invisible characters.'), s)
+        )
+    return s

wcs/qommon/publisher.py

@@ -20,6 +20,7 @@ import collections
 import configparser
 import datetime
 import hashlib
+import html
 import inspect
 import io
 import json
@@ -468,6 +469,7 @@ class QommonPublisher(Publisher):
                 'unused-files-behaviour': 'remove',
                 'rich-text-wf-displaymsg': 'auto-ckeditor',
                 'timezone': 'Europe/Paris',
+                'honeypots': '',
             },
         }
         if self.site_options is None:
@@ -840,8 +842,9 @@ class QommonPublisher(Publisher):
             attrs['data-max-bounds-lat2'], attrs['data-max-bounds-lng2'] = self.get_site_option(
                 'map-bounds-bottom-right'
             ).split(';')
-        attrs['data-map-attribution'] = self.get_site_option('map-attribution') or _(
-            'Map data &copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a>'
+        attrs['data-map-attribution'] = html.escape(
+            self.get_site_option('map-attribution')
+            or _('Map data &copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a>')
         )
         attrs['data-tile-urltemplate'] = (
             self.get_site_option('map-tile-urltemplate') or 'https://tiles.entrouvert.org/hdm/{z}/{x}/{y}.png'

wcs/qommon/static/css/dc2/admin.scss

@@ -1018,7 +1018,7 @@ ul#fields-filter li {
 	}
 }
 
-div.object--status-infos, p.snapshot-description {
+div.object--status-infos {
 	font-size: 80%;
 	margin: 0;
 }
@@ -1887,6 +1887,31 @@ ul.form-inspector li {
 	}
 }
 
+.display-codepoints .escaped-code-point[data-escaped] {
+	&::before {
+		content: attr(data-escaped);
+		color: var(--red);
+	}
+	.char {
+		display: none;
+	}
+}
+
+button.toggle-escape-button {
+	border: 0;
+	padding: 0;
+	display: inline-block;
+	width: 2em;
+	margin-left: -2em;
+	&::before {
+		content: "⚠️";
+	}
+	&:active, &:focus, &:hover {
+		border: inherit;
+		background: inherit;
+		outline: inherit;
+	}
+}
 
 div#inspect-test-tools form + br {
 	display: none;
@@ -2559,76 +2584,6 @@ p.snapshots-navigation {
 	text-align: center;
 }
 
-div.diff {
-	margin: 1em 0;
-}
-
-table.diff {
-	background: white;
-	border: 1px solid #f3f3f3;
-	border-collapse: collapse;
-	width: 100%;
-	colgroup, thead, tbody, td {
-		border: 1px solid #f3f3f3;
-	}
-	tbody tr:nth-child(even) {
-		background: #fdfdfd;
-	}
-	th, td {
-		max-width: 30vw;
-		/* it will not actually limit width as the table is set to
-		 * expand to 100% but it will prevent one side getting wider
-		 */
-		overflow: hidden;
-		text-overflow: ellipsis;
-		vertical-align: top;
-	}
-	.diff_header {
-		background: #f7f7f7;
-	}
-	td.diff_header {
-		text-align: right;
-		padding-right: 10px;
-		color: #606060;
-	}
-	.diff_next {
-		display: none;
-	}
-	.diff_add {
-		background-color: #aaffaa;
-	}
-	.diff_chg {
-		background-color: #ffff77;
-	}
-	.diff_sub {
-		background-color: #ffaaaa;
-	}
-}
-
-ins {
-	text-decoration: none;
-	background-color: #d4fcbc;
-}
-
-del {
-	text-decoration: line-through;
-	background-color: #fbb6c2;
-	color: #555;
-}
-
-.inspect-tabs h3 {
-	del, ins {
-		font-weight: bold;
-		background-color: transparent;
-	}
-	del, del a {
-		color: #fbb6c2 !important;
-	}
-	ins, ins a {
-		color: #d4fcbc !important;
-	}
-}
-
 #sidebar .operator-and-value-widget {
 	.title-and-operator {
 		display: flex;
@@ -3186,3 +3141,142 @@ form div.widget[data-widget-name="model_file_mode"] {
 		background: #eee;
 	}
 }
+
+#compact-table-dataview-switch {
+	position: absolute;
+	right: 0;
+	padding: 0 1em;
+	background: white;
+}
+
+div.dataview.compact-dataview {
+	div.field {
+		display: flex;
+		float: none;
+		width: 100%;
+		> .label {
+			width: 20%;
+			text-align: right;
+		}
+		> .value {
+			width: 100%;
+		}
+		&.field-type-block {
+			flex-wrap: wrap;
+			> .label {
+				text-align: left;
+			}
+		}
+	}
+	div.title, div.subtitle {
+		float: none;
+		display: flex;
+	}
+}
+
+.ro-documentation {
+	p:first-child {
+		margin-top: 0;
+	}
+	p:last-child {
+		margin-bottom: 0;
+	}
+}
+
+.bo-block.documentation {
+	position: relative;
+	margin-bottom: 0;
+	&.documentation-inspect-top {
+		margin-bottom: 1em;
+	}
+	button.save {
+		display: none;
+		position: absolute;
+		height: 2.5em;
+		bottom: -1.25em;
+		right: 0.5em;
+	}
+	&.active {
+		button.save {
+			display: block;
+		}
+		border-color: var(--primary-color);
+	}
+}
+
+.inspect-tabs .bo-block.documentation {
+	margin-top: 1em;
+	margin-bottom: 1em;
+}
+
+
+aside .bo-block.documentation {
+	// keep some space for godo toolbar
+	margin-top: 5em;
+}
+
+.focus-editor-link {
+	&::before {
+		font-family: FontAwesome;
+		content: "\f040";  // pencil
+	}
+}
+
+.godo.html-edition,
+.godo.html-edition--show {
+	--padding: 0.5em;
+	outline: none;
+	background: transparent;
+	padding-bottom: 0;
+	p:last-child {
+		margin-bottom: 0;
+	}
+	.godo--editor {
+		min-height: auto;
+		border: none;
+		padding: 0;
+	}
+}
+
+.godo.html-edition--show {
+	.godo--editor > :first-child {
+		padding-top: var(--padding);
+	}
+}
+
+.documentation-save-marks {
+	position: absolute;
+	right: 0.5em;
+	margin-top: -1.5em;
+	span {
+		visibility: hidden;
+		margin-left: -0.5em;
+	}
+}
+
+#appbar.field-edit {
+	margin-bottom: 0;
+}
+
+#appbar > h2 {
+	// always keep a bit of space, for documentation button
+	max-width: calc(100% - 80px);
+}
+
+.payload-preview {
+	&--structure {
+		font-family: monospace;
+		line-height: 150%;
+	}
+	&--template-value {
+		background: #ffc;
+	}
+	&--obj {
+		margin-left: 1em;
+		display: block;
+	}
+	&--item-separator::after {
+		content: '\a'; // force line-break
+		white-space: pre;
+	}
+}

wcs/qommon/static/js/qommon.admin.js

@@ -137,6 +137,7 @@ $(function() {
         $widget.find('select, input').each(function(idx, elem) {
           data[$(elem).attr('name').replace(prefix, '')] = $(elem).val();
         });
+        data['warn-on-datetime'] = ($('#form_timeout').length && ! $('#form_timeout').val());
         $.ajax({
           url: $widget.data('validation-url'),
           data: data,
@@ -255,7 +256,7 @@ $(function() {
       $('#sidebar-toggle').click(function() {
         if ($('#sticky-sidebar').css('display') === 'none') {
           $('#sidebar').animate(
-              {'max-width': '23rem'},
+              {'max-width': '24rem'},
               400,
               function() {
                 $('#sticky-sidebar').show()
@@ -303,6 +304,28 @@ $(function() {
       $window.trigger('scroll');
     }
 
+  $('div.WidgetDict[data-widget-name*="post_data"] input[type="text"]').on('change', function() {
+        var $widget = $(this).parents('div.WidgetDict');
+        var url = '/api/preview-payload-structure?' + $('input', $widget).serialize();
+        var preview_button_id = 'payload-preview-button';
+        var preview_button_selector = 'a#' + preview_button_id;
+        if ($widget.find(preview_button_selector).length) {
+           $widget.find(preview_button_selector).attr('href', url);
+        } else {
+          if ($(this).parents('div.dict-key').length < 0) return;
+          if (!$(this).val().includes('/')) return;
+          var eval_link = document.createElement('a');
+          eval_link.setAttribute('rel', 'popup');
+          eval_link.setAttribute('href', url);
+          eval_link.setAttribute('id', preview_button_id);
+          eval_link.setAttribute('class', 'pk-button');
+          eval_link.setAttribute('data-selector', 'div.payload-preview');
+          eval_link.setAttribute('data-title-selector', 'h2');
+          eval_link.innerHTML = WCS_I18N.preview_payload_structure;
+          $widget.append(eval_link);
+        }
+    }).trigger('change');
+
     $('#inspect-test-tools form').on('submit', function() {
       var data = $(this).serialize();
       $.ajax({url: 'inspect-tool',
@@ -488,4 +511,111 @@ $(function() {
       )
       varname_field_widget.dispatchEvent(new Event('keyup'))
     }
+
+    const compact_table_dataview_switch = document.querySelector('#compact-table-dataview-switch input')
+    if (compact_table_dataview_switch) {
+      compact_table_dataview_switch.addEventListener('change', function(event) {
+        document.querySelectorAll('.dataview').forEach(function(el) {
+          if (compact_table_dataview_switch.checked) {
+            el.classList.add('compact-dataview')
+          } else {
+            el.classList.remove('compact-dataview')
+          }
+        })
+        var pref_message = Object()
+        pref_message['use-compact-table-dataview'] = compact_table_dataview_switch.checked
+        fetch('/api/user/preferences', {
+          method: 'POST',
+          headers: {'Accept': 'application/json', 'Content-Type': 'application/json'},
+          body: JSON.stringify(pref_message)
+        })
+      })
+      if (compact_table_dataview_switch.checked && ! document.querySelector('.dataview.compact-dataview')) {
+       compact_table_dataview_switch.dispatchEvent(new Event('change'))
+      }
+    }
+
+    document.querySelectorAll('.toggle-escape-button').forEach(
+      el => el.addEventListener('click', (event) => {
+        event.preventDefault()
+        el.parentNode.classList.toggle('display-codepoints')
+      })
+    )
+
+    const documentation_block = document.querySelector('.bo-block.documentation')
+    const editor = document.getElementById('documentation-editor')
+    const editor_link = document.querySelector('.focus-editor-link')
+    const title_byline = document.querySelector('.object--status-infos')
+    const documentation_save_button = document.querySelector('.bo-block.documentation button.save')
+    var clear_documentation_save_marks_timeout_id = null
+    if (editor_link) {
+      document.querySelector('#documentation-editor .godo--editor').setAttribute('contenteditable', 'false')
+
+      documentation_save_button.addEventListener('click', (e) => {
+          editor.sourceContent = editor.getHTML()
+          var documentation_message = Object()
+          documentation_message['content'] = editor.sourceContent.innerHTML
+          document.querySelector('.documentation-save-marks .mark-error').style.visibility = 'hidden'
+          document.querySelector('.documentation-save-marks .mark-success').style.visibility = 'hidden'
+          document.querySelector('.documentation-save-marks .mark-sent').style.visibility = 'visible'
+          fetch(`${window.location.pathname}update-documentation`, {
+            method: 'POST',
+            headers: {'Accept': 'application/json', 'Content-Type': 'application/json'},
+            body: JSON.stringify(documentation_message)
+          }).then((response) => {
+            if (! response.ok) {
+              return
+            }
+            return response.json()
+          }).then((json) => {
+            if (json && json.err == 0) {
+              if (json.changed) {
+                document.querySelector('.documentation-save-marks .mark-success').style.visibility = 'visible'
+              } else {
+                document.querySelector('.documentation-save-marks .mark-sent').style.visibility = 'hidden'
+                document.querySelector('.documentation-save-marks .mark-success').style.visibility = 'hidden'
+              }
+              if (json.empty) {
+                document.querySelector('.bo-block.documentation').setAttribute('hidden', 'hidden')
+              }
+            } else {
+              document.querySelector('.documentation-save-marks .mark-error').style.visibility = 'visible'
+            }
+            if (clear_documentation_save_marks_timeout_id) clearTimeout(clear_documentation_save_marks_timeout_id)
+            clear_documentation_save_marks_timeout_id = setTimeout(
+              function() {
+                document.querySelector('.documentation-save-marks .mark-error').style.visibility = 'hidden'
+                document.querySelector('.documentation-save-marks .mark-success').style.visibility = 'hidden'
+                document.querySelector('.documentation-save-marks .mark-sent').style.visibility = 'hidden'
+              }, 5000)
+          })
+        })
+
+      editor_link.addEventListener('click', (e) => {
+        e.preventDefault()
+        if (editor_link.getAttribute('aria-pressed') == 'true') {
+          editor.validEdition()
+          documentation_save_button.dispatchEvent(new Event('click'))
+          documentation_block.classList.remove('active')
+          document.querySelector('#documentation-editor .godo--editor').setAttribute('contenteditable', 'false')
+          editor_link.setAttribute('aria-pressed', false)
+          if (title_byline) title_byline.style.visibility = 'visible'
+        } else {
+          documentation_block.classList.add('active')
+          document.querySelector('.bo-block.documentation').removeAttribute('hidden')
+          if (document.querySelector('aside .bo-block.documentation')) {
+            document.getElementById('sidebar').style.display = 'block'
+            document.getElementById('sidebar').removeAttribute('hidden')
+            if (document.getElementById('sticky-sidebar').style.display == 'none') {
+              document.getElementById('sidebar-toggle').dispatchEvent(new Event('click'))
+            }
+          }
+          if (title_byline) title_byline.style.visibility = 'hidden'
+          editor_link.setAttribute('aria-pressed', true)
+          document.querySelector('#documentation-editor .godo--editor').setAttribute('contenteditable', 'true')
+          editor.showEdition()
+          editor.view.focus()
+        }
+      })
+    }
 });

wcs/qommon/static/js/qommon.forms.js

@@ -350,6 +350,12 @@ $(function() {
      $(this).parent().find('.qommon-map').trigger('qommon:invalidate');
   });
 
+  // fill honeypot field
+  const honeypot = document.querySelector('[name="f002"]')
+  if (honeypot) {
+    honeypot.value = honeypot.form.dataset.honeyPotValue
+  }
+
   var autosave_timeout_id = null;
   var autosave_is_running = false;
   var autosave_button_to_click_on_complete = null;

wcs/qommon/static/js/qommon.geolocation.js

@@ -66,6 +66,17 @@ function init_sync_from_template_address() {
 
   const widget_selector = '.JsonpSingleSelectWidget.template-address';
   const hidden_parts_selector = '.hide-address-parts';
+
+  // mark address field as required if any of its components are required.
+  $(widget_selector + ':not(.widget-required)').each(function(idx, elem) {
+    const $widget = $(elem);
+    if ($widget.nextUntil(widget_selector, 'div[data-geolocation].widget-required:not(.template-address):not(.MapWidget)').length) {
+      $widget.addClass('widget-required')
+      var $required_marker = $('.title span.required').first().clone();
+      $required_marker.appendTo($widget.find('.title label'));
+    }
+  })
+
   $(widget_selector + ' select').on('change', function() {
     var data = $(this).select2('data');
     var widget_name = $(this).parents('div.widget').data('widget-name');

wcs/qommon/static/js/qommon.wysiwyg.js

@@ -3,6 +3,10 @@
 CKEDITOR.env.origIsCompatible = CKEDITOR.env.isCompatible;
 CKEDITOR.env.isCompatible = true;
 
+/* do not turn all contenteditable into ckeditor, as some pages may have both
+ * godo and ckeditor */
+CKEDITOR.disableAutoInline = true;
+
 $(document).ready( function() {
   if (CKEDITOR.env.origIsCompatible == false) {
     /* bail out if ckeditor advertised itself as not supported */

wcs/snapshots.py

@@ -354,6 +354,10 @@ class Snapshot:
             if self.object_type in self._category_types:
                 # set position
                 instance.position = max(i.position or 0 for i in self.get_object_class().select()) + 1
+            elif self.object_type == 'testdef':
+                instance.workflow_tests.id = None
+                for response in instance.get_webservice_responses():
+                    response.id = None
             if hasattr(instance, 'disabled'):
                 instance.disabled = True
         else:

wcs/sql.py

@@ -1787,8 +1787,8 @@ WITH
             --          distance search is done using pg_trgm, https://www.postgresql.org/docs/current/pgtrgm.html
             -- otherwise: token as is and likely no search result later
             SELECT word,
-                coalesce((select plainto_tsquery(perfect.token) FROM wcs_search_tokens AS perfect WHERE perfect.token = plainto_tsquery(word)::text),
-                         tsquery_agg_or(plainto_tsquery(partial.token)),
+                coalesce((select perfect.token::tsquery FROM wcs_search_tokens AS perfect WHERE perfect.token = plainto_tsquery(word)::text),
+                         tsquery_agg_or(partial.token::tsquery),
                          plainto_tsquery(word)) AS tokens
             FROM tokenized
             LEFT JOIN wcs_search_tokens AS partial ON partial.token % plainto_tsquery('simple', word)::text AND word not similar to '%[0-9]{2,}%'
@@ -3876,12 +3876,12 @@ class Snapshot(SqlMixin, wcs.snapshots.Snapshot):
     @classmethod
     def select_object_history(cls, obj, clause=None):
         return cls.select(
-            [Equal('object_type', obj.xml_root_node), Equal('object_id', obj.id)] + (clause or []),
+            [Equal('object_type', obj.xml_root_node), Equal('object_id', str(obj.id))] + (clause or []),
             order_by='-timestamp',
         )
 
     def is_from_object(self, obj):
-        return self.object_type == obj.xml_root_node and self.object_id == obj.id
+        return self.object_type == obj.xml_root_node and self.object_id == str(obj.id)
 
     @classmethod
     def _row2ob(cls, row, **kwargs):
@@ -3912,7 +3912,7 @@ class Snapshot(SqlMixin, wcs.snapshots.Snapshot):
         )
         cur.execute(
             sql_statement,
-            {'object_type': object_type, 'object_id': object_id, 'max_timestamp': max_timestamp},
+            {'object_type': object_type, 'object_id': str(object_id), 'max_timestamp': max_timestamp},
         )
         row = cur.fetchone()
         cur.close()
@@ -5115,8 +5115,7 @@ class SearchableFormDef(SqlMixin):
     def search(cls, obj_type, string):
         _, cur = get_connection_and_cursor()
         cur.execute(
-            ### TEMP REVERT TO PINPOINT THE FTS ISSUE
-            'SELECT object_id FROM searchable_formdefs WHERE fts @@ plainto_tsquery(%s)',
+            'SELECT object_id FROM searchable_formdefs WHERE fts @@ wcs_tsquery(%s)',
             (FtsMatch.get_fts_value(string),),
         )
         ids = [x[0] for x in cur.fetchall()]

wcs/templates/wcs/backoffice.html

@@ -52,7 +52,7 @@
 
 {% block sidebar %}
   {% if sidebar or has_sidebar %}
-    <aside id="sidebar">
+    <aside id="sidebar" {% block sidebar-attrs %}{{ sidebar_attrs|default:"" }}{% endblock %}>
       <button id="sidebar-toggle" aria-label="{% trans "Toggle sidebar" %}">&#8286;</button>
       <div id="sticky-sidebar">
         {% block sidebar-content %}

wcs/templates/wcs/backoffice/backoffice_fields_section.html

@@ -0,0 +1,4 @@
+{% extends "wcs/formdata_summary.html" %}
+
+{% block compact-table-dataview-switch %}{% endblock %}
+{% block field-user %}{% endblock %}

wcs/templates/wcs/backoffice/block-inspect.html

@@ -5,6 +5,10 @@
 
 {% block content %}
 
+  {% if blockdef.documentation %}
+    <div class="bo-block documentation documentation-inspect-top"><div class="ro-documentation">{{ blockdef.documentation|safe }}</div></div>
+  {% endif %}
+
   <div class="pk-tabs inspect-tabs inspect-form-tabs">
     <div class="pk-tabs--tab-list" role="tablist">
       <button role="tab" aria-selected="true" aria-controls="inspect-infos" id="tab-infos" tabindex="0">{% trans "Information" %}</button>

wcs/templates/wcs/backoffice/carddef.html

@@ -17,6 +17,7 @@
 
   {{ publisher.get_request.session.display_message|safe }}
   {% include "wcs/backoffice/includes/sql-fields-integrity.html" %}
+  {% include "wcs/backoffice/includes/documentation.html" with element=formdef object=formdef %}
 
   <div class="bo-block">
     <h3>{% trans "Information" %}</h3>

wcs/templates/wcs/backoffice/comment-template.html

@@ -5,14 +5,13 @@
 
 {% block appbar-actions %}
   {% if not comment_template.is_readonly %}
+    {% include "wcs/backoffice/includes/documentation-editor-link.html" %}
     <a href="edit">{% trans "Edit" %}</a>
   {% endif %}
 {% endblock %}
 
 {% block content %}
-  {% if comment_template.description %}
-    <div class="bo-block">{{ comment_template.description }}</div>
-  {% endif %}
+  {% include "wcs/backoffice/includes/documentation.html" with element=comment_template object=comment_template %}
 
   {% if comment_template.comment %}
     <div class="section">

wcs/templates/wcs/backoffice/data-source.html

@@ -5,14 +5,13 @@
     <h2>{% trans "Data Source" %} - {{ datasource.name }}</h2>
     <span class="actions">
       {% if not datasource.is_readonly %}
+        {% include "wcs/backoffice/includes/documentation-editor-link.html" %}
         <a href="edit">{% trans "Edit" %}</a>
       {% endif %}
     </span>
   </div>
 
-  {% if datasource.description %}
-    <div class="bo-block">{{ datasource.description }}</div>
-  {% endif %}
+  {% include "wcs/backoffice/includes/documentation.html" with element=datasource object=datasource %}
 
   {% if datasource.data_source %}
     <div class="section">

wcs/templates/wcs/backoffice/formdef-inspect.html

@@ -5,6 +5,10 @@
 
 {% block content %}
 
+  {% if formdef.documentation %}
+    <div class="bo-block documentation documentation-inspect-top"><div class="ro-documentation">{{ formdef.documentation|safe }}</div></div>
+  {% endif %}
+
   <div class="pk-tabs inspect-tabs inspect-form-tabs">
     <div class="pk-tabs--tab-list" role="tablist">
       <button role="tab" aria-selected="true" aria-controls="inspect-infos" id="tab-infos" tabindex="0">{% trans "Information" %}</button>

wcs/templates/wcs/backoffice/formdef.html

@@ -16,6 +16,7 @@
       <li><a href="export">{% trans "Export" %}</a></li>
       <li><a href="delete" rel="popup">{% trans "Delete" %}</a></li>
     </ul>
+    {% include "wcs/backoffice/includes/documentation-editor-link.html" %}
     <a rel="popup" href="title">{% trans "change title" %}</a>
   {% endif %}
 {% endblock %}
@@ -35,7 +36,9 @@
   </div>
 
   {{ publisher.get_request.session.display_message|safe }}
+
   {% include "wcs/backoffice/includes/sql-fields-integrity.html" %}
+  {% include "wcs/backoffice/includes/documentation.html" with element=formdef object=formdef %}
 
   <div class="bo-block">
     <h3>{% trans "Information" %}</h3>

wcs/templates/wcs/backoffice/includes/documentation-editor-link.html

@@ -0,0 +1,2 @@
+{% load i18n %}
+<a class="focus-editor-link" title="{% trans "Edit documentation" %}"><span class="sr-only">{% trans "Edit documentation" %}</span></a>

wcs/templates/wcs/backoffice/includes/documentation.html

@@ -0,0 +1,21 @@
+{% load i18n %}
+<div class="bo-block documentation" {% if not element.documentation %}hidden{% endif %}>
+  {% if object.is_readonly %}
+    <div class="ro-documentation">{{ element.documentation|safe }}</div>
+  {% else %}
+    <script type="module" src="/static/xstatic/js/godo.js?{{version_hash}}"></script>
+    <div class="documentation-save-marks">
+      <span class="mark-error">✘</span>
+      <span class="mark-success">✔</span>
+      <span class="mark-sent">✔</span>
+    </div>
+    <div id="div-godo-source" >{{ element.documentation|default:"<p></p>"|safe }}</div>
+    <godo-editor
+      tabindex="0"
+      linked-source="div-godo-source"
+      heading-levels="3,4"
+      id="documentation-editor"
+    ></godo-editor>
+    <button class="save">{% trans "Save" %}</button>
+  {% endif %}
+</div>

wcs/templates/wcs/backoffice/includes/inspect-field.html

@@ -7,5 +7,8 @@
       {% if field.key == 'block' %}</a>{% endif %}
     </span>
   </h4>
+  {% if field.documentation %}
+    <div class="bo-block documentation"><div class="ro-documentation">{{ field.documentation|safe }}</div></div>
+  {% endif %}
   {{ field.get_parameters_view|safe }}
 </div>

wcs/templates/wcs/backoffice/mail-template.html

@@ -5,14 +5,13 @@
 
 {% block appbar-actions %}
   {% if not mail_template.is_readonly %}
+    {% include "wcs/backoffice/includes/documentation-editor-link.html" %}
     <a href="edit">{% trans "Edit" %}</a>
   {% endif %}
 {% endblock %}
 
 {% block content %}
-  {% if mail_template.description %}
-    <div class="bo-block">{{ mail_template.description }}</div>
-  {% endif %}
+  {% include "wcs/backoffice/includes/documentation.html" with element=mail_template object=mail_template %}
 
   {% if mail_template.subject and mail_template.body %}
     <div class="section">

wcs/templates/wcs/backoffice/snapshots_compare.html

@@ -11,7 +11,7 @@
 
 {% block content %}
   <p class="snapshot-description">{{ fromdesc|safe }} ➔ {{ todesc|safe }}</p>
-  <div class="diff">
+  <div class="snapshot-diff">
     {% if mode == 'xml' %}
       {{ diff_serialization|safe }}
     {% else %}

wcs/templates/wcs/backoffice/test-users.html

@@ -6,6 +6,8 @@
 {% block sidebar-content %}
   <h3>{% trans "Actions" %}</h3>
   <a class="button button-paragraph" href="new" rel="popup">{% trans "New" %}</a>
+  <a class="button button-paragraph" href="import" rel="popup">{% trans "Import" %}</a>
+  <a class="button button-paragraph" href="export">{% trans "Export" %}</a>
 {% endblock %}
 
 {% block body %}

wcs/templates/wcs/backoffice/test-webservice-responses.html

@@ -6,6 +6,7 @@
 {% block sidebar-content %}
   <h3>{% trans "Actions" %}</h3>
   <a class="button button-paragraph" href="new" rel="popup">{% trans "New" %}</a>
+  <a class="button button-paragraph" href="import" rel="popup">{% trans "Import from other test" %}</a>
 {% endblock %}
 
 {% block body %}
@@ -20,8 +21,10 @@
                 <i>({% trans "not configured" %})</i>
               {% endif %}
             </a>
-            <a rel="popup" class="delete" href="{{ response.id }}/delete">{% trans "Remove" %}</a>
-            <a class="link-action-icon duplicate" href="{{ response.id }}/duplicate">{% trans "Duplicate" %}</a>
+            {% if not testdef.is_readonly %}
+              <a rel="popup" class="delete" href="{{ response.id }}/delete">{% trans "Remove" %}</a>
+              <a class="link-action-icon duplicate" href="{{ response.id }}/duplicate">{% trans "Duplicate" %}</a>
+            {% endif %}
           </li>
         {% endfor %}
       </ul>

wcs/templates/wcs/backoffice/test_sidebar.html

@@ -10,6 +10,7 @@
 <h3>{% trans "Navigation" %}</h3>
 <ul class="sidebar--buttons">
   <li><a class="button button-paragraph" rel="popup" href="edit">{% trans "Options" %}</a></li>
+  <li><a class="button button-paragraph" href="history/">{% trans "History" %}</a></li>
   <li><a class="button button-paragraph" href="webservice-responses/">{% trans "Webservice responses" %}</a></li>
   <li><a class="button button-paragraph" href="inspect">{% trans "Inspect" %}</a></li>
 </ul>

wcs/templates/wcs/backoffice/workflow-action.html

@@ -0,0 +1,29 @@
+{% extends "wcs/backoffice.html" %}
+{% load i18n %}
+
+{% block appbar-title %}{{ action.description }}{% endblock %}
+
+{% block appbar-actions %}
+  {% if not workflow.is_readonly %}
+    {% include "wcs/backoffice/includes/documentation-editor-link.html" %}
+  {% endif %}
+{% endblock %}
+
+{% block content %}
+  {{ block.super }}
+
+  {{ html_form.render|safe }}
+
+  {% if action.support_substitution_variables %}
+    {{ get_substitution_html_table|safe }}
+  {% endif %}
+
+{% endblock %}
+
+{% block sidebar-attrs %}
+  {% if not action.documentation %}style="display: none"{% endif %}
+{% endblock %}
+
+{% block sidebar-content %}
+  {% include "wcs/backoffice/includes/documentation.html" with element=action object=workflow %}
+{% endblock %}