wscalls: preview unflattened payload (#66916)

(out-of-sync since #28303)

tests/admin_pages/test_block.py

@@ -685,3 +685,37 @@ def test_block_test_results(pub):
     resp.form['varname'] = 'test_3'
     resp = resp.form.submit('submit').follow()
     assert TestResult.count() == 1
+
+
+def test_block_documentation(pub):
+    create_superuser(pub)
+
+    BlockDef.wipe()
+    blockdef = FormDef()
+    blockdef.name = 'block title'
+    blockdef.fields = [fields.BoolField(id='1', label='Bool')]
+    blockdef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get(blockdef.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(blockdef.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    blockdef.refresh_from_storage()
+    assert blockdef.documentation == '<p>doc</p>'
+    resp = app.get(blockdef.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')
+
+    resp = app.get(blockdef.get_admin_url() + 'fields/1/')
+    assert resp.pyquery('.documentation[hidden]')
+    assert resp.pyquery('#sidebar[hidden]')
+    resp = app.post_json(
+        blockdef.get_admin_url() + 'fields/1/update-documentation', {'content': '<p>doc</p>'}
+    )
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    blockdef.refresh_from_storage()
+    assert blockdef.fields[0].documentation == '<p>doc</p>'
+    resp = app.get(blockdef.get_admin_url() + 'fields/1/')
+    assert resp.pyquery('.documentation:not([hidden])')
+    assert resp.pyquery('#sidebar:not([hidden])')

tests/admin_pages/test_card.py

@@ -1180,3 +1180,35 @@ def test_cards_management_options(pub):
     assert_option_display(resp, 'Templates', 'Custom')
     resp = resp.click('Management', href='options/management')
     assert resp.form['history_pane_default_mode'].value == 'expanded'
+
+
+def test_card_documentation(pub):
+    create_superuser(pub)
+
+    CardDef.wipe()
+    carddef = FormDef()
+    carddef.name = 'card title'
+    carddef.fields = [fields.BoolField(id='1', label='Bool')]
+    carddef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get(carddef.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(carddef.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    carddef.refresh_from_storage()
+    assert carddef.documentation == '<p>doc</p>'
+    resp = app.get(carddef.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')
+
+    resp = app.get(carddef.get_admin_url() + 'fields/1/')
+    assert resp.pyquery('.documentation[hidden]')
+    assert resp.pyquery('#sidebar[hidden]')
+    resp = app.post_json(carddef.get_admin_url() + 'fields/1/update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    carddef.refresh_from_storage()
+    assert carddef.fields[0].documentation == '<p>doc</p>'
+    resp = app.get(carddef.get_admin_url() + 'fields/1/')
+    assert resp.pyquery('.documentation:not([hidden])')
+    assert resp.pyquery('#sidebar:not([hidden])')

tests/admin_pages/test_datasource.py

@@ -195,7 +195,6 @@ def test_data_sources_new(pub):
     resp = app.get('/backoffice/settings/data-sources/')
     resp = resp.click('New Data Source')
     resp.forms[0]['name'] = 'a new data source'
-    resp.forms[0]['description'] = 'description of the data source'
 
     resp.forms[0]['data_source$type'] = 'python'
     resp.forms[0]['data_source$value'] = repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}])
@@ -209,13 +208,11 @@ def test_data_sources_new(pub):
     assert 'Edit Data Source' in resp.text
 
     assert NamedDataSource.get(1).name == 'a new data source'
-    assert NamedDataSource.get(1).description == 'description of the data source'
 
     # add a second one
     resp = app.get('/backoffice/settings/data-sources/')
     resp = resp.click('New Data Source')
     resp.forms[0]['name'] = 'an other data source'
-    resp.forms[0]['description'] = 'description of the data source'
     resp.forms[0]['data_source$type'] = 'python'
     resp = resp.forms[0].submit('data_source$apply')
     resp.forms[0]['data_source$value'] = repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}])
@@ -408,7 +405,6 @@ def test_data_sources_category(pub):
     resp = app.get('/backoffice/settings/data-sources/categories/')
     resp = resp.click('New Category')
     resp.form['name'] = 'a new category'
-    resp.form['description'] = 'description of the category'
     resp = resp.form.submit('submit')
     assert DataSourceCategory.count() == 1
     category = DataSourceCategory.select()[0]
@@ -440,7 +436,6 @@ def test_data_sources_category(pub):
     resp = app.get('/backoffice/settings/data-sources/categories/')
     resp = resp.click('New Category')
     resp.form['name'] = 'a second category'
-    resp.form['description'] = 'description of the category'
     resp = resp.form.submit('submit')
     assert DataSourceCategory.count() == 2
     category2 = [x for x in DataSourceCategory.select() if x.id != category.id][0]
@@ -872,13 +867,10 @@ def test_data_sources_edit(pub):
 
     resp = resp.click(href='edit')
     assert resp.forms[0]['name'].value == 'foobar'
-    resp.forms[0]['description'] = 'data source description'
     resp = resp.forms[0].submit('submit')
     assert resp.location == 'http://example.net/backoffice/settings/data-sources/1/'
     resp = resp.follow()
 
-    assert NamedDataSource.get(1).description == 'data source description'
-
     resp = app.get('/backoffice/settings/data-sources/1/edit')
     assert '>Data Attribute</label>' in resp.text
     assert '>Id Attribute</label>' in resp.text
@@ -1146,3 +1138,22 @@ def test_data_sources_agenda_refresh(mock_collect, pub, chrono_url):
     resp = resp.follow()
     assert 'Agendas will be updated in the background.' in resp.text
     assert NamedDataSource.count() == 2
+
+
+def test_datasource_documentation(pub):
+    create_superuser(pub)
+
+    NamedDataSource.wipe()
+    data_source = NamedDataSource(name='foobar')
+    data_source.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get(data_source.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(data_source.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    data_source.refresh_from_storage()
+    assert data_source.documentation == '<p>doc</p>'
+    resp = app.get(data_source.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')

tests/admin_pages/test_form.py

@@ -5179,3 +5179,35 @@ def test_admin_form_sql_integrity_error(pub):
         == 'There are integrity errors in the database column types.'
     )
     assert resp.pyquery('.errornotice li').text() == 'String, expected: character varying, got: boolean.'
+
+
+def test_form_documentation(pub):
+    create_superuser(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [fields.BoolField(id='1', label='Bool')]
+    formdef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get(formdef.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(formdef.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    formdef.refresh_from_storage()
+    assert formdef.documentation == '<p>doc</p>'
+    resp = app.get(formdef.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')
+
+    resp = app.get(formdef.get_admin_url() + 'fields/1/')
+    assert resp.pyquery('.documentation[hidden]')
+    assert resp.pyquery('#sidebar[hidden]')
+    resp = app.post_json(formdef.get_admin_url() + 'fields/1/update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    formdef.refresh_from_storage()
+    assert formdef.fields[0].documentation == '<p>doc</p>'
+    resp = app.get(formdef.get_admin_url() + 'fields/1/')
+    assert resp.pyquery('.documentation:not([hidden])')
+    assert resp.pyquery('#sidebar:not([hidden])')

tests/admin_pages/test_tests.py

@@ -99,6 +99,9 @@ def test_tests_page(pub):
     resp = resp.click('Second test')
     assert 'This test is empty' in resp.text
 
+    resp = resp.click('History')
+    assert 'Creation (empty)' in resp.text
+
     # test run with empty test is allowed
     app.get('/backoffice/forms/1/tests/results/run').follow()
 
@@ -374,6 +377,129 @@ def test_tests_status_page_image_field(pub):
     resp.follow(status=404)
 
 
+def test_tests_history_page(pub):
+    user = create_superuser(pub)
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.fields = [fields.StringField(id='1', varname='test_field', label='Test Field')]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.receipt_time = make_aware(datetime.datetime(2021, 1, 1, 0, 0))
+    formdata.data['1'] = 'This is a test'
+    formdata.user_id = user.id
+    formdata.store()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.name = 'Test 1'
+    testdef.workflow_tests.actions = [
+        workflow_tests.ButtonClick(id='1', button_name='xxx'),
+    ]
+    # create one snapshot
+    testdef.store()
+
+    response = WebserviceResponse()
+    response.testdef_id = testdef.id
+    response.name = 'Fake response'
+    response.url = 'http://example.com/json'
+    response.payload = '{"foo": "bar"}'
+    response.store()
+
+    # create second snapshot
+    testdef.name = 'Test 2'
+    testdef.store()
+
+    # create third snapshot
+    testdef.name = 'Test 3'
+    testdef.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
+    resp = resp.click('History')
+    assert [x.attrib['class'] for x in resp.pyquery.find('.snapshots-list tr')] == [
+        'new-day',
+        'collapsed',
+        'collapsed',
+    ]
+
+    # export snapshot
+    resp_export = resp.click('Export', index=1)
+    assert resp_export.content_type == 'application/x-wcs-snapshot'
+    assert '>Test 2<' in resp_export.text
+
+    # view snapshot
+    view_resp = resp.click('View', index=1)
+    assert '<h2>Test 2</h2>' in view_resp.text
+    assert 'Options' not in resp.text
+    assert 'Delete' not in resp.text
+    assert 'Edit' not in resp.text
+
+    resp = view_resp.click('Workflow tests')
+    assert 'Simulate click on action button' in resp.text
+    assert 'Add' not in resp.text
+    assert 'Delete' not in resp.text
+    assert 'Duplicate' not in resp.text
+
+    resp = resp.click('Edit')
+    assert '>Submit<' not in resp.text
+
+    resp = view_resp.click('Webservice responses')
+    assert 'New' not in resp.text
+    assert 'Remove' not in resp.text
+    assert 'Duplicate' not in resp.text
+
+    resp = resp.click('Fake response')
+    assert 'Edit webservice response' in resp.text
+    assert '>Submit<' not in resp.text
+
+    resp = view_resp.click('Inspect')
+
+    assert 'form_var_test_field' in resp.text
+
+    resp.form['django-condition'] = 'form_var_test_field == "This is a test"'
+    resp = resp.form.submit()
+    assert 'Condition result' in resp.text
+    assert 'result-true' in resp.text
+
+    # restore as new
+    assert TestDef.count() == 1
+    assert WorkflowTests.count() == 1
+    assert WebserviceResponse.count() == 1
+
+    resp = view_resp.click('Restore version')
+    resp.form['action'] = 'as-new'
+    resp = resp.form.submit('submit').follow()
+
+    assert TestDef.count() == 2
+    assert WorkflowTests.count() == 2
+    assert WebserviceResponse.count() == 2
+    assert '<h2>Test 2</h2>' in resp.text
+
+    # restore as current
+    resp = view_resp.click('Restore version')
+    resp.form['action'] = 'overwrite'
+    resp = resp.form.submit('submit').follow()
+
+    assert TestDef.count() == 2
+    assert WorkflowTests.count() == 2
+    assert WebserviceResponse.count() == 2
+    assert '<h2>Test 2</h2>' in resp.text
+
+    # restore first version as current, making sure webservice response is deleted
+    resp = resp.click('History')
+    resp = resp.click('Restore', index=2)
+    resp.form['action'] = 'overwrite'
+    resp = resp.form.submit('submit').follow()
+
+    assert TestDef.count() == 2
+    assert WorkflowTests.count() == 2
+    assert WebserviceResponse.count() == 1
+    assert '<h2>Test 1</h2>' in resp.text
+
+
 def test_tests_edit(pub):
     create_superuser(pub)
     user = pub.user_class(name='test user')
@@ -1457,6 +1583,21 @@ def test_tests_webservice_response(pub):
 
     assert 'must start with http://' in resp.text
 
+    testdef2 = TestDef.create_from_formdata(formdef, formdata)
+    testdef.name = 'Second test'
+    testdef2.store()
+
+    resp = app.get('/backoffice/forms/1/tests/%s/webservice-responses/' % testdef2.id)
+    assert 'Test response' not in resp.text
+
+    resp = resp.click('Import from other test')
+    resp.form['testdef_id'] = testdef.id
+    resp = resp.form.submit().follow()
+
+    assert 'Test response' in resp.text
+    assert len(testdef.get_webservice_responses()) == 1
+    assert len(testdef2.get_webservice_responses()) == 1
+
 
 def test_tests_test_users_management(pub):
     create_superuser(pub)
@@ -1509,9 +1650,11 @@ def test_tests_test_users_management(pub):
 
     real_user = pub.user_class(name='new user')
     real_user.email = 'jane@example.com'
+    real_user.roles = [role.id]
     real_user.form_data = {
         '1': 'Jane',
         '2': 'Doe',
+        '3': 'jane@example.com',
     }
     real_user.store()
 
@@ -1524,6 +1667,7 @@ def test_tests_test_users_management(pub):
     user = pub.user_class.select([NotNull('test_uuid')], order_by='id')[1]
     assert user.name == 'User test 2'
     assert user.email == 'jane@example.com'
+    assert user.roles == [role.id]
     assert user.form_data['1'] == 'Jane'
     assert user.form_data['2'] == 'Doe'
 
@@ -1542,8 +1686,32 @@ def test_tests_test_users_management(pub):
 
     assert 'A test user with this email already exists.' in resp.text
 
+    user_test_2_export_resp = resp.click('Export')
+
     resp = app.get('/backoffice/forms/test-users/')
     resp = resp.click('Remove', href=str(user.id))
     resp = resp.form.submit().follow()
 
     assert 'User test 2' not in resp.text
+
+    resp = resp.click('Import')
+    resp.form['file'] = Upload('export.json', user_test_2_export_resp.body, 'application/json')
+    resp = resp.form.submit().follow()
+
+    assert 'Test users have been successfully imported.' in resp.text
+    assert 'User test 2' in resp.text
+
+    user = pub.user_class.select([NotNull('test_uuid')], order_by='id')[1]
+    assert user.name == 'User test 2'
+    assert user.email == 'jane@example.com'
+    assert user.roles == [role.id]
+    assert user.form_data['1'] == 'Jane'
+    assert user.form_data['2'] == 'Doe'
+
+    global_export_resp = resp.click('Export')
+
+    resp = resp.click('Import')
+    resp.form['file'] = Upload('export.json', global_export_resp.body, 'application/json')
+    resp = resp.form.submit().follow()
+
+    assert 'Some already existing users were not imported.' in resp.text

tests/admin_pages/test_workflow.py

@@ -4429,3 +4429,115 @@ def test_workflow_test_results(pub):
     assert TestResult.count() == 2
     result = TestResult.select(order_by='id')[1]
     assert result.reason == 'Workflow: New status "new status"'
+
+
+def test_workflow_documentation(pub):
+    create_superuser(pub)
+
+    Workflow.wipe()
+    workflow = Workflow(name='Workflow One')
+    status = workflow.add_status(name='New status')
+    status.add_action('anonymise')
+    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
+    workflow.backoffice_fields_formdef.fields = [
+        fields.StringField(id='bo234', label='bo field 1'),
+    ]
+    workflow.variables_formdef = WorkflowVariablesFieldsFormDef(workflow=workflow)
+    workflow.variables_formdef.fields = [
+        fields.StringField(id='va123', label='bo field 1'),
+    ]
+    global_action = workflow.add_global_action('action1')
+    workflow.store()
+
+    app = login(get_app(pub))
+    resp = app.get(workflow.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    assert app.post_json(workflow.get_admin_url() + 'update-documentation', {}).json.get('err') == 1
+    resp = app.post_json(workflow.get_admin_url() + 'update-documentation', {'content': ''})
+    assert resp.json == {'err': 0, 'empty': True, 'changed': False}
+    resp = app.post_json(workflow.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    workflow.refresh_from_storage()
+    assert workflow.documentation == '<p>doc</p>'
+
+    # check forbidden HTML is cleaned
+    resp = app.post_json(
+        workflow.get_admin_url() + 'update-documentation',
+        {'content': '<p>iframe</p><iframe src="xx"></iframe>'},
+    )
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    workflow.refresh_from_storage()
+    assert workflow.documentation == '<p>iframe</p>'
+
+    resp = app.get(workflow.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')
+
+    resp = app.get(workflow.get_admin_url() + 'variables/fields/')
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(
+        workflow.get_admin_url() + 'variables/fields/update-documentation', {'content': '<p>doc</p>'}
+    )
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    workflow.refresh_from_storage()
+    assert workflow.variables_formdef.documentation == '<p>doc</p>'
+    resp = app.get(workflow.get_admin_url() + 'variables/fields/')
+    assert resp.pyquery('.documentation:not([hidden])')
+
+    resp = app.get(workflow.get_admin_url() + 'variables/fields/va123/')
+    assert resp.pyquery('.documentation[hidden]')
+    assert resp.pyquery('#sidebar[hidden]')
+    resp = app.post_json(
+        workflow.get_admin_url() + 'variables/fields/va123/update-documentation', {'content': '<p>doc</p>'}
+    )
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    workflow.refresh_from_storage()
+    assert workflow.variables_formdef.fields[0].documentation == '<p>doc</p>'
+    resp = app.get(workflow.get_admin_url() + 'variables/fields/va123/')
+    assert resp.pyquery('.documentation:not([hidden])')
+    assert resp.pyquery('#sidebar:not([hidden])')
+
+    resp = app.get(workflow.get_admin_url() + 'backoffice-fields/fields/')
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(
+        workflow.get_admin_url() + 'backoffice-fields/fields/update-documentation', {'content': '<p>doc</p>'}
+    )
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    workflow.refresh_from_storage()
+    assert workflow.backoffice_fields_formdef.documentation == '<p>doc</p>'
+    resp = app.get(workflow.get_admin_url() + 'backoffice-fields/fields/')
+    assert resp.pyquery('.documentation:not([hidden])')
+
+    resp = app.get(workflow.get_admin_url() + 'backoffice-fields/fields/bo234/')
+    assert resp.pyquery('.documentation[hidden]')
+    assert resp.pyquery('#sidebar[hidden]')
+    resp = app.post_json(
+        workflow.get_admin_url() + 'backoffice-fields/fields/bo234/update-documentation',
+        {'content': '<p>doc</p>'},
+    )
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    workflow.refresh_from_storage()
+    assert workflow.backoffice_fields_formdef.fields[0].documentation == '<p>doc</p>'
+    resp = app.get(workflow.get_admin_url() + 'backoffice-fields/fields/bo234/')
+    assert resp.pyquery('.documentation:not([hidden])')
+    assert resp.pyquery('#sidebar:not([hidden])')
+
+    resp = app.get(global_action.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(global_action.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    workflow.refresh_from_storage()
+    assert workflow.global_actions[0].documentation == '<p>doc</p>'
+    resp = app.get(global_action.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')
+
+    resp = app.get(status.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(status.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    workflow.refresh_from_storage()
+    assert workflow.possible_status[0].documentation == '<p>doc</p>'
+    resp = app.get(status.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')
+
+    resp = app.get(workflow.get_admin_url() + 'inspect')
+    assert resp.pyquery('.documentation').length == 5

tests/admin_pages/test_workflow_tests.py

@@ -1,5 +1,4 @@
 import datetime
-import os
 
 import pytest
 from django.utils.html import escape
@@ -24,12 +23,6 @@ def pub():
     pub.cfg['identification'] = {'methods': ['password']}
     pub.write_cfg()
 
-    if not pub.site_options.has_section('options'):
-        pub.site_options.add_section('options')
-    pub.site_options.set('options', 'enable-workflow-tests', 'true')
-    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
-        pub.site_options.write(fd)
-
     pub.user_class.wipe()
     FormDef.wipe()
     TestDef.wipe()
@@ -41,34 +34,6 @@ def teardown_module(module):
     clean_temporary_pub()
 
 
-def test_workflow_tests_link_feature_flag(pub):
-    create_superuser(pub)
-
-    formdef = FormDef()
-    formdef.name = 'test title'
-    formdef.store()
-
-    formdata = formdef.data_class()()
-    formdata.just_created()
-    formdata.store()
-
-    testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.name = 'First test'
-    testdef.store()
-
-    app = login(get_app(pub))
-
-    resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
-    assert 'Workflow tests' in resp.text
-
-    pub.site_options.set('options', 'enable-workflow-tests', 'false')
-    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
-        pub.site_options.write(fd)
-
-    resp = app.get('/backoffice/forms/1/tests/%s/' % testdef.id)
-    assert 'Workflow tests' not in resp.text
-
-
 def test_workflow_tests_options(pub):
     create_superuser(pub)
     user = pub.user_class(name='test user')

tests/admin_pages/test_wscall.py

@@ -36,7 +36,6 @@ def teardown_module(module):
 def wscall():
     NamedWsCall.wipe()
     wscall = NamedWsCall(name='xxx')
-    wscall.description = 'description'
     wscall.notify_on_errors = True
     wscall.record_on_errors = True
     wscall.request = {
@@ -68,7 +67,6 @@ def test_wscalls_new(pub, value):
     assert resp.form['notify_on_errors'].value is None
     assert resp.form['record_on_errors'].value == 'yes'
     resp.form['name'] = 'a new webservice call'
-    resp.form['description'] = 'description'
     resp.form['notify_on_errors'] = value
     resp.form['record_on_errors'] = value
     resp.form['request$url'] = 'http://remote.example.net/json'
@@ -111,14 +109,12 @@ def test_wscalls_edit(pub, wscall):
     assert resp.form['notify_on_errors'].value == 'yes'
     assert resp.form['record_on_errors'].value == 'yes'
     assert 'slug' in resp.form.fields
-    resp.form['description'] = 'bla bla bla'
     resp.form['notify_on_errors'] = False
     resp.form['record_on_errors'] = False
     resp = resp.form.submit('submit')
     assert resp.location == 'http://example.net/backoffice/settings/wscalls/xxx/'
     resp = resp.follow()
 
-    assert NamedWsCall.get('xxx').description == 'bla bla bla'
     assert NamedWsCall.get('xxx').notify_on_errors is False
     assert NamedWsCall.get('xxx').record_on_errors is False
 
@@ -235,7 +231,6 @@ def test_wscalls_empty_param_values(pub):
     resp = app.get('/backoffice/settings/wscalls/')
     resp = resp.click('New webservice call')
     resp.form['name'] = 'a new webservice call'
-    resp.form['description'] = 'description'
     resp.form['request$qs_data$element0key'] = 'foo'
     resp.form['request$post_data$element0key'] = 'bar'
     resp = resp.form.submit('submit').follow()
@@ -253,7 +248,6 @@ def test_wscalls_timeout(pub):
     resp = app.get('/backoffice/settings/wscalls/')
     resp = resp.click('New webservice call')
     resp.form['name'] = 'a new webservice call'
-    resp.form['description'] = 'description'
     resp.form['request$timeout'] = 'plop'
     resp = resp.form.submit('submit')
     assert resp.pyquery('[data-widget-name="request$timeout"].widget-with-error')
@@ -300,3 +294,22 @@ def test_wscalls_usage(pub, wscall):
     formdef.store()
     resp = app.get(usage_url)
     assert 'No usage detected.' in resp.text
+
+
+def test_wscall_documentation(pub):
+    create_superuser(pub)
+
+    NamedWsCall.wipe()
+    wscall = NamedWsCall(name='foobar')
+    wscall.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get(wscall.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(wscall.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    wscall.refresh_from_storage()
+    assert wscall.documentation == '<p>doc</p>'
+    resp = app.get(wscall.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')

tests/api/test_all.py

@@ -305,3 +305,53 @@ def test_afterjobs_base_directory(pub):
     get_app(pub).get('/api/jobs/', status=403)
     # base directory is 404
     get_app(pub).get(sign_url('/api/jobs/?orig=coucou', '1234'), status=404)
+
+
+def test_preview_payload_structure(pub, admin_user):
+    get_app(pub).get('/api/preview-payload-structure', status=403)
+    app = login(get_app(pub))
+    resp = app.get('/api/preview-payload-structure')
+
+    assert resp.pyquery('div.payload-preview').length == 1
+    assert '<h2>Payload structure preview</h2>' in resp.text
+    assert resp.pyquery('div.payload-preview').text() == '{}'
+    params = {
+        'request$post_data$added_elements': 1,
+        'request$post_data$element1key': 'user/first_name',
+        'request$post_data$element1value$value_template': 'Foo',
+        'request$post_data$element1value$value_python': '',
+        'request$post_data$element2key': 'user/last_name',
+        'request$post_data$element2value$value_template': 'Bar',
+        'request$post_data$element2value$value_python': '',
+        'request$post_data$element3key': 'user/0',
+    }
+    resp = app.get('/api/preview-payload-structure', params=params)
+    assert resp.pyquery('div.payload-preview div.errornotice').length == 0
+    assert resp.pyquery('div.payload-preview').text() == '{"user": {"first_name": "Foo","last_name": "Bar"}}'
+    params.update(
+        {
+            'request$post_data$element3value$value_template': 'value',
+            'request$post_data$element3value$value_python': '',
+        }
+    )
+    resp = app.get('/api/preview-payload-structure', params=params)
+
+    assert resp.pyquery('div.payload-preview div.errornotice').length == 1
+    assert 'Unable to preview payload' in resp.pyquery('div.payload-preview div.errornotice').text()
+    assert (
+        'Following error occured: there is a mix between lists and dicts'
+        in resp.pyquery('div.payload-preview div.errornotice').text()
+    )
+
+    params = {
+        'post_data$element1key': '0/0',
+        'post_data$element1value$value_template': 'Foo',
+        'post_data$element1value$value_python': '',
+        'post_data$element2key': '0/1',
+        'post_data$element2value$value_template': '{{ form_name }}',
+        'post_data$element2value$value_python': '',
+        'post_data$element3key': '1/0',
+        'post_data$element3value$value_template': '',
+    }
+    resp = app.get('/api/preview-payload-structure', params=params)
+    assert resp.pyquery('div.payload-preview').text() == '[["Foo",{{ form_name }}],[""]]'

tests/backoffice_pages/test_all.py

@@ -1603,8 +1603,18 @@ def test_backoffice_map(pub):
     resp = app.get('/backoffice/management/form-title/')
     assert 'Plot on a Map' in resp.text
     resp = resp.click('Plot on a Map')
-    assert 'data-geojson-url' in resp.text
-    assert 'tiles.entrouvert.org/' in resp.text
+    assert (
+        resp.pyquery('.qommon-map')[0].attrib['data-geojson-url']
+        == 'http://example.net/backoffice/management/form-title/geojson?'
+    )
+    assert (
+        resp.pyquery('.qommon-map')[0].attrib['data-tile-urltemplate']
+        == 'https://tiles.entrouvert.org/hdm/{z}/{x}/{y}.png'
+    )
+    assert (
+        resp.pyquery('.qommon-map')[0].attrib['data-map-attribution']
+        == 'Map data &copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a>'
+    )
 
     if not pub.site_options.has_section('options'):
         pub.site_options.add_section('options')
@@ -1615,7 +1625,10 @@ def test_backoffice_map(pub):
 
     resp = app.get('/backoffice/management/form-title/')
     resp = resp.click('Plot on a Map')
-    assert 'tile.example.net/' in resp.text
+    assert (
+        resp.pyquery('.qommon-map')[0].attrib['data-tile-urltemplate']
+        == 'https://{s}.tile.example.net/{z}/{x}/{y}.png'
+    )
 
     # check query string is kept
     resp = app.get('/backoffice/management/form-title/map?filter=all')

tests/form_pages/test_draft.py

@@ -596,6 +596,92 @@ def test_nothing_to_update_add_row(pub):
     assert 'Technical error saving draft, please try again.' in resp.text
 
 
+def test_draft_with_block_data(pub):
+    FormDef.wipe()
+    BlockDef.wipe()
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [fields.StringField(id='123', required=True, label='Test1')]
+    block.store()
+
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.PageField(id='0', label='1st page'),
+        fields.PageField(id='2', label='2nd page'),
+        fields.BlockField(id='3', label='block', block_slug='foobar', max_items=3),
+    ]
+    formdef.store()
+
+    formdef.data_class().wipe()
+
+    create_user(pub)
+    app = get_app(pub)
+    resp = login(app, username='foo', password='foo').get('/test/')
+
+    resp = resp.form.submit('submit')  # -> page 2
+    resp.form['f3$element0$f123'] = 'foo'
+    resp = resp.form.submit('submit')  # -> confirmation page
+
+    resp = app.get('/test/')
+    resp = resp.click('Continue with draft').follow()
+    assert resp.forms[1]['f3$element0$f123'].value == 'foo'
+    resp = resp.forms[1].submit('previous')  # -> page 2
+    assert resp.forms[1]['f3$element0$f123'].value == 'foo'
+    resp = resp.forms[1].submit('submit')  # -> confirmation page
+    resp = resp.forms[1].submit('submit')  # -> submit
+    assert formdef.data_class().count() == 1
+    formdata = formdef.data_class().select()[0]
+    assert formdata.data == {
+        '3': {'data': [{'123': 'foo'}], 'schema': {'123': 'string'}},
+        '3_display': 'foobar',
+    }
+
+
+def test_draft_with_block_data_tracking_code(pub):
+    FormDef.wipe()
+    BlockDef.wipe()
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [fields.StringField(id='123', required=True, label='Test1')]
+    block.store()
+
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.PageField(id='0', label='1st page'),
+        fields.PageField(id='2', label='2nd page'),
+        fields.BlockField(id='3', label='block', block_slug='foobar', max_items=3),
+    ]
+    formdef.enable_tracking_codes = True
+    formdef.store()
+
+    formdef.data_class().wipe()
+
+    resp = get_app(pub).get('/test/')
+    resp = resp.form.submit('submit')  # -> page 2
+    resp.form['f3$element0$f123'] = 'foo'
+    resp = resp.form.submit('submit')  # -> confirmation page
+    tracking_code = get_displayed_tracking_code(resp)
+
+    resp = get_app(pub).get('/')
+    resp.form['code'] = tracking_code
+    resp = resp.form.submit().follow().follow().follow()
+    assert resp.forms[1]['f3$element0$f123'].value == 'foo'
+    resp = resp.forms[1].submit('previous')  # -> page 2
+    assert resp.forms[1]['f3$element0$f123'].value == 'foo'
+    resp = resp.forms[1].submit('submit')  # -> confirmation page
+    resp = resp.forms[1].submit('submit')  # -> submit
+    assert formdef.data_class().count() == 1
+    formdata = formdef.data_class().select()[0]
+    assert formdata.data == {
+        '3': {'data': [{'123': 'foo'}], 'schema': {'123': 'string'}},
+        '3_display': 'foobar',
+    }
+
+
 def test_draft_store_page_id(pub):
     formdef = create_formdef()
     formdef.enable_tracking_codes = True

tests/form_pages/test_prefill.py

@@ -1717,3 +1717,44 @@ def test_form_page_user_data_source(pub):
         resp = login(get_app(pub), username='foo', password='foo').get('/test/')
         assert resp.form['f1'].value == str(user.id)
         assert 'invalid value selected' not in resp.text
+
+
+def test_form_page_template_block_rows_prefilled_with_form_data(pub):
+    BlockDef.wipe()
+    create_user(pub)
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [
+        fields.StringField(
+            id='123',
+            required=True,
+            label='Test',
+            varname='test',
+            prefill={'type': 'string', 'value': '{{ form_var_foo }}'},
+        ),
+    ]
+    block.store()
+
+    formdef = create_formdef()
+    formdef.data_class().wipe()
+    formdef.fields = [
+        fields.PageField(id='1', label='page1'),
+        fields.StringField(id='2', label='text', varname='foo'),
+        fields.PageField(id='3', label='page2'),
+        fields.BlockField(id='4', label='test', block_slug='foobar', max_items=3),
+    ]
+    formdef.store()
+
+    resp = get_app(pub).get('/test/')
+    resp.form['f2'] = 'foo'
+    resp = resp.form.submit('submit')  # -> second page
+    assert resp.form['f4$element0$f123'].value == 'foo'
+    resp = resp.form.submit('f4$add_element')
+    assert resp.form['f4$element1$f123'].value == 'foo'
+    resp.form['f4$element0$f123'] = 'bar'
+    resp = resp.form.submit('previous')  # -> first page
+    resp.form['f2'] = 'baz'
+    resp = resp.form.submit('submit')  # -> second page
+    assert resp.form['f4$element0$f123'].value == 'bar'  # not changed
+    assert resp.form['f4$element1$f123'].value == 'baz'  # updated

tests/test_comment_template.py

@@ -508,3 +508,40 @@ def test_comment_template_with_category(pub):
     export = ET.tostring(comment_template.export_to_xml(include_id=True))
     comment_template3 = CommentTemplate.import_from_xml_tree(ET.fromstring(export), include_id=True)
     assert comment_template3.category_id is None
+
+
+def test_comment_template_migration(pub):
+    comment_template = CommentTemplate(name='test template')
+    comment_template.description = 'hello'
+    assert comment_template.migrate() is True
+    assert not comment_template.description
+    assert comment_template.documentation == 'hello'
+
+
+def test_comment_template_legacy_xml(pub):
+    comment_template = CommentTemplate(name='test template')
+    comment_template.documentation = 'hello'
+    export = ET.tostring(export_to_indented_xml(comment_template))
+    export = export.replace(b'documentation>', b'description>')
+
+    comment_template2 = CommentTemplate.import_from_xml_tree(ET.fromstring(export))
+    comment_template2.store()
+    comment_template2.refresh_from_storage()
+    assert comment_template2.documentation
+
+
+def test_comment_template_documentation(pub, superuser):
+    CommentTemplate.wipe()
+    comment_template = CommentTemplate(name='foobar')
+    comment_template.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get(comment_template.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(comment_template.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    comment_template.refresh_from_storage()
+    assert comment_template.documentation == '<p>doc</p>'
+    resp = app.get(comment_template.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')

tests/test_mail_templates.py

@@ -542,3 +542,40 @@ def test_mail_template_with_category(pub):
     export = ET.tostring(mail_template.export_to_xml(include_id=True))
     mail_template3 = MailTemplate.import_from_xml_tree(ET.fromstring(export), include_id=True)
     assert mail_template3.category_id is None
+
+
+def test_mail_template_migration(pub):
+    mail_template = MailTemplate(name='test template')
+    mail_template.description = 'hello'
+    assert mail_template.migrate() is True
+    assert not mail_template.description
+    assert mail_template.documentation == 'hello'
+
+
+def test_mail_template_legacy_xml(pub):
+    mail_template = MailTemplate(name='test template')
+    mail_template.documentation = 'hello'
+    export = ET.tostring(export_to_indented_xml(mail_template))
+    export = export.replace(b'documentation>', b'description>')
+
+    mail_template2 = MailTemplate.import_from_xml_tree(ET.fromstring(export))
+    mail_template2.store()
+    mail_template2.refresh_from_storage()
+    assert mail_template2.documentation
+
+
+def test_mail_template_documentation(pub, superuser):
+    MailTemplate.wipe()
+    mail_template = MailTemplate(name='foobar')
+    mail_template.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get(mail_template.get_admin_url())
+    assert resp.pyquery('.documentation[hidden]')
+    resp = app.post_json(mail_template.get_admin_url() + 'update-documentation', {'content': '<p>doc</p>'})
+    assert resp.json == {'err': 0, 'empty': False, 'changed': True}
+    mail_template.refresh_from_storage()
+    assert mail_template.documentation == '<p>doc</p>'
+    resp = app.get(mail_template.get_admin_url())
+    assert resp.pyquery('.documentation:not([hidden])')

tests/test_testdef.py

@@ -78,6 +78,7 @@ def test_testdef_export_to_xml(pub):
     WebserviceResponse.wipe()
 
     testdef2 = TestDef.import_from_xml(io.BytesIO(testdef_xml), formdef)
+    testdef2.store()
     assert testdef2.name == 'test'
     assert testdef2.object_type == 'formdefs'
     assert testdef2.object_id == str(formdef.id)

tests/test_workflow_import.py

@@ -1101,3 +1101,38 @@ def test_import_root_node_error():
         excinfo.value.msg
         == 'Provided XML file is invalid, it starts with a <wrong_root_node> tag instead of <workflow>'
     )
+
+
+def test_documentation_attributes(pub):
+    Workflow.wipe()
+    workflow = Workflow(name='Workflow One')
+    workflow.documentation = 'doc1'
+    status = workflow.add_status(name='New status')
+    status.documentation = 'doc2'
+    action = status.add_action('anonymise')
+    action.documentation = 'doc3'
+    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
+    workflow.backoffice_fields_formdef.documentation = 'doc4'
+    workflow.backoffice_fields_formdef.fields = [
+        StringField(id='bo234', label='bo field 1'),
+    ]
+    workflow.backoffice_fields_formdef.fields[0].documentation = 'doc5'
+    workflow.variables_formdef = WorkflowVariablesFieldsFormDef(workflow=workflow)
+    workflow.variables_formdef.documentation = 'doc6'
+    workflow.variables_formdef.fields = [
+        StringField(id='va123', label='var field 1'),
+    ]
+    workflow.variables_formdef.fields[0].documentation = 'doc7'
+    global_action = workflow.add_global_action('action1')
+    global_action.documentation = 'doc8'
+    workflow.store()
+
+    wf2 = assert_import_export_works(workflow)
+    assert wf2.documentation == 'doc1'
+    assert wf2.possible_status[0].documentation == 'doc2'
+    assert wf2.possible_status[0].items[0].documentation == 'doc3'
+    assert wf2.backoffice_fields_formdef.documentation == 'doc4'
+    assert wf2.backoffice_fields_formdef.fields[0].documentation == 'doc5'
+    assert wf2.variables_formdef.documentation == 'doc6'
+    assert wf2.variables_formdef.fields[0].documentation == 'doc7'
+    assert wf2.global_actions[0].documentation == 'doc8'

tests/test_workflow_tests.py

@@ -1010,6 +1010,79 @@ def test_workflow_tests_backoffice_fields(pub):
     assert str(excinfo.value) == 'Field bo2 not found (expected value "abc").'
 
 
+def test_workflow_tests_dispatch(pub):
+    role = pub.role_class(name='test role')
+    role.store()
+    user = pub.user_class(name='test user')
+    user.test_uuid = '42'
+    user.roles = [role.id]
+    user.store()
+
+    other_role = pub.role_class(name='test role')
+    other_role.store()
+    other_user = pub.user_class(name='test user')
+    other_user.test_uuid = '43'
+    other_user.roles = [other_role.id]
+    other_user.store()
+
+    workflow = Workflow(name='Workflow One')
+    new_status = workflow.add_status(name='New status')
+    middle_status = workflow.add_status(name='Middle status')
+    end_status = workflow.add_status(name='End status')
+
+    dispatch = new_status.add_action('dispatch')
+    dispatch.dispatch_type = 'manual'
+    dispatch.role_key = '_receiver'
+    dispatch.role_id = role.id
+
+    choice = new_status.add_action('choice')
+    choice.label = 'Go to middle status'
+    choice.status = middle_status.id
+    choice.by = ['_receiver']
+
+    dispatch = middle_status.add_action('dispatch')
+    dispatch.dispatch_type = 'manual'
+    dispatch.role_key = '_receiver'
+    dispatch.role_id = other_role.id
+
+    choice = middle_status.add_action('choice')
+    choice.label = 'Go to end status'
+    choice.status = end_status.id
+    choice.by = ['_receiver']
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertStatus(status_name='New status'),
+        workflow_tests.ButtonClick(button_name='Go to middle status', who='other', who_id=user.test_uuid),
+        workflow_tests.AssertStatus(status_name='Middle status'),
+        workflow_tests.ButtonClick(button_name='Go to end status', who='other', who_id=other_user.test_uuid),
+        workflow_tests.AssertStatus(status_name='End status'),
+    ]
+
+    testdef.run(formdef)
+
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertStatus(status_name='New status'),
+        workflow_tests.ButtonClick(
+            button_name='Go to middle status', who='other', who_id=other_user.test_uuid
+        ),
+    ]
+
+    with pytest.raises(WorkflowTestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Button "Go to middle status" is not displayed.'
+
+
 def test_workflow_tests_webservice(pub):
     workflow = Workflow(name='Workflow One')
     new_status = workflow.add_status(name='New status')

tests/test_wscall.py

@@ -289,6 +289,47 @@ def test_webservice_empty_param_values(http_requests, pub):
     assert http_requests.get_last('body') == '{"toto": ""}'
 
 
+def test_webservice_with_unflattened_payload_keys(http_requests, pub):
+    NamedWsCall.wipe()
+
+    wscall = NamedWsCall()
+    wscall.name = 'Hello world'
+    wscall.request = {
+        'method': 'POST',
+        'url': 'http://remote.example.net/json',
+        'post_data': {'foo/0': 'first', 'foo/1': 'second', 'bar': 'example', 'foo/2': ''},
+    }
+    wscall.store()
+
+    wscall.call()
+    assert http_requests.get_last('url') == 'http://remote.example.net/json'
+    assert http_requests.get_last('body') == '{"bar": "example", "foo": ["first", "second", ""]}'
+    assert http_requests.count() == 1
+
+    wscall.request = {
+        'method': 'POST',
+        'url': 'http://remote.example.net/json',
+        'post_data': {'foo/0': 'first', 'foo/1': 'second', 'foo/bar': 'example'},
+    }
+    wscall.store()
+
+    pub.loggederror_class.wipe()
+    http_requests.empty()
+    wscall.call()
+    assert http_requests.count() == 0
+    assert pub.loggederror_class.count() == 0
+
+    wscall.record_on_errors = True
+    wscall.store()
+    pub.loggederror_class.wipe()
+    wscall.call()
+    assert pub.loggederror_class.count() == 1
+    assert (
+        pub.loggederror_class.select()[0].summary
+        == '[WSCALL] Webservice call failure because unable to unflatten payload keys (there is a mix between lists and dicts)'
+    )
+
+
 def test_webservice_timeout(http_requests, pub):
     NamedWsCall.wipe()
 

tests/workflow/test_wscall.py

@@ -574,6 +574,82 @@ def test_webservice_call(http_requests, pub):
     assert payload == {'one': 1, 'str': 'abcd', 'evalme': formdata.get_display_id()}
 
 
+def test_webservice_with_unflattened_payload_keys(http_requests, pub):
+    wf = Workflow(name='wf1')
+    wf.add_status('Status1', 'st1')
+    wf.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.workflow_id = wf.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.store()
+
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.post_data = {
+        'foo/0': 'first',
+        'foo/1': 'second',
+        'foo/2': '{{ form_name }}',
+        'bar': 'example',
+        'form//name': '{{ form_name }}',
+    }
+    pub.substitutions.feed(formdata)
+    item.perform(formdata)
+    assert http_requests.count() == 1
+    assert http_requests.get_last('url') == 'http://remote.example.net/'
+    assert http_requests.get_last('method') == 'POST'
+    payload = json.loads(http_requests.get_last('body'))
+    assert payload == {'foo': ['first', 'second', 'baz'], 'bar': 'example', 'form/name': 'baz'}
+
+    http_requests.empty()
+    pub.loggederror_class.wipe()
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.record_on_errors = True
+    item.post_data = {'foo/1': 'first', 'foo/2': 'second'}
+
+    item.perform(formdata)
+    assert http_requests.count() == 0
+    assert pub.loggederror_class.count() == 1
+    assert (
+        pub.loggederror_class.select()[0].summary
+        == '[WSCALL] Webservice call failure because unable to unflatten payload keys (incomplete array before key "foo/1")'
+    )
+
+    http_requests.empty()
+    pub.loggederror_class.wipe()
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.record_on_errors = True
+    item.post_data = {'0/foo': 'value', '1/bar': 'value', 'name': '{{ form_name }}'}
+
+    item.perform(formdata)
+    assert (
+        pub.loggederror_class.select()[0].summary
+        == '[WSCALL] Webservice call failure because unable to unflatten payload keys (there is a mix between lists and dicts)'
+    )
+
+    http_requests.empty()
+    pub.loggederror_class.wipe()
+    item = WebserviceCallStatusItem()
+    item.url = 'http://remote.example.net'
+    item.record_on_errors = True
+    item.post_data = {'0/foo': 'value', '1/bar': 'value'}
+
+    pub.substitutions.feed(formdata)
+    item.perform(formdata)
+
+    assert http_requests.count() == 1
+    payload = json.loads(http_requests.get_last('body'))
+    assert payload == [{'foo': 'value'}, {'bar': 'value'}]
+
+
 def test_webservice_waitpoint(pub):
     item = WebserviceCallStatusItem()
     assert item.waitpoint

wcs/admin/blocks.py

@@ -55,6 +55,7 @@ class BlockDirectory(FieldsDirectory):
         'duplicate',
         ('history', 'snapshots_dir'),
         'overwrite',
+        ('update-documentation', 'update_documentation'),
     ]
     field_def_page_class = BlockFieldDefPage
     blacklisted_types = ['page', 'table', 'table-select', 'tablerows', 'ranked-items', 'blocks', 'computed']
@@ -113,11 +114,13 @@ class BlockDirectory(FieldsDirectory):
         r += htmltext('<li><a href="export">%s</a></li>') % _('Export')
         r += htmltext('<li><a href="delete" rel="popup">%s</a></li>') % _('Delete')
         r += htmltext('</ul>')
+        r += self.get_documentable_button()
         r += htmltext('<a href="settings" rel="popup" role="button">%s</a>') % _('Settings')
         r += htmltext('</span>')
         r += htmltext('</div>')
         r += utils.last_modification_block(obj=self.objectdef)
         r += get_session().display_message()
+        r += self.get_documentable_zone()
 
         if not self.objectdef.fields:
             r += htmltext('<div class="infonotice">%s</div>') % _('There are not yet any fields defined.')

wcs/admin/comment_templates.py

@@ -20,6 +20,7 @@ from quixote.html import TemplateIO, htmltext
 
 from wcs.admin import utils
 from wcs.admin.categories import CommentTemplateCategoriesDirectory, get_categories
+from wcs.admin.documentable import DocumentableMixin
 from wcs.backoffice.applications import ApplicationsDirectory
 from wcs.backoffice.snapshots import SnapshotsDirectory
 from wcs.categories import CommentTemplateCategory
@@ -169,7 +170,7 @@ class CommentTemplatesDirectory(Directory):
         return redirect('%s/' % comment_template.id)
 
 
-class CommentTemplatePage(Directory):
+class CommentTemplatePage(Directory, DocumentableMixin):
     _q_exports = [
         '',
         'edit',
@@ -177,6 +178,7 @@ class CommentTemplatePage(Directory):
         'duplicate',
         'export',
         ('history', 'snapshots_dir'),
+        ('update-documentation', 'update_documentation'),
     ]
     do_not_call_in_templates = True
 
@@ -187,6 +189,8 @@ class CommentTemplatePage(Directory):
             raise errors.TraversalError()
         get_response().breadcrumb.append((component + '/', self.comment_template.name))
         self.snapshots_dir = SnapshotsDirectory(self.comment_template)
+        self.documented_object = self.comment_template
+        self.documented_element = self.comment_template
 
     def get_sidebar(self):
         r = TemplateIO(html=True)
@@ -247,14 +251,6 @@ class CommentTemplatePage(Directory):
                 value=self.comment_template.category_id,
             )
 
-        form.add(
-            TextWidget,
-            'description',
-            title=_('Description'),
-            cols=80,
-            rows=3,
-            value=self.comment_template.description,
-        )
         form.add(
             TextWidget,
             'comment',
@@ -307,7 +303,6 @@ class CommentTemplatePage(Directory):
         self.comment_template.name = name
         if form.get_widget('category_id'):
             self.comment_template.category_id = form.get_widget('category_id').parse()
-        self.comment_template.description = form.get_widget('description').parse()
         self.comment_template.comment = form.get_widget('comment').parse()
         self.comment_template.attachments = form.get_widget('attachments').parse()
         if slug_widget:

wcs/admin/data_sources.py

@@ -20,6 +20,7 @@ from quixote.html import TemplateIO, htmltext
 
 from wcs.admin import utils
 from wcs.admin.categories import DataSourceCategoriesDirectory, get_categories
+from wcs.admin.documentable import DocumentableMixin
 from wcs.backoffice.applications import ApplicationsDirectory
 from wcs.backoffice.snapshots import SnapshotsDirectory
 from wcs.carddef import CardDef
@@ -45,7 +46,6 @@ from wcs.qommon.form import (
     SingleSelectWidget,
     SlugWidget,
     StringWidget,
-    TextWidget,
     WidgetDict,
     WidgetList,
     get_response,
@@ -73,14 +73,6 @@ class NamedDataSourceUI:
                 options=category_options,
                 value=self.datasource.category_id,
             )
-        form.add(
-            TextWidget,
-            'description',
-            title=_('Description'),
-            cols=40,
-            rows=5,
-            value=self.datasource.description,
-        )
         if not self.datasource or (
             self.datasource.type != 'wcs:users' and self.datasource.external != 'agenda_manual'
         ):
@@ -297,7 +289,7 @@ class NamedDataSourceUI:
         self.datasource.store()
 
 
-class NamedDataSourcePage(Directory):
+class NamedDataSourcePage(Directory, DocumentableMixin):
     _q_exports = [
         '',
         'edit',
@@ -306,6 +298,7 @@ class NamedDataSourcePage(Directory):
         'duplicate',
         ('history', 'snapshots_dir'),
         ('preview-block', 'preview_block'),
+        ('update-documentation', 'update_documentation'),
     ]
     do_not_call_in_templates = True
 
@@ -319,6 +312,8 @@ class NamedDataSourcePage(Directory):
         self.datasource_ui = NamedDataSourceUI(self.datasource)
         get_response().breadcrumb.append((component + '/', self.datasource.name))
         self.snapshots_dir = SnapshotsDirectory(self.datasource)
+        self.documented_object = self.datasource
+        self.documented_element = self.datasource
 
     def get_sidebar(self):
         r = TemplateIO(html=True)

wcs/admin/documentable.py

@@ -0,0 +1,62 @@
+# w.c.s. - web application for online forms
+# Copyright (C) 2005-2024  Entr'ouvert
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+
+import json
+
+from quixote import get_request, get_response
+from quixote.html import htmltext
+
+from wcs.qommon import _, template
+from wcs.qommon.form import RichTextWidget
+
+
+class DocumentableMixin:
+    def get_documentable_button(self):
+        return htmltext(template.render('wcs/backoffice/includes/documentation-editor-link.html', {}))
+
+    def get_documentable_zone(self):
+        return htmltext('<span class="actions">%s</span>') % template.render(
+            'wcs/backoffice/includes/documentation.html',
+            {'element': self.documented_element, 'object': self.documented_object},
+        )
+
+    def update_documentation(self):
+        get_request().ignore_session = True
+        get_response().set_content_type('application/json')
+        try:
+            content = get_request().json['content']
+        except (KeyError, TypeError):
+            return json.dumps({'err': 1})
+        content = RichTextWidget('').clean_html(content) or None
+        changed = False
+        if content != self.documented_element.documentation:
+            changed = True
+            self.documented_element.documentation = content
+            self.documented_object.store(_('Documentation update'))
+        return json.dumps(
+            {'err': 0, 'empty': not bool(self.documented_element.documentation), 'changed': changed}
+        )
+
+
+class DocumentableFieldMixin:
+    def documentation_part(self):
+        if not self.field.documentation:
+            get_response().filter['sidebar_attrs'] = 'hidden'
+        return template.render(
+            'wcs/backoffice/includes/documentation.html',
+            {'element': self.documented_element, 'object': self.documented_object},
+        )

wcs/admin/fields.py

@@ -26,18 +26,21 @@ from wcs.admin import utils
 from wcs.carddef import CardDef
 from wcs.fields import BlockField, get_field_options
 from wcs.formdef import FormDef, UpdateStatisticsDataAfterJob
-from wcs.qommon import _, errors, get_cfg, misc
+from wcs.qommon import _, errors, get_cfg, misc, template
 from wcs.qommon.admin.menu import command_icon
 from wcs.qommon.form import CheckboxWidget, Form, HtmlWidget, OptGroup, SingleSelectWidget, StringWidget
 from wcs.qommon.substitution import CompatibilityNamesDict
 
+from .documentable import DocumentableFieldMixin, DocumentableMixin
 
-class FieldDefPage(Directory):
-    _q_exports = ['', 'delete', 'duplicate']
+
+class FieldDefPage(Directory, DocumentableMixin, DocumentableFieldMixin):
+    _q_exports = ['', 'delete', 'duplicate', ('update-documentation', 'update_documentation')]
 
     large = False
     page_id = None
     blacklisted_attributes = []
+    is_documentable = True
 
     def __init__(self, objectdef, field_id):
         self.objectdef = objectdef
@@ -47,6 +50,8 @@ class FieldDefPage(Directory):
             raise errors.TraversalError()
         if not self.field.label:
             self.field.label = str(_('None'))
+        self.documented_object = objectdef
+        self.documented_element = self.field
         label = misc.ellipsize(self.field.unhtmled_label, 40)
         last_breadcrumb_url_part, last_breadcrumb_label = get_response().breadcrumb[-1]
         get_response().breadcrumb = get_response().breadcrumb[:-1]
@@ -67,7 +72,11 @@ class FieldDefPage(Directory):
         return form
 
     def get_sidebar(self):
-        return None
+        if not self.is_documentable:
+            return None
+        r = TemplateIO(html=True)
+        r += self.documentation_part()
+        return r.getvalue()
 
     def _q_index(self):
         form = self.form()
@@ -94,9 +103,15 @@ class FieldDefPage(Directory):
             get_response().set_title(self.objectdef.name)
             get_response().filter['sidebar'] = self.get_sidebar()  # noqa pylint: disable=assignment-from-none
             r = TemplateIO(html=True)
+            r += htmltext('<div id="appbar" class="field-edit">')
             r += htmltext('<h2 class="field-edit--title">%s</h2>') % misc.ellipsize(
                 self.field.unhtmled_label, 80
             )
+            if self.is_documentable:
+                r += htmltext('<span class="actions">%s</span>') % template.render(
+                    'wcs/backoffice/includes/documentation-editor-link.html', {}
+                )
+            r += htmltext('</div>')
             if isinstance(self.field, BlockField):
                 try:
                     block_field = self.field.block
@@ -329,8 +344,15 @@ class FieldsPagesDirectory(Directory):
         return directory
 
 
-class FieldsDirectory(Directory):
-    _q_exports = ['', 'update_order', 'move_page_fields', 'new', 'pages']
+class FieldsDirectory(Directory, DocumentableMixin):
+    _q_exports = [
+        '',
+        'update_order',
+        'move_page_fields',
+        'new',
+        'pages',
+        ('update-documentation', 'update_documentation'),
+    ]
     field_def_page_class = FieldDefPage
     blacklisted_types = []
     page_id = None
@@ -345,6 +367,8 @@ class FieldsDirectory(Directory):
 
     def __init__(self, objectdef):
         self.objectdef = objectdef
+        self.documented_object = self.objectdef
+        self.documented_element = self.objectdef
         self.pages = FieldsPagesDirectory(self)
 
     def _q_traverse(self, path):

wcs/admin/forms.py

@@ -69,6 +69,7 @@ from . import utils
 from .blocks import BlocksDirectory
 from .categories import CategoriesDirectory, get_categories
 from .data_sources import NamedDataSourcesDirectory
+from .documentable import DocumentableMixin
 from .fields import FieldDefPage, FieldsDirectory
 from .logged_errors import LoggedErrorsDirectory
 
@@ -621,7 +622,7 @@ class WorkflowRoleDirectory(Directory):
             return redirect('..')
 
 
-class FormDefPage(Directory, TempfileDirectoryMixin):
+class FormDefPage(Directory, TempfileDirectoryMixin, DocumentableMixin):
     do_not_call_in_templates = True
     _q_exports = [
         '',
@@ -648,6 +649,7 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
         ('backoffice-submission-roles', 'backoffice_submission_roles'),
         ('logged-errors', 'logged_errors_dir'),
         ('history', 'snapshots_dir'),
+        ('update-documentation', 'update_documentation'),
     ]
 
     formdef_class = FormDef
@@ -691,6 +693,8 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
             parent_dir=self, formdef_class=self.formdef_class, formdef_id=self.formdef.id
         )
         self.snapshots_dir = SnapshotsDirectory(self.formdef)
+        self.documented_object = self.formdef
+        self.documented_element = self.formdef
 
     def add_option_line(self, link, label, current_value, popup=True):
         return htmltext(

wcs/admin/mail_templates.py

@@ -20,6 +20,7 @@ from quixote.html import TemplateIO, htmltext
 
 from wcs.admin import utils
 from wcs.admin.categories import MailTemplateCategoriesDirectory, get_categories
+from wcs.admin.documentable import DocumentableMixin
 from wcs.backoffice.applications import ApplicationsDirectory
 from wcs.backoffice.snapshots import SnapshotsDirectory
 from wcs.categories import MailTemplateCategory
@@ -167,7 +168,7 @@ class MailTemplatesDirectory(Directory):
         return redirect('%s/' % mail_template.id)
 
 
-class MailTemplatePage(Directory):
+class MailTemplatePage(Directory, DocumentableMixin):
     _q_exports = [
         '',
         'edit',
@@ -175,6 +176,7 @@ class MailTemplatePage(Directory):
         'duplicate',
         'export',
         ('history', 'snapshots_dir'),
+        ('update-documentation', 'update_documentation'),
     ]
     do_not_call_in_templates = True
 
@@ -185,6 +187,8 @@ class MailTemplatePage(Directory):
             raise errors.TraversalError()
         get_response().breadcrumb.append((component + '/', self.mail_template.name))
         self.snapshots_dir = SnapshotsDirectory(self.mail_template)
+        self.documented_object = self.mail_template
+        self.documented_element = self.mail_template
 
     def get_sidebar(self):
         r = TemplateIO(html=True)
@@ -242,15 +246,6 @@ class MailTemplatePage(Directory):
                 options=category_options,
                 value=self.mail_template.category_id,
             )
-
-        form.add(
-            TextWidget,
-            'description',
-            title=_('Description'),
-            cols=80,
-            rows=3,
-            value=self.mail_template.description,
-        )
         form.add(
             StringWidget,
             'subject',
@@ -312,7 +307,6 @@ class MailTemplatePage(Directory):
         self.mail_template.name = name
         if form.get_widget('category_id'):
             self.mail_template.category_id = form.get_widget('category_id').parse()
-        self.mail_template.description = form.get_widget('description').parse()
         self.mail_template.subject = form.get_widget('subject').parse()
         self.mail_template.body = form.get_widget('body').parse()
         self.mail_template.attachments = form.get_widget('attachments').parse()

wcs/admin/settings.py

@@ -131,6 +131,7 @@ authentication is unavailable. Lasso must be installed to use it.'
 
 class UserFieldDefPage(FieldDefPage):
     blacklisted_attributes = ['condition']
+    is_documentable = False
 
 
 class UserFieldsDirectory(FieldsDirectory):

wcs/admin/tests.py

@@ -25,10 +25,14 @@ from quixote import get_publisher, get_request, get_response, get_session, redir
 from quixote.directory import Directory
 from quixote.html import TemplateIO, htmltext
 
+from wcs.admin import utils
 from wcs.admin.workflow_tests import WorkflowTestsDirectory
 from wcs.api import posted_json_data_to_formdata_data
 from wcs.backoffice.management import FormBackofficeEditPage, FormBackOfficeStatusPage
 from wcs.backoffice.pagination import pagination_links
+from wcs.backoffice.snapshots import SnapshotsDirectory
+from wcs.carddef import CardDef
+from wcs.formdef import FormDef
 from wcs.forms.common import FormStatusPage
 from wcs.qommon import _, misc, template
 from wcs.qommon.afterjobs import AfterJob
@@ -130,12 +134,12 @@ class TestEditPage(FormBackofficeEditPage):
         self.testdef.data = testdef.data
 
         self.testdef.expected_error = get_request().form.get('error')
-        self.testdef.store()
+        self.testdef.store(comment=_('Mark test as failing'))
         return redirect('..')
 
     def change_submission_mode(self):
         self.testdef.is_in_backoffice = not self.testdef.is_in_backoffice
-        self.testdef.store()
+        self.testdef.store(comment=_('Change submission mode'))
         return redirect('.')
 
 
@@ -148,13 +152,19 @@ class TestPage(FormBackOfficeStatusPage):
         'duplicate',
         ('workflow', 'workflow_tests'),
         ('webservice-responses', 'webservice_responses'),
+        ('history', 'snapshots_dir'),
     ]
 
-    def __init__(self, component, objectdef):
+    def __init__(self, component, objectdef=None, instance=None):
         try:
-            self.testdef = TestDef.get(component)
+            self.testdef = instance or TestDef.get(component)
         except KeyError:
             raise TraversalError()
+
+        if not objectdef:
+            klass = FormDef if self.testdef.object_type == 'formdefs' else CardDef
+            objectdef = klass.get(self.testdef.object_id)
+
         self.testdef.formdef = objectdef
 
         filled = self.testdef.build_formdata(objectdef, include_fields=True)
@@ -162,6 +172,7 @@ class TestPage(FormBackOfficeStatusPage):
 
         self.workflow_tests = WorkflowTestsDirectory(self.testdef, self.formdef)
         self.webservice_responses = WebserviceResponseDirectory(self.testdef)
+        self.snapshots_dir = SnapshotsDirectory(self.testdef)
 
     @property
     def edit_data(self):
@@ -179,16 +190,28 @@ class TestPage(FormBackOfficeStatusPage):
         return False
 
     def get_extra_context_bar(self, parent=None):
-        return render_to_string('wcs/backoffice/test_sidebar.html', context={})
+        if self.testdef.is_readonly():
+            r = TemplateIO(html=True)
+            r += htmltext('<div class="infonotice"><p>%s</p></div>') % _('This test is readonly.')
+            r += utils.snapshot_info_block(self.testdef.snapshot_object)
+            r += htmltext('<h3>%s</h3>') % _('Navigation')
+            r += htmltext(
+                '<li><a class="button button-paragraph" href="webservice-responses/">%s</a></li>'
+            ) % _('Webservice responses')
+            r += htmltext('<li><a class="button button-paragraph" href="inspect">%s</a></li>') % _('Inspect')
+            r += htmltext('</h3>')
+            return r.getvalue()
+        else:
+            return render_to_string('wcs/backoffice/test_sidebar.html', context={})
 
     def status(self):
         r = TemplateIO(html=True)
         r += htmltext('<div id="appbar">')
         r += htmltext('<h2>%s</h2>') % self.testdef
         r += htmltext('<span class="actions">')
-        r += htmltext('<a href="edit-data/">%s</a>') % _('Edit data')
-        if get_publisher().has_site_option('enable-workflow-tests'):
-            r += htmltext('<a href="workflow/">%s</a>') % _('Workflow tests')
+        if not self.testdef.is_readonly():
+            r += htmltext('<a href="edit-data/">%s</a>') % _('Edit data')
+        r += htmltext('<a href="workflow/">%s</a>') % _('Workflow tests')
         r += htmltext('</span>')
         r += htmltext('</div>')
         if self.testdef.expected_error:
@@ -255,7 +278,7 @@ class TestPage(FormBackOfficeStatusPage):
             self.testdef.name = form.get_widget('name').parse()
             self.testdef.user_uuid = form.get_widget('user').parse()
 
-            self.testdef.store()
+            self.testdef.store(comment=_('Change in options'))
             return redirect('.')
 
     def duplicate(self):
@@ -286,7 +309,7 @@ class TestPage(FormBackOfficeStatusPage):
 
         self.testdef.name = form.get_widget('name').parse()
         self.testdef = TestDef.import_from_xml_tree(self.testdef.export_to_xml(), self.formdef)
-        self.testdef.store()
+        self.testdef.store(comment=_('Creation (from duplication)'))
 
         return redirect(self.testdef.get_admin_url())
 
@@ -343,11 +366,8 @@ class TestsDirectory(Directory):
             creation_options = [
                 ('empty', _('Fill data manually'), 'empty'),
                 ('formdata', _('Import data from form'), 'formdata'),
+                ('formdata-wf', _('Import data from form (and initialise workflow tests)'), 'formdata-wf'),
             ]
-            if get_publisher().has_site_option('enable-workflow-tests'):
-                creation_options.append(
-                    ('formdata-wf', _('Import data from form (and initialise workflow tests)'), 'formdata-wf')
-                )
             form.add(
                 RadiobuttonsWidget,
                 'creation_mode',
@@ -390,7 +410,7 @@ class TestsDirectory(Directory):
             testdef = TestDef.create_from_formdata(self.objectdef, self.objectdef.data_class()())
             testdef.name = form.get_widget('name').parse()
             testdef.agent_id = test_agent_user.test_uuid
-            testdef.store()
+            testdef.store(comment=_('Creation (empty)'))
             return redirect(testdef.get_admin_url() + 'edit-data/')
         else:
             formdata_id = form.get_widget('formdata').parse()
@@ -403,7 +423,7 @@ class TestsDirectory(Directory):
             )
             testdef.name = form.get_widget('name').parse()
             testdef.agent_id = test_agent_user.test_uuid
-            testdef.store()
+            testdef.store(comment=_('Creation (from formdata)'))
             return redirect(testdef.get_admin_url())
 
     def p_import(self):
@@ -426,9 +446,6 @@ class TestsDirectory(Directory):
         get_response().set_title(_('Import Test'))
         r = TemplateIO(html=True)
         r += htmltext('<h2>%s</h2>') % _('Import Test')
-        r += htmltext('<p>%s</p>') % _(
-            'You can add a new test or update an existing one by importing a JSON file.'
-        )
         r += form.render()
         return r.getvalue()
 
@@ -441,6 +458,7 @@ class TestsDirectory(Directory):
             form.set_error('file', _('Invalid File'))
             raise e
 
+        testdef.store(comment=_('Creation (from import)'))
         get_session().message = ('info', _('Test "%s" has been successfully imported.') % testdef.name)
         return redirect('.')
 
@@ -825,7 +843,8 @@ class WebserviceResponsePage(Directory):
             },
         )
 
-        form.add_submit('submit', _('Submit'))
+        if not self.testdef.is_readonly():
+            form.add_submit('submit', _('Submit'))
         form.add_submit('cancel', _('Cancel'))
         form.add_media()
 
@@ -847,6 +866,7 @@ class WebserviceResponsePage(Directory):
         self.webservice_response.method = form.get_widget('method').parse()
         self.webservice_response.post_data = form.get_widget('post_data').parse()
         self.webservice_response.store()
+        self.testdef.store(comment=_('Change webservice response "%s"') % self.webservice_response.name)
 
         return redirect('..')
 
@@ -872,11 +892,14 @@ class WebserviceResponsePage(Directory):
         new_webservice_response.id = None
         new_webservice_response.name = '%s %s' % (new_webservice_response.name, _('(copy)'))
         new_webservice_response.store()
+        self.testdef.store(
+            comment=_('Duplication of webservice response "%s"') % self.webservice_response.name
+        )
         return redirect('..')
 
 
 class WebserviceResponseDirectory(Directory):
-    _q_exports = ['', 'new']
+    _q_exports = ['', 'new', ('import', 'p_import')]
 
     def __init__(self, testdef):
         self.testdef = testdef
@@ -891,7 +914,8 @@ class WebserviceResponseDirectory(Directory):
     def _q_index(self):
         context = {
             'webservice_responses': self.testdef.get_webservice_responses(),
-            'has_sidebar': True,
+            'has_sidebar': bool(not self.testdef.is_readonly()),
+            'testdef': self.testdef,
         }
         get_response().add_javascript(['popup.js'])
         get_response().set_title(_('Webservice responses'))
@@ -923,12 +947,53 @@ class WebserviceResponseDirectory(Directory):
         webservice_response.testdef_id = self.testdef.id
         webservice_response.name = form.get_widget('name').parse()
         webservice_response.store()
+        self.testdef.store(comment=_('New webservice response "%s"') % webservice_response.name)
 
         return redirect(self.testdef.get_admin_url() + 'webservice-responses/%s/' % webservice_response.id)
 
+    def p_import(self):
+        form = Form(enctype='multipart/form-data')
+
+        testdef_options = [
+            (x.id, x, x.id)
+            for x in TestDef.select_for_objectdef(self.testdef.formdef)
+            if x.id != self.testdef.id
+        ]
+        form.add(
+            SingleSelectWidget,
+            'testdef_id',
+            required=False,
+            options=testdef_options,
+            **{'data-autocomplete': 'true'},
+        )
+
+        form.add_submit('submit', _('Import'))
+        form.add_submit('cancel', _('Cancel'))
+
+        if form.get_submit() == 'cancel':
+            return redirect('.')
+
+        if not form.is_submitted() or form.has_errors():
+            get_response().breadcrumb.append(('import', _('Import')))
+            get_response().set_title(_('Import webservice responses'))
+            r = TemplateIO(html=True)
+            r += htmltext('<h2>%s</h2>') % _('Import webservice responses')
+            r += form.render()
+            return r.getvalue()
+
+        testdef_id = form.get_widget('testdef_id').parse()
+        testdef = TestDef.get(testdef_id)
+
+        for response in testdef.get_webservice_responses():
+            response.id = None
+            response.testdef_id = self.testdef.id
+            response.store()
+
+        return redirect('.')
+
 
 class TestUserPage(Directory):
-    _q_exports = ['', 'delete']
+    _q_exports = ['', 'delete', 'export']
 
     def __init__(self, component):
         try:
@@ -988,7 +1053,12 @@ class TestUserPage(Directory):
 
         get_response().breadcrumb.append(('edit', _('Edit test user')))
         r = TemplateIO(html=True)
+        r += htmltext('<div id="appbar">')
         r += htmltext('<h2>%s</h2>') % (_('Edit test user'))
+        r += htmltext('<span class="actions">')
+        r += htmltext('<a href="export">%s</a>') % _('Export')
+        r += htmltext('</span>')
+        r += htmltext('</div>')
         r += form.render()
         return r.getvalue()
 
@@ -1009,9 +1079,16 @@ class TestUserPage(Directory):
         self.user.remove_object(self.user.id)
         return redirect('..')
 
+    def export(self):
+        get_response().set_content_type('application/json')
+        get_response().set_header(
+            'content-disposition', 'attachment; filename=wcs_test_user_%s.json' % self.user.name
+        )
+        return json.dumps({'test-users': [self.user.get_json_export_dict(full=True, include_roles=True)]})
+
 
 class TestUsersDirectory(Directory):
-    _q_exports = ['', 'new']
+    _q_exports = ['', 'new', 'export', ('import', 'p_import')]
 
     def _q_traverse(self, path):
         get_response().breadcrumb.append(('test-users/', _('Test users')))
@@ -1086,3 +1163,71 @@ class TestUsersDirectory(Directory):
         r += htmltext('<h2>%s</h2>') % _('New test user')
         r += form.render()
         return r.getvalue()
+
+    def export(self):
+        get_response().set_content_type('application/json')
+        get_response().set_header('content-disposition', 'attachment; filename=wcs_test_users.json')
+        users = get_publisher().user_class.select([NotNull('test_uuid')])
+        return json.dumps(
+            {'test-users': [x.get_json_export_dict(full=True, include_roles=True) for x in users]}
+        )
+
+    def p_import(self):
+        form = Form(enctype='multipart/form-data')
+
+        form.add(FileWidget, 'file', title=_('File'), required=True)
+        form.add_submit('submit', _('Import'))
+        form.add_submit('cancel', _('Cancel'))
+
+        if form.get_submit() == 'cancel':
+            return redirect('.')
+
+        if form.is_submitted() and not form.has_errors():
+            try:
+                return self.import_submit(form)
+            except ValueError:
+                pass
+
+        get_response().breadcrumb.append(('import', _('Import')))
+        get_response().set_title(_('Import test users'))
+        r = TemplateIO(html=True)
+        r += htmltext('<h2>%s</h2>') % _('Import test users')
+        r += form.render()
+        return r.getvalue()
+
+    def import_submit(self, form):
+        fp = form.get_widget('file').parse().fp
+        try:
+            data = json.loads(fp.read())
+        except ValueError:
+            form.set_error('file', _('Invalid JSON file'))
+            raise ValueError
+
+        existing_users = get_publisher().user_class.select([NotNull('test_uuid')])
+        existing_uuids = {x.test_uuid for x in existing_users}
+        existing_emails = {x.email for x in existing_users}
+
+        users = []
+        users_were_ignored = False
+        for user_dict in data.get('test-users', []):
+            try:
+                user = get_publisher().user_class.import_from_json(user_dict)
+            except KeyError:
+                form.set_error('file', _('Invalid File'))
+                raise ValueError
+
+            if user.test_uuid in existing_uuids or user.email in existing_emails:
+                users_were_ignored = True
+                continue
+
+            users.append(user)
+
+        for user in users:
+            user.store()
+
+        if users_were_ignored:
+            get_session().message = ('warning', _('Some already existing users were not imported.'))
+        else:
+            get_session().message = ('success', _('Test users have been successfully imported.'))
+
+        return redirect('.')

wcs/admin/workflow_tests.py

@@ -47,7 +47,8 @@ class WorkflowTestActionPage(Directory):
         if not form.widgets:
             form.add_global_errors([htmltext(self.action.empty_form_error)])
         else:
-            form.add_submit('submit', _('Submit'))
+            if not self.testdef.is_readonly():
+                form.add_submit('submit', _('Submit'))
         form.add_submit('cancel', _('Cancel'))
 
         if form.get_widget('cancel').parse():
@@ -69,7 +70,7 @@ class WorkflowTestActionPage(Directory):
 
             setattr(self.action, widget.name, value)
 
-        self.testdef.store()
+        self.testdef.store(comment=_('Change in workflow test action "%s"') % self.action.label)
         return redirect('..')
 
     def delete(self):
@@ -90,7 +91,7 @@ class WorkflowTestActionPage(Directory):
         self.testdef.workflow_tests.actions = [
             x for x in self.testdef.workflow_tests.actions if x.id != self.action.id
         ]
-        self.testdef.store()
+        self.testdef.store(comment=_('Deletion of workflow test action "%s"') % self.action.label)
         return redirect('..')
 
     def duplicate(self):
@@ -98,7 +99,7 @@ class WorkflowTestActionPage(Directory):
         new_action.id = self.testdef.workflow_tests.get_new_action_id()
         action_position = self.testdef.workflow_tests.actions.index(self.action)
         self.testdef.workflow_tests.actions.insert(action_position + 1, new_action)
-        self.testdef.store()
+        self.testdef.store(comment=_('Duplication of workflow test action "%s"') % self.action.label)
         return redirect('..')
 
 
@@ -120,7 +121,7 @@ class WorkflowTestsDirectory(Directory):
     def _q_index(self):
         context = {
             'testdef': self.testdef,
-            'has_sidebar': True,
+            'has_sidebar': bool(not self.testdef.is_readonly()),
             'sidebar_form': self.get_sidebar_form(),
         }
 
@@ -161,7 +162,8 @@ class WorkflowTestsDirectory(Directory):
             **{'data-autocomplete': 'true'},
         )
 
-        form.add_submit('submit', _('Submit'))
+        if not self.testdef.is_readonly():
+            form.add_submit('submit', _('Submit'))
         form.add_submit('cancel', _('Cancel'))
 
         if form.get_widget('cancel').parse():
@@ -176,7 +178,7 @@ class WorkflowTestsDirectory(Directory):
             return r.getvalue()
 
         self.testdef.agent_id = form.get_widget('agent').parse()
-        self.testdef.store()
+        self.testdef.store(comment=_('Change in workflow test options'))
         return redirect('.')
 
     def new(self):
@@ -190,7 +192,7 @@ class WorkflowTestsDirectory(Directory):
         action_type = form.get_widget('type').parse()
         action_class = get_test_action_class_by_type(action_type)
         self.testdef.workflow_tests.add_action(action_class)
-        self.testdef.store()
+        self.testdef.store(comment=_('New test action "%s"') % action_class.label)
 
         return redirect('.')
 
@@ -219,7 +221,7 @@ class WorkflowTestsDirectory(Directory):
             return json.dumps({'success': 'ko'})
 
         self.testdef.workflow_tests.actions = new_actions
-        self.testdef.store()
+        self.testdef.store(comment=_('Change in workflow test actions order'))
 
         return json.dumps(
             {

wcs/admin/workflows.py

@@ -67,6 +67,7 @@ from wcs.workflows import (
 from . import utils
 from .comment_templates import CommentTemplatesDirectory
 from .data_sources import NamedDataSourcesDirectory
+from .documentable import DocumentableFieldMixin, DocumentableMixin
 from .fields import FieldDefPage, FieldsDirectory
 from .logged_errors import LoggedErrorsDirectory
 from .mail_templates import MailTemplatesDirectory
@@ -440,8 +441,14 @@ class WorkflowUI:
         return workflow
 
 
-class WorkflowItemPage(Directory):
-    _q_exports = ['', 'delete', 'copy']
+class WorkflowItemPage(Directory, DocumentableMixin):
+    _q_exports = [
+        '',
+        'delete',
+        'copy',
+        ('update-documentation', 'update_documentation'),
+    ]
+    do_not_call_in_templates = True
 
     def __init__(self, workflow, parent, component):
         try:
@@ -450,6 +457,8 @@ class WorkflowItemPage(Directory):
             raise errors.TraversalError()
         self.workflow = workflow
         self.parent = parent
+        self.documented_object = self.workflow
+        self.documented_element = self.item
         get_response().breadcrumb.append(('items/%s/' % component, self.item.description))
 
     def _q_index(self):
@@ -491,12 +500,20 @@ class WorkflowItemPage(Directory):
                 return redirect('..')
 
         get_response().set_title('%s - %s' % (_('Workflow'), self.workflow.name))
-        r = TemplateIO(html=True)
-        r += htmltext('<h2>%s</h2>') % self.item.description
-        r += form.render()
-        if self.item.support_substitution_variables:
-            r += get_publisher().substitutions.get_substitution_html_table()
-        return r.getvalue()
+        get_response().add_javascript(['jquery-ui.js'])
+        context = {
+            'view': self,
+            'html_form': form,
+            'workflow': self.workflow,
+            'has_sidebar': True,
+            'action': self.item,
+            'get_substitution_html_table': get_publisher().substitutions.get_substitution_html_table,
+        }
+        return template.QommonTemplateResponse(
+            templates=['wcs/backoffice/workflow-action.html'],
+            context=context,
+            is_django_native=True,
+        )
 
     def delete(self):
         form = Form(enctype='multipart/form-data')
@@ -667,7 +684,7 @@ class GlobalActionItemsDir(ToChildDirectory):
     klass = WorkflowItemPage
 
 
-class WorkflowStatusPage(Directory):
+class WorkflowStatusPage(Directory, DocumentableMixin):
     _q_exports = [
         '',
         'delete',
@@ -681,6 +698,7 @@ class WorkflowStatusPage(Directory):
         'fullscreen',
         ('schema.svg', 'svg'),
         'svg',
+        ('update-documentation', 'update_documentation'),
     ]
     do_not_call_in_templates = True
 
@@ -692,6 +710,8 @@ class WorkflowStatusPage(Directory):
             raise errors.TraversalError()
 
         self.items_dir = WorkflowItemsDir(workflow, self.status)
+        self.documented_object = self.workflow
+        self.documented_element = self.status
         get_response().breadcrumb.append(('status/%s/' % status_id, self.status.name))
 
     def _q_index(self):
@@ -1081,9 +1101,16 @@ class WorkflowStatusDirectory(Directory):
         return r.getvalue()
 
 
-class WorkflowVariablesFieldDefPage(FieldDefPage):
+class WorkflowVariablesFieldDefPage(FieldDefPage, DocumentableFieldMixin):
     section = 'workflows'
     blacklisted_attributes = ['condition', 'prefill', 'display_locations', 'anonymise']
+    has_documentation = True
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.workflow = self.objectdef.workflow
+        self.documented_object = self.workflow
+        self.documented_element = self.field
 
     def form(self):
         form = super().form()
@@ -1109,7 +1136,7 @@ class WorkflowVariablesFieldDefPage(FieldDefPage):
         super().submit(form)
 
 
-class WorkflowBackofficeFieldDefPage(FieldDefPage):
+class WorkflowBackofficeFieldDefPage(FieldDefPage, DocumentableFieldMixin):
     section = 'workflows'
     blacklisted_attributes = ['condition']
 
@@ -1122,21 +1149,23 @@ class WorkflowBackofficeFieldDefPage(FieldDefPage):
                 continue
             if any(x.get('field_id') == self.field.id for x in action.fields or []):
                 usage_actions.append(action)
-        if not usage_actions:
-            return
+
         r = TemplateIO(html=True)
-        r += htmltext('<div class="actions-using-this-field">')
-        r += htmltext('<h3>%s</h3>') % _('Actions using this field')
-        r += htmltext('<ul>')
-        for action in usage_actions:
-            label = _('"%s" action') % action.label if action.label else _('Action')
-            if isinstance(action.parent, WorkflowGlobalAction):
-                location = _('in global action "%s"') % action.parent.name
-            else:
-                location = _('in status "%s"') % action.parent.name
-            r += htmltext(f'<li><a href="{action.get_admin_url()}">%s %s</a></li>') % (label, location)
-        r += htmltext('</ul>')
-        r += htmltext('<div>')
+        r += self.documentation_part()
+        if usage_actions:
+            get_response().filter['sidebar_attrs'] = ''
+            r += htmltext('<div class="actions-using-this-field">')
+            r += htmltext('<h3>%s</h3>') % _('Actions using this field')
+            r += htmltext('<ul>')
+            for action in usage_actions:
+                label = _('"%s" action') % action.label if action.label else _('Action')
+                if isinstance(action.parent, WorkflowGlobalAction):
+                    location = _('in global action "%s"') % action.parent.name
+                else:
+                    location = _('in status "%s"') % action.parent.name
+                r += htmltext(f'<li><a href="{action.get_admin_url()}">%s %s</a></li>') % (label, location)
+            r += htmltext('</ul>')
+            r += htmltext('<div>')
         return r.getvalue()
 
     def form(self):
@@ -1158,7 +1187,7 @@ class WorkflowBackofficeFieldDefPage(FieldDefPage):
 
 
 class WorkflowVariablesFieldsDirectory(FieldsDirectory):
-    _q_exports = ['', 'update_order', 'new']
+    _q_exports = ['', 'update_order', 'new', ('update-documentation', 'update_documentation')]
 
     section = 'workflows'
     field_def_page_class = WorkflowVariablesFieldDefPage
@@ -1175,8 +1204,12 @@ class WorkflowVariablesFieldsDirectory(FieldsDirectory):
 
     def index_top(self):
         r = TemplateIO(html=True)
+        r += htmltext('<div id="appbar">')
         r += htmltext('<h2>%s - %s - %s</h2>') % (_('Workflow'), self.objectdef.name, _('Variables'))
+        r += htmltext('<span class="actions">%s</span>') % self.get_documentable_button()
+        r += htmltext('</div>')
         r += get_session().display_message()
+        r += self.get_documentable_zone()
         if not self.objectdef.fields:
             r += htmltext('<p>%s</p>') % _('There are not yet any variables.')
         return r.getvalue()
@@ -1186,7 +1219,7 @@ class WorkflowVariablesFieldsDirectory(FieldsDirectory):
 
 
 class WorkflowBackofficeFieldsDirectory(FieldsDirectory):
-    _q_exports = ['', 'update_order', 'new']
+    _q_exports = ['', 'update_order', 'new', ('update-documentation', 'update_documentation')]
 
     section = 'workflows'
     field_def_page_class = WorkflowBackofficeFieldDefPage
@@ -1201,10 +1234,19 @@ class WorkflowBackofficeFieldsDirectory(FieldsDirectory):
     fields_count_total_soft_limit = 40
     fields_count_total_hard_limit = 80
 
+    def __init__(self, objectdef):
+        super().__init__(objectdef)
+        self.documented_object = objectdef
+        self.documented_element = objectdef
+
     def index_top(self):
         r = TemplateIO(html=True)
+        r += htmltext('<div id="appbar">')
         r += htmltext('<h2>%s - %s - %s</h2>') % (_('Workflow'), self.objectdef.name, _('Backoffice Fields'))
+        r += htmltext('<span class="actions">%s</span>') % self.get_documentable_button()
+        r += htmltext('</div>')
         r += get_session().display_message()
+        r += self.get_documentable_zone()
         if not self.objectdef.fields:
             r += htmltext('<p>%s</p>') % _('There are not yet any backoffice fields.')
         return r.getvalue()
@@ -1425,6 +1467,7 @@ class GlobalActionPage(WorkflowStatusPage):
         ('triggers', 'triggers_dir'),
         'update_triggers_order',
         'options',
+        ('update-documentation', 'update_documentation'),
     ]
 
     def __init__(self, workflow, action_id):
@@ -1436,6 +1479,8 @@ class GlobalActionPage(WorkflowStatusPage):
         self.status = self.action
         self.items_dir = GlobalActionItemsDir(workflow, self.action)
         self.triggers_dir = GlobalActionTriggersDir(workflow, self.action)
+        self.documented_object = self.workflow
+        self.documented_element = self.action
 
     def _q_traverse(self, path):
         get_response().breadcrumb.append(
@@ -1613,7 +1658,7 @@ class GlobalActionsDirectory(Directory):
         return r.getvalue()
 
 
-class WorkflowPage(Directory):
+class WorkflowPage(Directory, DocumentableMixin):
     _q_exports = [
         '',
         'edit',
@@ -1637,6 +1682,7 @@ class WorkflowPage(Directory):
         ('logged-errors', 'logged_errors_dir'),
         ('history', 'snapshots_dir'),
         ('fullscreen'),
+        ('update-documentation', 'update_documentation'),
     ]
     do_not_call_in_templates = True
 
@@ -1659,6 +1705,8 @@ class WorkflowPage(Directory):
         self.criticality_levels_dir = CriticalityLevelsDirectory(self.workflow)
         self.logged_errors_dir = LoggedErrorsDirectory(parent_dir=self, workflow_id=self.workflow.id)
         self.snapshots_dir = SnapshotsDirectory(self.workflow)
+        self.documented_object = self.workflow
+        self.documented_element = self.workflow
         if component:
             get_response().breadcrumb.append((component + '/', self.workflow.name))
 

wcs/admin/wscalls.py

@@ -21,10 +21,11 @@ from quixote.directory import Directory
 from quixote.html import TemplateIO, htmltext
 
 from wcs.admin import utils
+from wcs.admin.documentable import DocumentableMixin
 from wcs.backoffice.applications import ApplicationsDirectory
 from wcs.backoffice.snapshots import SnapshotsDirectory
 from wcs.qommon import _, errors, misc, template
-from wcs.qommon.form import CheckboxWidget, FileWidget, Form, HtmlWidget, SlugWidget, StringWidget, TextWidget
+from wcs.qommon.form import CheckboxWidget, FileWidget, Form, HtmlWidget, SlugWidget, StringWidget
 from wcs.utils import grep_strings
 from wcs.wscalls import NamedWsCall, NamedWsCallImportError, WsCallRequestWidget
 
@@ -38,9 +39,6 @@ class NamedWsCallUI:
     def get_form(self):
         form = Form(enctype='multipart/form-data', use_tabs=True)
         form.add(StringWidget, 'name', title=_('Name'), required=True, size=30, value=self.wscall.name)
-        form.add(
-            TextWidget, 'description', title=_('Description'), cols=40, rows=5, value=self.wscall.description
-        )
         if self.wscall.slug:
             form.add(
                 SlugWidget,
@@ -100,7 +98,6 @@ class NamedWsCallUI:
             raise ValueError()
 
         self.wscall.name = name
-        self.wscall.description = form.get_widget('description').parse()
         self.wscall.notify_on_errors = form.get_widget('notify_on_errors').parse()
         self.wscall.record_on_errors = form.get_widget('record_on_errors').parse()
         self.wscall.request = form.get_widget('request').parse()
@@ -109,7 +106,7 @@ class NamedWsCallUI:
         self.wscall.store()
 
 
-class NamedWsCallPage(Directory):
+class NamedWsCallPage(Directory, DocumentableMixin):
     do_not_call_in_templates = True
     _q_exports = [
         '',
@@ -118,6 +115,7 @@ class NamedWsCallPage(Directory):
         'export',
         ('history', 'snapshots_dir'),
         'usage',
+        ('update-documentation', 'update_documentation'),
     ]
 
     def __init__(self, component, instance=None):
@@ -128,6 +126,8 @@ class NamedWsCallPage(Directory):
         self.wscall_ui = NamedWsCallUI(self.wscall)
         get_response().breadcrumb.append((component + '/', self.wscall.name))
         self.snapshots_dir = SnapshotsDirectory(self.wscall)
+        self.documented_object = self.wscall
+        self.documented_element = self.wscall
 
     def get_sidebar(self):
         r = TemplateIO(html=True)
@@ -203,6 +203,7 @@ class NamedWsCallPage(Directory):
                 return redirect('../%s/' % self.wscall.id)
 
         get_response().breadcrumb.append(('edit', _('Edit')))
+        get_response().add_javascript(['jquery-ui.js'])
         get_response().set_title(_('Edit webservice call'))
         r = TemplateIO(html=True)
         r += htmltext('<h2>%s</h2>') % _('Edit webservice call')

wcs/api.py

@@ -27,6 +27,7 @@ from django.utils.timezone import localtime, make_naive
 from quixote import get_publisher, get_request, get_response, get_session, redirect
 from quixote.directory import Directory
 from quixote.errors import MethodNotAllowedError, RequestError
+from quixote.html import TemplateIO, htmltext
 
 import wcs.qommon.storage as st
 from wcs.admin.settings import UserFieldsFormDef
@@ -57,6 +58,7 @@ from wcs.sql_criterias import (
     StrictNotEqual,
 )
 from wcs.workflows import ContentSnapshotPart
+from wcs.wscalls import UnflattenKeysException, unflatten_keys
 
 from .backoffice.data_management import CardPage as BackofficeCardPage
 from .backoffice.management import FormPage as BackofficeFormPage
@@ -1407,6 +1409,7 @@ class ApiDirectory(Directory):
         'geojson',
         'jobs',
         ('card-file-by-token', 'card_file_by_token'),
+        ('preview-payload-structure', 'preview_payload_structure'),
         ('sign-url-token', 'sign_url_token'),
     ]
 
@@ -1434,6 +1437,81 @@ class ApiDirectory(Directory):
         get_response().set_content_type('application/json')
         return json.dumps({'err': 0, 'data': list_roles})
 
+    def preview_payload_structure(self):
+        if not (get_request().user and get_request().user.can_go_in_admin()):
+            raise AccessForbiddenError('user has no access to backoffice')
+
+        def parse_payload():
+            payload = {}
+            for param, value in get_request().form.items():
+                # skip elements which are not part of payload
+                if 'post_data$element' not in param or param.endswith('value_python'):
+                    continue
+                prefix, order, field = re.split(r'(\d)(?!\d)', param)  # noqa pylint: disable=unused-variable
+                # skip elements that aren't ordered
+                if not order:
+                    continue
+
+                if order not in payload:
+                    payload[order] = []
+
+                if field == 'key':
+                    # skip empty keys
+                    if not value:
+                        continue
+                    # insert key on first position
+                    payload[order].insert(0, value)
+                else:
+                    payload[order].append(value)
+            return dict([v for v in payload.values() if len(v) > 1])
+
+        def format_payload(o, html=htmltext(''), last_element=True):
+            if isinstance(o, (list, tuple)):
+                html += htmltext('[<span class="payload-preview--obj">')
+                while True:
+                    try:
+                        head, tail = o[0], o[1:]
+                    except IndexError:
+                        break
+                    html = format_payload(head, html=html, last_element=len(tail) < 1)
+                    o = tail
+                html += htmltext('</span>]')
+            elif isinstance(o, dict):
+                html += htmltext('{<span class="payload-preview--obj">')
+                for i, (k, v) in enumerate(o.items()):
+                    html += htmltext('<span class="payload-preview--key">"%s"</span>: ' % k)
+                    html = format_payload(v, html=html, last_element=i == len(o) - 1)
+                html += htmltext('</span>}')
+            else:
+                # check if it's empty string, a template with text around or just text
+                if not o or re.sub('^({[{|%]).+([%|}]})$', '', o):
+                    # and add double quotes
+                    html += htmltext('<span class="payload-preview--value">"%s"</span>' % o)
+                else:
+                    html += htmltext('<span class="payload-preview--template-value">%s</span>' % o)
+            # last element doesn't need separator
+            if not last_element:
+                html += htmltext('<span class="payload-preview--item-separator">,</span>')
+            return html
+
+        payload = parse_payload()
+        r = TemplateIO(html=True)
+        r += htmltext('<h2>%s</h2>') % _('Payload structure preview')
+        r += htmltext('<div class="payload-preview">')
+        try:
+            unflattened_payload = unflatten_keys(payload)
+            r += htmltext('<div class="payload-preview--structure">')
+            r += format_payload(unflattened_payload)
+            r += htmltext('</div>')
+        except UnflattenKeysException as e:
+            r += htmltext('<div class="errornotice"><p>%s</p><p>%s %s</p></div>') % (
+                _('Unable to preview payload.'),
+                _('Following error occured: '),
+                e,
+            )
+        r += htmltext('</div>')
+        return r.getvalue()
+
     def _q_traverse(self, path):
         get_request().is_json_marker = True
         return super()._q_traverse(path)

wcs/backoffice/management.py

@@ -4559,3 +4559,10 @@ class JsonFileExportAfterJob(CsvExportAfterJob):
         )
         self.content_type = 'application/json'
         self.store()
+
+
+class FakeField:
+    # 2024-04-12, legacy class, for transition from FakeField to filter_fields.*
+    # can be removed once all afterjobs referencing fake fields are removed.
+    # (= 2 days after update)
+    pass

wcs/backoffice/snapshots.py

@@ -52,7 +52,8 @@ class SnapshotsDirectory(Directory):
             templates=['wcs/backoffice/snapshots.html'],
             context={
                 'view': self,
-                'form_has_tests': bool(TestDef.select_for_objectdef(self.obj)),
+                'form_has_tests': self.object_type in ('formdef', 'carddef')
+                and bool(TestDef.select_for_objectdef(self.obj)),
             },
         )
 
@@ -261,7 +262,7 @@ class SnapshotsDirectory(Directory):
         current_date = None
         snapshots = get_publisher().snapshot_class.select_object_history(self.obj)
         test_results = TestResult.select(
-            [Equal('object_type', self.obj.get_table_name()), Equal('object_id', self.obj.id)]
+            [Equal('object_type', self.obj.get_table_name()), Equal('object_id', str(self.obj.id))]
         )
         test_results_by_id = {x.id: x for x in test_results}
         day_snapshot = None

wcs/blocks.py

@@ -60,11 +60,12 @@ class BlockDef(StorableObject):
     fields = None
     digest_template = None
     category_id = None
+    documentation = None
 
     SLUG_DASH = '_'
 
     # declarations for serialization
-    TEXT_ATTRIBUTES = ['name', 'slug', 'digest_template']
+    TEXT_ATTRIBUTES = ['name', 'slug', 'digest_template', 'documentation']
 
     def __init__(self, name=None, **kwargs):
         super().__init__(**kwargs)

wcs/comment_templates.py

@@ -34,7 +34,7 @@ class CommentTemplate(XmlStorableObject):
 
     name = None
     slug = None
-    description = None
+    documentation = None
     comment = None
     attachments = []
     category_id = None
@@ -43,7 +43,8 @@ class CommentTemplate(XmlStorableObject):
     XML_NODES = [
         ('name', 'str'),
         ('slug', 'str'),
-        ('description', 'str'),
+        ('description', 'str'),  # legacy
+        ('documentation', 'str'),
         ('comment', 'str'),
         ('attachments', 'str_list'),
     ]
@@ -52,6 +53,16 @@ class CommentTemplate(XmlStorableObject):
         XmlStorableObject.__init__(self)
         self.name = name
 
+    def migrate(self):
+        changed = False
+        if getattr(self, 'description', None):  # 2024-04-07
+            self.documentation = getattr(self, 'description')
+            self.description = None
+            changed = True
+        if changed:
+            self.store(comment=_('Automatic update'), snapshot_store_user=False)
+        return changed
+
     @property
     def category(self):
         return CommentTemplateCategory.get(self.category_id, ignore_errors=True)
@@ -67,14 +78,16 @@ class CommentTemplate(XmlStorableObject):
         base_url = get_publisher().get_backoffice_url()
         return '%s/workflows/comment-templates/%s/' % (base_url, self.id)
 
-    def store(self, comment=None, application=None, *args, **kwargs):
+    def store(self, comment=None, snapshot_store_user=True, application=None, *args, **kwargs):
         assert not self.is_readonly()
         if self.slug is None:
             # set slug if it's not yet there
             self.slug = self.get_new_slug()
         super().store(*args, **kwargs)
         if get_publisher().snapshot_class:
-            get_publisher().snapshot_class.snap(instance=self, comment=comment, application=application)
+            get_publisher().snapshot_class.snap(
+                instance=self, store_user=snapshot_store_user, comment=comment, application=application
+            )
 
     def get_places_of_use(self):
         from wcs.workflows import Workflow

wcs/data_sources.py

@@ -676,7 +676,7 @@ class NamedDataSource(XmlStorableObject):
 
     name = None
     slug = None
-    description = None
+    documentation = None
     data_source = None
     cache_duration = None
     query_parameter = None
@@ -702,7 +702,8 @@ class NamedDataSource(XmlStorableObject):
     XML_NODES = [
         ('name', 'str'),
         ('slug', 'str'),
-        ('description', 'str'),
+        ('description', 'str'),  # legacy
+        ('documentation', 'str'),
         ('cache_duration', 'str'),
         ('query_parameter', 'str'),
         ('id_parameter', 'str'),
@@ -737,6 +738,11 @@ class NamedDataSource(XmlStorableObject):
                 self.data_source['value'] = translate_url(publisher, url)
                 changed = True
 
+        if getattr(self, 'description', None):  # 2024-04-07
+            self.documentation = getattr(self, 'description')
+            self.description = None
+            changed = True
+
         if changed:
             self.store(comment=_('Automatic update'), snapshot_store_user=False)
 

wcs/fields/base.py

@@ -223,6 +223,7 @@ class Field:
     is_no_data_field = False
     can_include_in_listing = False
     available_for_filter = False
+    documentation = None
 
     # flag a field for removal by AnonymiseWorkflowStatusItem
     # possible values are final, intermediate, no.
@@ -232,7 +233,7 @@ class Field:
 
     # declarations for serialization, they are mostly for legacy files,
     # new exports directly include typing attributes.
-    TEXT_ATTRIBUTES = ['label', 'type', 'hint', 'varname', 'extra_css_class']
+    TEXT_ATTRIBUTES = ['label', 'type', 'hint', 'varname', 'extra_css_class', 'documentation']
 
     def __init__(self, **kwargs):
         for k, v in kwargs.items():
@@ -320,7 +321,7 @@ class Field:
 
     def export_to_xml(self, include_id=False):
         field = ET.Element('field')
-        extra_fields = ['default_value']  # specific to workflow variables
+        extra_fields = ['default_value', 'documentation']  # default_value is specific to workflow variables
         if include_id:
             extra_fields.append('id')
         ET.SubElement(field, 'type').text = self.key
@@ -359,7 +360,7 @@ class Field:
         return field
 
     def init_with_xml(self, elem, include_id=False, snapshot=False):
-        extra_fields = ['default_value']  # specific to workflow variables
+        extra_fields = ['documentation', 'default_value']  # default_value is specific to workflow variables
         for attribute in self.get_admin_attributes() + extra_fields:
             el = elem.find(attribute)
             if hasattr(self, '%s_init_with_xml' % attribute):

wcs/formdef.py

@@ -186,6 +186,7 @@ class FormDef(StorableObject):
     drafts_lifespan = None
     drafts_max_per_user = None
     user_support = None
+    documentation = None
 
     geolocations = None
     history_pane_default_mode = 'expanded'
@@ -219,6 +220,7 @@ class FormDef(StorableObject):
         'drafts_lifespan',
         'drafts_max_per_user',
         'user_support',
+        'documentation',
     ]
     BOOLEAN_ATTRIBUTES = [
         'discussion',
@@ -1766,7 +1768,8 @@ class FormDef(StorableObject):
         from .testdef import TestDef
 
         for testdef in self.xml_testdefs:
-            TestDef.import_from_xml_tree(testdef, self)
+            obj = TestDef.import_from_xml_tree(testdef, self)
+            obj.store()
 
     def get_detailed_email_form(self, formdata, url):
         r = ''

wcs/forms/root.py

@@ -472,6 +472,9 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
             prefilled = False
             locked = False
 
+            if block:
+                field_key = f'{block.id}${block_idx}${field_key}'
+
             if field.get_prefill_configuration():
                 prefill_user = get_request().user
                 if get_request().is_in_backoffice():
@@ -1992,7 +1995,7 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
             testdef = TestDef.create_from_formdata(self.formdef, self.edited_data)
             self.testdef.data = testdef.data
             self.testdef.expected_error = None
-            self.testdef.store()
+            self.testdef.store(comment=_('Change in test data'))
             return redirect(self.formdef.get_admin_url() + 'tests/%s/' % self.testdef.id)
 
         evo = self.edited_data.evolution[-1]

wcs/locale/fr/LC_MESSAGES/django.po

@@ -4,8 +4,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: wcs 0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-04-12 08:19+0200\n"
-"PO-Revision-Date: 2024-04-12 08:19+0200\n"
+"POT-Creation-Date: 2024-04-15 12:00+0200\n"
+"PO-Revision-Date: 2024-04-15 12:00+0200\n"
 "Last-Translator: Thomas Noël <tnoel@entrouvert.com>\n"
 "Language-Team: french\n"
 "Language: fr\n"
@@ -16,16 +16,14 @@ msgstr ""
 
 #: admin/api_access.py admin/blocks.py admin/comment_templates.py
 #: admin/data_sources.py admin/forms.py admin/mail_templates.py admin/tests.py
-#: admin/users.py admin/workflows.py admin/wscalls.py backoffice/management.py
-#: fields/base.py qommon/ident/franceconnect.py
+#: admin/users.py admin/workflows.py admin/wscalls.py
+#: backoffice/filter_fields.py fields/base.py qommon/ident/franceconnect.py
 #: templates/wcs/backoffice/test-result.html wf/profile.py workflow_tests.py
 msgid "Name"
 msgstr "Nom"
 
-#: admin/api_access.py admin/categories.py admin/comment_templates.py
-#: admin/data_sources.py admin/forms.py admin/mail_templates.py
-#: admin/wscalls.py qommon/ident/franceconnect.py
-#: templates/wcs/backoffice/formdef-inspect.html
+#: admin/api_access.py admin/categories.py admin/forms.py
+#: qommon/ident/franceconnect.py templates/wcs/backoffice/formdef-inspect.html
 #: templates/wcs/backoffice/snapshots.html
 msgid "Description"
 msgstr "Description"
@@ -176,10 +174,11 @@ msgid "Applications"
 msgstr "Applications"
 
 #: admin/blocks.py admin/comment_templates.py admin/data_sources.py
-#: admin/mail_templates.py admin/settings.py admin/wscalls.py
+#: admin/mail_templates.py admin/settings.py admin/tests.py admin/wscalls.py
 #: backoffice/i18n.py backoffice/management.py
 #: templates/wcs/backoffice/category.html templates/wcs/backoffice/formdef.html
 #: templates/wcs/backoffice/i18n.html templates/wcs/backoffice/snapshots.html
+#: templates/wcs/backoffice/test-users.html
 #: templates/wcs/backoffice/test_sidebar.html
 #: templates/wcs/backoffice/workflow.html
 msgid "Export"
@@ -238,7 +237,7 @@ msgstr "Enregistrer une sauvegarde"
 msgid "Overwrite with new import"
 msgstr "Écraser avec un nouvel import"
 
-#: admin/blocks.py templates/wcs/backoffice/blocks.html
+#: admin/blocks.py admin/tests.py templates/wcs/backoffice/blocks.html
 #: templates/wcs/backoffice/cards.html templates/wcs/backoffice/category.html
 #: templates/wcs/backoffice/comment-templates.html
 #: templates/wcs/backoffice/data-sources.html
@@ -254,6 +253,7 @@ msgstr "Navigation"
 #: admin/mail_templates.py admin/wscalls.py backoffice/snapshots.py
 #: templates/wcs/backoffice/category.html templates/wcs/backoffice/formdef.html
 #: templates/wcs/backoffice/snapshots.html
+#: templates/wcs/backoffice/test_sidebar.html
 #: templates/wcs/backoffice/workflow.html
 msgid "History"
 msgstr "Historique"
@@ -379,7 +379,8 @@ msgstr "Importer un bloc de champs"
 #: templates/wcs/backoffice/data-sources.html
 #: templates/wcs/backoffice/forms.html templates/wcs/backoffice/i18n.html
 #: templates/wcs/backoffice/mail-templates.html
-#: templates/wcs/backoffice/tests.html templates/wcs/backoffice/workflows.html
+#: templates/wcs/backoffice/test-users.html templates/wcs/backoffice/tests.html
+#: templates/wcs/backoffice/workflows.html
 #: templates/wcs/backoffice/wscalls.html
 msgid "Import"
 msgstr "Importer"
@@ -870,7 +871,11 @@ msgid "Agendas will be updated in the background."
 msgstr ""
 "Les sources basées sur les agendas vont être actualisées en arrière-plan."
 
-#: admin/fields.py admin/settings.py admin/users.py backoffice/management.py
+#: admin/documentable.py
+msgid "Documentation update"
+msgstr "Mise à jour de la documentation"
+
+#: admin/fields.py admin/settings.py admin/users.py backoffice/filter_fields.py
 #: data_sources.py fields/base.py qommon/form.py qommon/ident/password.py
 #: statistics/views.py wf/create_formdata.py workflow_tests.py
 msgid "None"
@@ -1296,7 +1301,7 @@ msgstr ""
 msgid "Appearance"
 msgstr "Apparence"
 
-#: admin/forms.py backoffice/management.py
+#: admin/forms.py backoffice/filter_fields.py backoffice/management.py
 #: templates/wcs/backoffice/formdef-inspect.html
 msgid "Digest"
 msgstr "Résumé"
@@ -1633,8 +1638,8 @@ msgstr "Un changement de workflow est déjà en cours."
 msgid "Invalid target workflow."
 msgstr "Workflow cible invalide."
 
-#: admin/forms.py backoffice/management.py formdef.py
-#: templates/wcs/backoffice/test-results.html workflows.py
+#: admin/forms.py backoffice/filter_fields.py backoffice/management.py
+#: formdef.py templates/wcs/backoffice/test-results.html workflows.py
 msgid "Status"
 msgstr "Statut"
 
@@ -1813,8 +1818,8 @@ msgstr "Gabarit"
 msgid "Text"
 msgstr "Texte"
 
-#: admin/logged_errors.py backoffice/management.py formdata.py
-#: templates/wcs/backoffice/includes/inspect-draft-by-page.html
+#: admin/logged_errors.py backoffice/filter_fields.py backoffice/management.py
+#: formdata.py templates/wcs/backoffice/includes/inspect-draft-by-page.html
 #: wf/create_formdata.py workflows.py
 msgid "Unknown"
 msgstr "Inconnu"
@@ -2113,6 +2118,7 @@ msgstr ""
 "converties vers le type correspondant."
 
 #: admin/settings.py admin/workflows.py backoffice/management.py
+#: templates/wcs/backoffice/includes/documentation.html
 msgid "Save"
 msgstr "Enregistrer"
 
@@ -2401,8 +2407,8 @@ msgstr "URL"
 msgid "Database Name"
 msgstr "Nom de la base de données"
 
-#: admin/settings.py admin/tests.py admin/users.py backoffice/journal.py
-#: backoffice/management.py templates/wcs/backoffice/journal.html
+#: admin/settings.py admin/tests.py admin/users.py backoffice/filter_fields.py
+#: backoffice/journal.py templates/wcs/backoffice/journal.html
 #: templates/wcs/backoffice/snapshots.html users.py
 msgid "User"
 msgstr "Utilisateur"
@@ -2594,6 +2600,29 @@ msgstr "Modifier les données"
 msgid "Mark as failing"
 msgstr "Marquer comme devant échouer"
 
+#: admin/tests.py templates/wcs/backoffice/test_edit_sidebar.html
+msgid "Mark test as failing"
+msgstr "Marquer le test comme devant échouer"
+
+#: admin/tests.py
+msgid "Change submission mode"
+msgstr "Modification du mode de soumission"
+
+#: admin/tests.py
+msgid "This test is readonly."
+msgstr "Ce test est en lecture seule."
+
+#: admin/tests.py templates/wcs/backoffice/test-webservice-responses.html
+#: templates/wcs/backoffice/test_sidebar.html
+msgid "Webservice responses"
+msgstr "Réponses webservice"
+
+#: admin/tests.py templates/wcs/backoffice/snapshots_compare.html
+#: templates/wcs/backoffice/test-result.html
+#: templates/wcs/backoffice/test_sidebar.html
+msgid "Inspect"
+msgstr "Inspecteur"
+
 #: admin/tests.py admin/workflow_tests.py
 #: templates/wcs/backoffice/workflow-tests.html
 msgid "Workflow tests"
@@ -2616,10 +2645,18 @@ msgstr "Suppression du test :"
 msgid "Edit test"
 msgstr "Modifier le test"
 
+#: admin/tests.py
+msgid "Change in options"
+msgstr "Changement dans les options"
+
 #: admin/tests.py
 msgid "Duplicate test"
 msgstr "Dupliquer le test"
 
+#: admin/tests.py
+msgid "Creation (from duplication)"
+msgstr "Création (via duplication)"
+
 #: admin/tests.py templates/wcs/backoffice/formdef.html
 #: templates/wcs/backoffice/snapshots.html templates/wcs/backoffice/tests.html
 msgid "Tests"
@@ -2655,15 +2692,21 @@ msgstr ""
 msgid "New test"
 msgstr "Nouveau test"
 
+#: admin/tests.py
+msgid "Creation (empty)"
+msgstr "Création (vide)"
+
+#: admin/tests.py
+msgid "Creation (from formdata)"
+msgstr "Création (depuis une demande)"
+
 #: admin/tests.py
 msgid "Import Test"
 msgstr "Importer un test"
 
 #: admin/tests.py
-msgid ""
-"You can add a new test or update an existing one by importing a JSON file."
-msgstr ""
-"Vous pouvez créer ou mettre à jour un test en téléchargeant un fichier JSON."
+msgid "Creation (from import)"
+msgstr "Création (via importation)"
 
 #: admin/tests.py
 #, python-format
@@ -2747,19 +2790,33 @@ msgstr "Limiter au données contenues dans le corps de la requête"
 msgid "Edit webservice response"
 msgstr "Modifier la réponse webservice"
 
+#: admin/tests.py
+#, python-format
+msgid "Change webservice response \"%s\""
+msgstr "Modification de la réponse webservice « %s »"
+
 #: admin/tests.py
 msgid "Deleting:"
 msgstr "Suppression :"
 
-#: admin/tests.py templates/wcs/backoffice/test-webservice-responses.html
-#: templates/wcs/backoffice/test_sidebar.html
-msgid "Webservice responses"
-msgstr "Réponses webservice"
+#: admin/tests.py
+#, python-format
+msgid "Duplication of webservice response \"%s\""
+msgstr "Duplication de la réponse webservice « %s »"
 
 #: admin/tests.py
 msgid "New webservice response"
 msgstr "Nouvelle réponse webservice"
 
+#: admin/tests.py
+#, python-format
+msgid "New webservice response \"%s\""
+msgstr "Nouvelle réponse webservice « %s »"
+
+#: admin/tests.py
+msgid "Import webservice responses"
+msgstr "Importer des réponses webservice"
+
 #: admin/tests.py
 msgid "Test user label"
 msgstr "Libellé de l’utilisateur de test"
@@ -2789,6 +2846,22 @@ msgstr "Copier un utilisateur existant"
 msgid "New test user"
 msgstr "Nouvel utilisateur de test"
 
+#: admin/tests.py
+msgid "Import test users"
+msgstr "Importer des utilisateurs de test"
+
+#: admin/tests.py backoffice/data_management.py
+msgid "Invalid JSON file"
+msgstr "Fichier JSON invalide"
+
+#: admin/tests.py
+msgid "Some already existing users were not imported."
+msgstr "Certains utilisateurs existaient déjà et n’ont pas été importés."
+
+#: admin/tests.py
+msgid "Test users have been successfully imported."
+msgstr "Les utilisateurs de test ont été importés correctement."
+
 #: admin/users.py fields/base.py fields/email.py formdata.py formdef.py
 #: forms/root.py qommon/admin/emails.py qommon/ident/franceconnect.py
 #: qommon/ident/idp.py qommon/ident/password.py wf/profile.py wf/sendmail.py
@@ -2920,19 +2993,47 @@ msgstr "Inspecter cette version"
 msgid "Edit action"
 msgstr "Modifier l’action"
 
+#: admin/workflow_tests.py
+#, python-format
+msgid "Change in workflow test action \"%s\""
+msgstr "Modification de l’action de test « %s »"
+
 #: admin/workflow_tests.py
 msgid "Deleting action:"
 msgstr "Suppression de l’action :"
 
+#: admin/workflow_tests.py
+#, python-format
+msgid "Deletion of workflow test action \"%s\""
+msgstr "Suppression de l’action de test « %s »"
+
+#: admin/workflow_tests.py
+#, python-format
+msgid "Duplication of workflow test action \"%s\""
+msgstr "Duplication de l’action de test « %s »"
+
 #: admin/workflow_tests.py workflow_tests.py
 msgid "Backoffice user"
 msgstr "Utilisateur agent"
 
+#: admin/workflow_tests.py
+msgid "Change in workflow test options"
+msgstr "Changement dans les options de test de workflow"
+
+#: admin/workflow_tests.py
+#, python-format
+msgid "New test action \"%s\""
+msgstr "Nouvelle action de test « %s »"
+
+#: admin/workflow_tests.py
+msgid "Change in workflow test actions order"
+msgstr "Changement de l’ordre des actions de test"
+
 #: admin/workflows.py
 msgid "Workflow Name"
 msgstr "Nom du workflow"
 
-#: admin/workflows.py backoffice/management.py fields/base.py
+#: admin/workflows.py backoffice/filter_fields.py fields/base.py
 #: fields/computed.py fields/page.py qommon/form.py
 #: templates/wcs/backoffice/snapshots.html
 #: templates/wcs/backoffice/test-results.html wf/attachment.py wf/choice.py
@@ -4003,9 +4104,9 @@ msgstr "Importer des fiches depuis un fichier"
 msgid "will be ignored - type %s not supported"
 msgstr "sera ignoré - type %s pas pris en charge"
 
-#: backoffice/data_management.py backoffice/management.py fields/base.py
-#: fields/bool.py formdata.py statistics/views.py
-#: templates/wcs/backoffice/data-source.html workflows.py
+#: backoffice/data_management.py backoffice/filter_fields.py
+#: backoffice/management.py fields/base.py fields/bool.py formdata.py
+#: statistics/views.py templates/wcs/backoffice/data-source.html workflows.py
 msgid "Yes"
 msgstr "Oui"
 
@@ -4078,10 +4179,6 @@ msgstr "(lignes %s)"
 msgid "(line numbers %s and more)"
 msgstr "(lignes %s et plus)"
 
-#: backoffice/data_management.py
-msgid "Invalid JSON file"
-msgstr "Fichier JSON invalide"
-
 #: backoffice/data_management.py
 msgid "This card has already been submitted."
 msgstr "Cette fiche a déjà été enregistrée."
@@ -4238,6 +4335,113 @@ msgstr "Webservice"
 msgid "Python expression detected"
 msgstr "Expression Python détectée"
 
+#: backoffice/filter_fields.py backoffice/management.py fields/base.py
+#: fields/bool.py formdata.py statistics/views.py
+#: templates/wcs/backoffice/data-source.html workflows.py
+msgid "No"
+msgstr "Non"
+
+#: backoffice/filter_fields.py
+#, python-format
+msgid "%s of User"
+msgstr "%s de l’usager"
+
+#: backoffice/filter_fields.py
+msgid "User Label"
+msgstr "Nom de l’usager"
+
+#: backoffice/filter_fields.py
+msgid "Status to display"
+msgstr "Statuts à afficher"
+
+#: backoffice/filter_fields.py backoffice/management.py
+msgid "Waiting for an action"
+msgstr "En attente de votre part"
+
+#: backoffice/filter_fields.py backoffice/management.py statistics/views.py
+msgid "All"
+msgstr "Tous"
+
+#: backoffice/filter_fields.py backoffice/management.py
+msgctxt "formdata"
+msgid "Open"
+msgstr "En attente"
+
+#: backoffice/filter_fields.py backoffice/management.py statistics/views.py
+msgid "Done"
+msgstr "Terminé"
+
+#: backoffice/filter_fields.py
+msgid "Status (for user)"
+msgstr "Statut (visible à l’usager)"
+
+#: backoffice/filter_fields.py
+msgid "Start"
+msgstr "Début"
+
+#: backoffice/filter_fields.py
+msgid "End"
+msgstr "Fin"
+
+#: backoffice/filter_fields.py
+msgid "Start (modification time)"
+msgstr "Début (date de modification)"
+
+#: backoffice/filter_fields.py
+msgid "End (modification time)"
+msgstr "Fin (date de modification)"
+
+#: backoffice/filter_fields.py
+msgid "Current user"
+msgstr "Utilisateur connecté"
+
+#: backoffice/filter_fields.py
+msgid "Current User Function"
+msgstr "Fonction de l’utilisateur connecté"
+
+#: backoffice/filter_fields.py backoffice/submission.py
+msgid "Submission Agent"
+msgstr "Agent à la saisie"
+
+#: backoffice/filter_fields.py
+msgid "Invalid user"
+msgstr "Utilisateur invalide"
+
+#: backoffice/filter_fields.py backoffice/management.py
+#: backoffice/submission.py statistics/views.py
+msgid "Channel"
+msgstr "Canal"
+
+#: backoffice/filter_fields.py
+msgid "Criticality Level"
+msgstr "Niveau de criticité"
+
+#: backoffice/filter_fields.py
+msgctxt "criticality-level"
+msgid "All"
+msgstr "Tous"
+
+#: backoffice/filter_fields.py fields/base.py
+msgid "Number"
+msgstr "Numéro"
+
+#: backoffice/filter_fields.py backoffice/management.py
+#: backoffice/submission.py
+msgid "Created"
+msgstr "Date de création"
+
+#: backoffice/filter_fields.py backoffice/management.py
+msgid "Last Modified"
+msgstr "Dernière modification"
+
+#: backoffice/filter_fields.py
+msgid "Anonymised"
+msgstr "Anonymisé"
+
+#: backoffice/filter_fields.py
+msgid "Distance"
+msgstr "Distance"
+
 #: backoffice/i18n.py backoffice/root.py templates/wcs/backoffice/i18n.html
 msgid "Multilinguism"
 msgstr "Multilinguisme"
@@ -4300,23 +4504,6 @@ msgstr "Formulaire / Modèle de fiche"
 msgid "Form/Card Identifier"
 msgstr "Identifiant de la demande/fiche"
 
-#: backoffice/management.py
-msgid "Waiting for an action"
-msgstr "En attente de votre part"
-
-#: backoffice/management.py
-msgctxt "formdata"
-msgid "Open"
-msgstr "En attente"
-
-#: backoffice/management.py statistics/views.py
-msgid "Done"
-msgstr "Terminé"
-
-#: backoffice/management.py statistics/views.py
-msgid "All"
-msgstr "Tous"
-
 #: backoffice/management.py qommon/admin/menu.py
 #: templates/wcs/backoffice/snapshots.html
 msgid "View"
@@ -4386,10 +4573,6 @@ msgstr "Toutes"
 msgid "Add Category"
 msgstr "Ajouter une catégorie"
 
-#: backoffice/management.py backoffice/submission.py statistics/views.py
-msgid "Channel"
-msgstr "Canal"
-
 #: backoffice/management.py statistics/views.py
 msgctxt "channel"
 msgid "All"
@@ -4458,14 +4641,6 @@ msgstr[1] "%(total)s éléments"
 msgid "Reference"
 msgstr "Référence"
 
-#: backoffice/management.py backoffice/submission.py
-msgid "Created"
-msgstr "Date de création"
-
-#: backoffice/management.py
-msgid "Last Modified"
-msgstr "Dernière modification"
-
 #: backoffice/management.py
 msgctxt "frontoffice"
 msgid "User"
@@ -4519,26 +4694,6 @@ msgstr "défaut"
 msgid "custom value"
 msgstr "valeur personnalisée"
 
-#: backoffice/management.py
-msgid "Start"
-msgstr "Début"
-
-#: backoffice/management.py
-msgid "End"
-msgstr "Fin"
-
-#: backoffice/management.py
-msgid "Current User Function"
-msgstr "Fonction de l’utilisateur connecté"
-
-#: backoffice/management.py backoffice/submission.py
-msgid "Submission Agent"
-msgstr "Agent à la saisie"
-
-#: backoffice/management.py
-msgid "Criticality Level"
-msgstr "Niveau de criticité"
-
 #: backoffice/management.py
 msgid "Current view"
 msgstr "Vue actuelle"
@@ -4563,24 +4718,6 @@ msgstr "Paramétrage des marqueurs"
 msgid "markers"
 msgstr "marqueurs"
 
-#: backoffice/management.py
-msgid "Status to display"
-msgstr "Statuts à afficher"
-
-#: backoffice/management.py
-msgid "Current user"
-msgstr "Utilisateur connecté"
-
-#: backoffice/management.py
-msgctxt "criticality-level"
-msgid "All"
-msgstr "Tous"
-
-#: backoffice/management.py fields/base.py fields/bool.py formdata.py
-#: statistics/views.py templates/wcs/backoffice/data-source.html workflows.py
-msgid "No"
-msgstr "Non"
-
 #: backoffice/management.py
 msgid "When nothing is checked the default settings will apply."
 msgstr "En l’absence de sélection le paramétrage par défaut s’applique."
@@ -4646,34 +4783,6 @@ msgstr ""
 msgid "Delete Custom View"
 msgstr "Supprimer la vue personnalisée"
 
-#: backoffice/management.py fields/base.py
-msgid "Number"
-msgstr "Numéro"
-
-#: backoffice/management.py
-msgid "Submission By"
-msgstr "Saisie par"
-
-#: backoffice/management.py
-msgid "Status (for user)"
-msgstr "Statut (visible à l’usager)"
-
-#: backoffice/management.py
-msgid "Anonymised"
-msgstr "Anonymisé"
-
-#: backoffice/management.py
-msgid "Start (modification time)"
-msgstr "Début (date de modification)"
-
-#: backoffice/management.py
-msgid "End (modification time)"
-msgstr "Fin (date de modification)"
-
-#: backoffice/management.py
-msgid "Distance"
-msgstr "Distance"
-
 #: backoffice/management.py
 #, python-format
 msgid ""
@@ -4967,15 +5076,6 @@ msgstr "supprimé"
 msgid "unset"
 msgstr "non définie"
 
-#: backoffice/management.py
-#, python-format
-msgid "%s of User"
-msgstr "%s de l’usager"
-
-#: backoffice/management.py
-msgid "User Label"
-msgstr "Nom de l’usager"
-
 #: backoffice/management.py
 msgid "Submissions by year"
 msgstr "Transmissions par année"
@@ -5217,7 +5317,8 @@ msgstr "Saisies entamées"
 msgid "Blocks of fields"
 msgstr "Blocs de champs"
 
-#: blocks.py data_sources.py formdef.py sql.py workflows.py
+#: blocks.py comment_templates.py data_sources.py formdef.py mail_templates.py
+#: sql.py workflows.py wscalls.py
 msgid "Automatic update"
 msgstr "Mise à jour automatique"
 
@@ -6735,6 +6836,10 @@ msgstr "Abandonner la saisie"
 msgid "leave this field blank to prove your humanity"
 msgstr "Laissez ce champ vide pour prouver votre humanité"
 
+#: forms/root.py
+msgid "and leave this field as prefilled by javascript"
+msgstr "et laissez ce champ prérempli automatiquement en l’était"
+
 #: forms/root.py
 #, python-format
 msgid "Value too long for field %(field)s: %(value)s (truncated)"
@@ -6779,8 +6884,8 @@ msgid "Technical error, please try again."
 msgstr "Erreur technique, veuillez réessayer."
 
 #: forms/root.py
-msgid "Honey pot should be left untouched."
-msgstr "Le pot de miel ne doit pas être touché."
+msgid "Honey pots should be left untouched."
+msgstr "Les pots de miel ne doivent pas être touchés."
 
 #: forms/root.py
 msgid "Technical error saving draft, please try again."
@@ -6790,6 +6895,10 @@ msgstr "Erreur technique à l’enregistrement du brouillon, veuillez réessayer
 msgid "Unexpected field error, please check."
 msgstr "Erreur inattendue sur un champ, veuillez vérifier ceux-ci."
 
+#: forms/root.py
+msgid "Change in test data"
+msgstr "Modification des données du formulaire"
+
 #: forms/root.py templates/wcs/formdata_sidebox.html
 msgid "Tracking code"
 msgstr "Code de suivi"
@@ -10020,6 +10129,10 @@ msgstr "non trouvé"
 msgid "There are no agendas."
 msgstr "Il n’y a pas d’agendas."
 
+#: templates/wcs/backoffice/includes/documentation-editor-link.html
+msgid "Edit documentation"
+msgstr "Modifier la documentation"
+
 #: templates/wcs/backoffice/includes/forms.html
 #, python-format
 msgid "Published from %(date1)s until %(date2)s"
@@ -10286,12 +10399,6 @@ msgstr "Comparer les sauvegardes"
 msgid "XML"
 msgstr "XML"
 
-#: templates/wcs/backoffice/snapshots_compare.html
-#: templates/wcs/backoffice/test-result.html
-#: templates/wcs/backoffice/test_sidebar.html
-msgid "Inspect"
-msgstr "Inspecteur"
-
 #: templates/wcs/backoffice/snapshots_compare.html
 msgid "Compare inspect"
 msgstr "Comparer les inspecteurs"
@@ -10425,6 +10532,10 @@ msgstr "Pas encore de résultats des tests."
 msgid "There are no test users yet."
 msgstr "Il n’y a pas encore d’utilisateurs de tests."
 
+#: templates/wcs/backoffice/test-webservice-responses.html
+msgid "Import from other test"
+msgstr "Importer depuis un autre test"
+
 #: templates/wcs/backoffice/test-webservice-responses.html
 #: wf/assign_carddata.py wf/create_formdata.py wf/redirect_to_url.py
 #: wf/roles.py workflow_tests.py
@@ -10453,10 +10564,6 @@ msgstr "frontoffice,backoffice"
 msgid "Switch to %(mode)s mode."
 msgstr "Passer en mode %(mode)s."
 
-#: templates/wcs/backoffice/test_edit_sidebar.html
-msgid "Mark test as failing"
-msgstr "Marquer le test comme devant échouer"
-
 #: templates/wcs/backoffice/test_edit_sidebar.html
 #, python-format
 msgid ""

wcs/mail_templates.py

@@ -34,7 +34,7 @@ class MailTemplate(XmlStorableObject):
 
     name = None
     slug = None
-    description = None
+    documentation = None
     subject = None
     body = None
     attachments = []
@@ -44,7 +44,8 @@ class MailTemplate(XmlStorableObject):
     XML_NODES = [
         ('name', 'str'),
         ('slug', 'str'),
-        ('description', 'str'),
+        ('description', 'str'),  # legacy
+        ('documentation', 'str'),
         ('subject', 'str'),
         ('body', 'str'),
         ('attachments', 'str_list'),
@@ -54,6 +55,16 @@ class MailTemplate(XmlStorableObject):
         XmlStorableObject.__init__(self)
         self.name = name
 
+    def migrate(self):
+        changed = False
+        if getattr(self, 'description', None):  # 2024-04-07
+            self.documentation = getattr(self, 'description')
+            self.description = None
+            changed = True
+        if changed:
+            self.store(comment=_('Automatic update'), snapshot_store_user=False)
+        return changed
+
     @property
     def category(self):
         return MailTemplateCategory.get(self.category_id, ignore_errors=True)
@@ -69,14 +80,16 @@ class MailTemplate(XmlStorableObject):
         base_url = get_publisher().get_backoffice_url()
         return '%s/workflows/mail-templates/%s/' % (base_url, self.id)
 
-    def store(self, comment=None, application=None, *args, **kwargs):
+    def store(self, comment=None, snapshot_store_user=True, application=None, *args, **kwargs):
         assert not self.is_readonly()
         if self.slug is None:
             # set slug if it's not yet there
             self.slug = self.get_new_slug()
         super().store(*args, **kwargs)
         if get_publisher().snapshot_class:
-            get_publisher().snapshot_class.snap(instance=self, comment=comment, application=application)
+            get_publisher().snapshot_class.snap(
+                instance=self, store_user=snapshot_store_user, comment=comment, application=application
+            )
 
     def get_places_of_use(self):
         from wcs.workflows import Workflow

wcs/publisher.py

@@ -82,6 +82,10 @@ class UnpicklerClass(pickle.Unpickler):
             ('wcs.qommon.storage', 'ElementIntersects'): 'wcs.sql',
             ('wcs.qommon.storage', 'Nothing'): 'wcs.sql',
             ('wcs.qommon.storage', 'Distance'): 'wcs.sql',
+            # filter field classes moved to their own file (2024-04-12)
+            ('wcs.backoffice.management', 'RelatedField'): 'wcs.backoffice.filter_fields',
+            ('wcs.backoffice.management', 'UserRelatedField'): 'wcs.backoffice.filter_fields',
+            ('wcs.backoffice.management', 'UserLabelRelatedField'): 'wcs.backoffice.filter_fields',
             # removed actions
             ('wcs.wf.redirect_to_status', 'RedirectToStatusWorkflowStatusItem'): 'NoLongerAvailableAction',
             ('wcs.workflows', 'RedirectToStatusWorkflowStatusItem'): 'NoLongerAvailableAction',
@@ -586,6 +590,7 @@ class WcsPublisher(QommonPublisher):
         from wcs.data_sources import NamedDataSource
         from wcs.formdef import FormDef
         from wcs.mail_templates import MailTemplate
+        from wcs.testdef import TestDef
         from wcs.workflows import Workflow
         from wcs.wscalls import NamedWsCall
 
@@ -605,6 +610,7 @@ class WcsPublisher(QommonPublisher):
             MailTemplateCategory,
             CommentTemplateCategory,
             DataSourceCategory,
+            TestDef,
         ):
             if klass.xml_root_node == object_type:
                 return klass

wcs/qommon/form.py

@@ -2681,49 +2681,49 @@ class WysiwygTextWidget(TextWidget):
     def get_plain_text_value(self):
         return misc.html2text(self.value)
 
+    def clean_html(self, value):
+        try:
+            from bleach.css_sanitizer import CSSSanitizer
+
+            css_sanitizer = CSSSanitizer(allowed_css_properties=self.ALL_STYLES)
+            kwargs = {
+                'css_sanitizer': css_sanitizer,
+            }
+        except ModuleNotFoundError:
+            # bleach < 5
+            kwargs = {'styles': self.ALL_STYLES}
+
+        cleaner = Cleaner(
+            tags=getattr(self, 'allowed_tags', None) or self.ALL_TAGS,
+            attributes=self.ALL_ATTRS,
+            strip=True,
+            strip_comments=False,
+            filters=[
+                partial(
+                    linkifier.LinkifyFilter,
+                    skip_tags=['pre'],
+                    parse_email=True,
+                    url_re=self.URL_RE,
+                    email_re=self.EMAIL_RE,
+                )
+            ],
+            **kwargs,
+        )
+        value = cleaner.clean(value).removeprefix('<br />').removesuffix('<br />')
+        if not strip_tags(value).strip() and not ('<img' in value or '<hr' in value):
+            value = ''
+        return value
+
     def _parse(self, request):
         TextWidget._parse(self, request, use_validation_function=False)
         if self.value:
-            all_tags = self.ALL_TAGS[:]
+            self.allowed_tags = self.ALL_TAGS[:]
             if get_publisher().get_site_option('ckeditor-allow-style-tag'):
-                all_tags.append('style')
+                self.allowed_tags.append('style')
             if get_publisher().get_site_option('ckeditor-allow-script-tag'):
-                all_tags.append('script')
+                self.allowed_tags.append('script')
 
-            try:
-                from bleach.css_sanitizer import CSSSanitizer
-
-                css_sanitizer = CSSSanitizer(allowed_css_properties=self.ALL_STYLES)
-                kwargs = {
-                    'css_sanitizer': css_sanitizer,
-                }
-            except ModuleNotFoundError:
-                # bleach < 5
-                kwargs = {'styles': self.ALL_STYLES}
-
-            cleaner = Cleaner(
-                tags=all_tags,
-                attributes=self.ALL_ATTRS,
-                strip=True,
-                strip_comments=False,
-                filters=[
-                    partial(
-                        linkifier.LinkifyFilter,
-                        skip_tags=['pre'],
-                        parse_email=True,
-                        url_re=self.URL_RE,
-                        email_re=self.EMAIL_RE,
-                    )
-                ],
-                **kwargs,
-            )
-            self.value = cleaner.clean(self.value)
-            if self.value.startswith('<br />'):
-                self.value = self.value[6:]
-            if self.value.endswith('<br />'):
-                self.value = self.value[:-6]
-            if not strip_tags(self.value).strip() and not ('<img' in self.value or '<hr' in self.value):
-                self.value = ''
+            self.value = self.clean_html(self.value)
 
             # unescape Django template tags
             def unquote_django(matchobj):

wcs/qommon/publisher.py

@@ -20,6 +20,7 @@ import collections
 import configparser
 import datetime
 import hashlib
+import html
 import inspect
 import io
 import json
@@ -833,8 +834,9 @@ class QommonPublisher(Publisher):
             attrs['data-max-bounds-lat2'], attrs['data-max-bounds-lng2'] = self.get_site_option(
                 'map-bounds-bottom-right'
             ).split(';')
-        attrs['data-map-attribution'] = self.get_site_option('map-attribution') or _(
-            'Map data &copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a>'
+        attrs['data-map-attribution'] = html.escape(
+            self.get_site_option('map-attribution')
+            or _('Map data &copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a>')
         )
         attrs['data-tile-urltemplate'] = (
             self.get_site_option('map-tile-urltemplate') or 'https://tiles.entrouvert.org/hdm/{z}/{x}/{y}.png'

wcs/qommon/static/css/dc2/admin.scss

@@ -3243,3 +3243,110 @@ div.dataview.compact-dataview {
 		display: flex;
 	}
 }
+
+.ro-documentation {
+	p:first-child {
+		margin-top: 0;
+	}
+	p:last-child {
+		margin-bottom: 0;
+	}
+}
+
+.bo-block.documentation {
+	position: relative;
+	margin-bottom: 0;
+	&.documentation-inspect-top {
+		margin-bottom: 1em;
+	}
+	button.save {
+		display: none;
+		position: absolute;
+		height: 2.5em;
+		bottom: -1.25em;
+		right: 0.5em;
+	}
+	&.active {
+		button.save {
+			display: block;
+		}
+		border-color: var(--primary-color);
+	}
+}
+
+.inspect-tabs .bo-block.documentation {
+	margin-top: 1em;
+	margin-bottom: 1em;
+}
+
+
+aside .bo-block.documentation {
+	// keep some space for godo toolbar
+	margin-top: 5em;
+}
+
+.focus-editor-link {
+	&::before {
+		font-family: FontAwesome;
+		content: "\f040";  // pencil
+	}
+}
+
+.godo.html-edition,
+.godo.html-edition--show {
+	--padding: 0.5em;
+	outline: none;
+	background: transparent;
+	padding-bottom: 0;
+	p:last-child {
+		margin-bottom: 0;
+	}
+	.godo--editor {
+		min-height: auto;
+		border: none;
+		padding: 0;
+	}
+}
+
+.godo.html-edition--show {
+	.godo--editor > :first-child {
+		padding-top: var(--padding);
+	}
+}
+
+.documentation-save-marks {
+	position: absolute;
+	right: 0.5em;
+	margin-top: -1.5em;
+	span {
+		visibility: hidden;
+		margin-left: -0.5em;
+	}
+}
+
+#appbar.field-edit {
+	margin-bottom: 0;
+}
+
+#appbar > h2 {
+	// always keep a bit of space, for documentation button
+	max-width: calc(100% - 80px);
+}
+
+.payload-preview {
+	&--structure {
+		font-family: monospace;
+		line-height: 150%;
+	}
+	&--template-value {
+		background: #ffc;
+	}
+	&--obj {
+		margin-left: 1em;
+		display: block;
+	}
+	&--item-separator::after {
+		content: '\a'; // force line-break
+		white-space: pre;
+	}
+}

wcs/qommon/static/js/qommon.admin.js

@@ -256,7 +256,7 @@ $(function() {
       $('#sidebar-toggle').click(function() {
         if ($('#sticky-sidebar').css('display') === 'none') {
           $('#sidebar').animate(
-              {'max-width': '23rem'},
+              {'max-width': '24rem'},
               400,
               function() {
                 $('#sticky-sidebar').show()
@@ -304,6 +304,28 @@ $(function() {
       $window.trigger('scroll');
     }
 
+  $('div.WidgetDict[data-widget-name*="post_data"] input[type="text"]').on('change', function() {
+        var $widget = $(this).parents('div.WidgetDict');
+        var url = '/api/preview-payload-structure?' + $('input', $widget).serialize();
+        var preview_button_id = 'payload-preview-button';
+        var preview_button_selector = 'a#' + preview_button_id;
+        if ($widget.find(preview_button_selector).length) {
+           $widget.find(preview_button_selector).attr('href', url);
+        } else {
+          if ($(this).parents('div.dict-key').length < 0) return;
+          if (!$(this).val().includes('/')) return;
+          var eval_link = document.createElement('a');
+          eval_link.setAttribute('rel', 'popup');
+          eval_link.setAttribute('href', url);
+          eval_link.setAttribute('id', preview_button_id);
+          eval_link.setAttribute('class', 'pk-button');
+          eval_link.setAttribute('data-selector', 'div.payload-preview');
+          eval_link.setAttribute('data-title-selector', 'h2');
+          eval_link.innerHTML = WCS_I18N.preview_payload_structure;
+          $widget.append(eval_link);
+        }
+    }).trigger('change');
+
     $('#inspect-test-tools form').on('submit', function() {
       var data = $(this).serialize();
       $.ajax({url: 'inspect-tool',
@@ -519,4 +541,81 @@ $(function() {
         el.parentNode.classList.toggle('display-codepoints')
       })
     )
+
+    const documentation_block = document.querySelector('.bo-block.documentation')
+    const editor = document.getElementById('documentation-editor')
+    const editor_link = document.querySelector('.focus-editor-link')
+    const title_byline = document.querySelector('.object--status-infos')
+    const documentation_save_button = document.querySelector('.bo-block.documentation button.save')
+    var clear_documentation_save_marks_timeout_id = null
+    if (editor_link) {
+      document.querySelector('#documentation-editor .godo--editor').setAttribute('contenteditable', 'false')
+
+      documentation_save_button.addEventListener('click', (e) => {
+          editor.sourceContent = editor.getHTML()
+          var documentation_message = Object()
+          documentation_message['content'] = editor.sourceContent.innerHTML
+          document.querySelector('.documentation-save-marks .mark-error').style.visibility = 'hidden'
+          document.querySelector('.documentation-save-marks .mark-success').style.visibility = 'hidden'
+          document.querySelector('.documentation-save-marks .mark-sent').style.visibility = 'visible'
+          fetch(`${window.location.pathname}update-documentation`, {
+            method: 'POST',
+            headers: {'Accept': 'application/json', 'Content-Type': 'application/json'},
+            body: JSON.stringify(documentation_message)
+          }).then((response) => {
+            if (! response.ok) {
+              return
+            }
+            return response.json()
+          }).then((json) => {
+            if (json && json.err == 0) {
+              if (json.changed) {
+                document.querySelector('.documentation-save-marks .mark-success').style.visibility = 'visible'
+              } else {
+                document.querySelector('.documentation-save-marks .mark-sent').style.visibility = 'hidden'
+                document.querySelector('.documentation-save-marks .mark-success').style.visibility = 'hidden'
+              }
+              if (json.empty) {
+                document.querySelector('.bo-block.documentation').setAttribute('hidden', 'hidden')
+              }
+            } else {
+              document.querySelector('.documentation-save-marks .mark-error').style.visibility = 'visible'
+            }
+            if (clear_documentation_save_marks_timeout_id) clearTimeout(clear_documentation_save_marks_timeout_id)
+            clear_documentation_save_marks_timeout_id = setTimeout(
+              function() {
+                document.querySelector('.documentation-save-marks .mark-error').style.visibility = 'hidden'
+                document.querySelector('.documentation-save-marks .mark-success').style.visibility = 'hidden'
+                document.querySelector('.documentation-save-marks .mark-sent').style.visibility = 'hidden'
+              }, 5000)
+          })
+        })
+
+      editor_link.addEventListener('click', (e) => {
+        e.preventDefault()
+        if (editor_link.getAttribute('aria-pressed') == 'true') {
+          editor.validEdition()
+          documentation_save_button.dispatchEvent(new Event('click'))
+          documentation_block.classList.remove('active')
+          document.querySelector('#documentation-editor .godo--editor').setAttribute('contenteditable', 'false')
+          editor_link.setAttribute('aria-pressed', false)
+          if (title_byline) title_byline.style.visibility = 'visible'
+        } else {
+          documentation_block.classList.add('active')
+          document.querySelector('.bo-block.documentation').removeAttribute('hidden')
+          if (document.querySelector('aside .bo-block.documentation')) {
+            document.getElementById('sidebar').style.display = 'block'
+            document.getElementById('sidebar').removeAttribute('hidden')
+            if (document.getElementById('sticky-sidebar').style.display == 'none') {
+              document.getElementById('sidebar-toggle').dispatchEvent(new Event('click'))
+            }
+          }
+          if (title_byline) title_byline.style.visibility = 'hidden'
+          editor_link.setAttribute('aria-pressed', true)
+          document.querySelector('#documentation-editor .godo--editor').setAttribute('contenteditable', 'true')
+          editor.showEdition()
+          editor.view.focus()
+        }
+      })
+    }
 });

wcs/qommon/static/js/qommon.wysiwyg.js

@@ -3,6 +3,10 @@
 CKEDITOR.env.origIsCompatible = CKEDITOR.env.isCompatible;
 CKEDITOR.env.isCompatible = true;
 
+/* do not turn all contenteditable into ckeditor, as some pages may have both
+ * godo and ckeditor */
+CKEDITOR.disableAutoInline = true;
+
 $(document).ready( function() {
   if (CKEDITOR.env.origIsCompatible == false) {
     /* bail out if ckeditor advertised itself as not supported */

wcs/snapshots.py

@@ -354,6 +354,10 @@ class Snapshot:
             if self.object_type in self._category_types:
                 # set position
                 instance.position = max(i.position or 0 for i in self.get_object_class().select()) + 1
+            elif self.object_type == 'testdef':
+                instance.workflow_tests.id = None
+                for response in instance.get_webservice_responses():
+                    response.id = None
             if hasattr(instance, 'disabled'):
                 instance.disabled = True
         else:

wcs/sql.py

@@ -3689,12 +3689,12 @@ class Snapshot(SqlMixin, wcs.snapshots.Snapshot):
     @classmethod
     def select_object_history(cls, obj, clause=None):
         return cls.select(
-            [Equal('object_type', obj.xml_root_node), Equal('object_id', obj.id)] + (clause or []),
+            [Equal('object_type', obj.xml_root_node), Equal('object_id', str(obj.id))] + (clause or []),
             order_by='-timestamp',
         )
 
     def is_from_object(self, obj):
-        return self.object_type == obj.xml_root_node and self.object_id == obj.id
+        return self.object_type == obj.xml_root_node and self.object_id == str(obj.id)
 
     @classmethod
     def _row2ob(cls, row, **kwargs):
@@ -3725,7 +3725,7 @@ class Snapshot(SqlMixin, wcs.snapshots.Snapshot):
         )
         cur.execute(
             sql_statement,
-            {'object_type': object_type, 'object_id': object_id, 'max_timestamp': max_timestamp},
+            {'object_type': object_type, 'object_id': str(object_id), 'max_timestamp': max_timestamp},
         )
         row = cur.fetchone()
         cur.close()

wcs/templates/wcs/backoffice.html

@@ -52,7 +52,7 @@
 
 {% block sidebar %}
   {% if sidebar or has_sidebar %}
-    <aside id="sidebar">
+    <aside id="sidebar" {% block sidebar-attrs %}{{ sidebar_attrs|default:"" }}{% endblock %}>
       <button id="sidebar-toggle" aria-label="{% trans "Toggle sidebar" %}">&#8286;</button>
       <div id="sticky-sidebar">
         {% block sidebar-content %}

wcs/templates/wcs/backoffice/block-inspect.html

@@ -5,6 +5,10 @@
 
 {% block content %}
 
+  {% if blockdef.documentation %}
+    <div class="bo-block documentation documentation-inspect-top"><div class="ro-documentation">{{ blockdef.documentation|safe }}</div></div>
+  {% endif %}
+
   <div class="pk-tabs inspect-tabs inspect-form-tabs">
     <div class="pk-tabs--tab-list" role="tablist">
       <button role="tab" aria-selected="true" aria-controls="inspect-infos" id="tab-infos" tabindex="0">{% trans "Information" %}</button>

wcs/templates/wcs/backoffice/carddef.html

@@ -17,6 +17,7 @@
 
   {{ publisher.get_request.session.display_message|safe }}
   {% include "wcs/backoffice/includes/sql-fields-integrity.html" %}
+  {% include "wcs/backoffice/includes/documentation.html" with element=formdef object=formdef %}
 
   <div class="bo-block">
     <h3>{% trans "Information" %}</h3>

wcs/templates/wcs/backoffice/comment-template.html

@@ -5,14 +5,13 @@
 
 {% block appbar-actions %}
   {% if not comment_template.is_readonly %}
+    {% include "wcs/backoffice/includes/documentation-editor-link.html" %}
     <a href="edit">{% trans "Edit" %}</a>
   {% endif %}
 {% endblock %}
 
 {% block content %}
-  {% if comment_template.description %}
-    <div class="bo-block">{{ comment_template.description }}</div>
-  {% endif %}
+  {% include "wcs/backoffice/includes/documentation.html" with element=comment_template object=comment_template %}
 
   {% if comment_template.comment %}
     <div class="section">

wcs/templates/wcs/backoffice/data-source.html

@@ -5,14 +5,13 @@
     <h2>{% trans "Data Source" %} - {{ datasource.name }}</h2>
     <span class="actions">
       {% if not datasource.is_readonly %}
+        {% include "wcs/backoffice/includes/documentation-editor-link.html" %}
         <a href="edit">{% trans "Edit" %}</a>
       {% endif %}
     </span>
   </div>
 
-  {% if datasource.description %}
-    <div class="bo-block">{{ datasource.description }}</div>
-  {% endif %}
+  {% include "wcs/backoffice/includes/documentation.html" with element=datasource object=datasource %}
 
   {% if datasource.data_source %}
     <div class="section">

wcs/templates/wcs/backoffice/formdef-inspect.html

@@ -5,6 +5,10 @@
 
 {% block content %}
 
+  {% if formdef.documentation %}
+    <div class="bo-block documentation documentation-inspect-top"><div class="ro-documentation">{{ formdef.documentation|safe }}</div></div>
+  {% endif %}
+
   <div class="pk-tabs inspect-tabs inspect-form-tabs">
     <div class="pk-tabs--tab-list" role="tablist">
       <button role="tab" aria-selected="true" aria-controls="inspect-infos" id="tab-infos" tabindex="0">{% trans "Information" %}</button>

wcs/templates/wcs/backoffice/formdef.html

@@ -16,6 +16,7 @@
       <li><a href="export">{% trans "Export" %}</a></li>
       <li><a href="delete" rel="popup">{% trans "Delete" %}</a></li>
     </ul>
+    {% include "wcs/backoffice/includes/documentation-editor-link.html" %}
     <a rel="popup" href="title">{% trans "change title" %}</a>
   {% endif %}
 {% endblock %}
@@ -35,7 +36,9 @@
   </div>
 
   {{ publisher.get_request.session.display_message|safe }}
+
   {% include "wcs/backoffice/includes/sql-fields-integrity.html" %}
+  {% include "wcs/backoffice/includes/documentation.html" with element=formdef object=formdef %}
 
   <div class="bo-block">
     <h3>{% trans "Information" %}</h3>

wcs/templates/wcs/backoffice/includes/documentation-editor-link.html

@@ -0,0 +1,2 @@
+{% load i18n %}
+<a class="focus-editor-link" title="{% trans "Edit documentation" %}"><span class="sr-only">{% trans "Edit documentation" %}</span></a>

wcs/templates/wcs/backoffice/includes/documentation.html

@@ -0,0 +1,21 @@
+{% load i18n %}
+<div class="bo-block documentation" {% if not element.documentation %}hidden{% endif %}>
+  {% if object.is_readonly %}
+    <div class="ro-documentation">{{ element.documentation|safe }}</div>
+  {% else %}
+    <script type="module" src="/static/xstatic/js/godo.js?{{version_hash}}"></script>
+    <div class="documentation-save-marks">
+      <span class="mark-error">✘</span>
+      <span class="mark-success">✔</span>
+      <span class="mark-sent">✔</span>
+    </div>
+    <div id="div-godo-source" >{{ element.documentation|default:"<p></p>"|safe }}</div>
+    <godo-editor
+      tabindex="0"
+      linked-source="div-godo-source"
+      heading-levels="3,4"
+      id="documentation-editor"
+    ></godo-editor>
+    <button class="save">{% trans "Save" %}</button>
+  {% endif %}
+</div>

wcs/templates/wcs/backoffice/includes/inspect-field.html

@@ -7,5 +7,8 @@
       {% if field.key == 'block' %}</a>{% endif %}
     </span>
   </h4>
+  {% if field.documentation %}
+    <div class="bo-block documentation"><div class="ro-documentation">{{ field.documentation|safe }}</div></div>
+  {% endif %}
   {{ field.get_parameters_view|safe }}
 </div>

wcs/templates/wcs/backoffice/mail-template.html

@@ -5,14 +5,13 @@
 
 {% block appbar-actions %}
   {% if not mail_template.is_readonly %}
+    {% include "wcs/backoffice/includes/documentation-editor-link.html" %}
     <a href="edit">{% trans "Edit" %}</a>
   {% endif %}
 {% endblock %}
 
 {% block content %}
-  {% if mail_template.description %}
-    <div class="bo-block">{{ mail_template.description }}</div>
-  {% endif %}
+  {% include "wcs/backoffice/includes/documentation.html" with element=mail_template object=mail_template %}
 
   {% if mail_template.subject and mail_template.body %}
     <div class="section">

wcs/templates/wcs/backoffice/test-users.html

@@ -6,6 +6,8 @@
 {% block sidebar-content %}
   <h3>{% trans "Actions" %}</h3>
   <a class="button button-paragraph" href="new" rel="popup">{% trans "New" %}</a>
+  <a class="button button-paragraph" href="import" rel="popup">{% trans "Import" %}</a>
+  <a class="button button-paragraph" href="export">{% trans "Export" %}</a>
 {% endblock %}
 
 {% block body %}

wcs/templates/wcs/backoffice/test-webservice-responses.html

@@ -6,6 +6,7 @@
 {% block sidebar-content %}
   <h3>{% trans "Actions" %}</h3>
   <a class="button button-paragraph" href="new" rel="popup">{% trans "New" %}</a>
+  <a class="button button-paragraph" href="import" rel="popup">{% trans "Import from other test" %}</a>
 {% endblock %}
 
 {% block body %}
@@ -20,8 +21,10 @@
                 <i>({% trans "not configured" %})</i>
               {% endif %}
             </a>
-            <a rel="popup" class="delete" href="{{ response.id }}/delete">{% trans "Remove" %}</a>
-            <a class="link-action-icon duplicate" href="{{ response.id }}/duplicate">{% trans "Duplicate" %}</a>
+            {% if not testdef.is_readonly %}
+              <a rel="popup" class="delete" href="{{ response.id }}/delete">{% trans "Remove" %}</a>
+              <a class="link-action-icon duplicate" href="{{ response.id }}/duplicate">{% trans "Duplicate" %}</a>
+            {% endif %}
           </li>
         {% endfor %}
       </ul>

wcs/templates/wcs/backoffice/test_sidebar.html

@@ -10,6 +10,7 @@
 <h3>{% trans "Navigation" %}</h3>
 <ul class="sidebar--buttons">
   <li><a class="button button-paragraph" rel="popup" href="edit">{% trans "Options" %}</a></li>
+  <li><a class="button button-paragraph" href="history/">{% trans "History" %}</a></li>
   <li><a class="button button-paragraph" href="webservice-responses/">{% trans "Webservice responses" %}</a></li>
   <li><a class="button button-paragraph" href="inspect">{% trans "Inspect" %}</a></li>
 </ul>

wcs/templates/wcs/backoffice/workflow-action.html

@@ -0,0 +1,29 @@
+{% extends "wcs/backoffice.html" %}
+{% load i18n %}
+
+{% block appbar-title %}{{ action.description }}{% endblock %}
+
+{% block appbar-actions %}
+  {% if not workflow.is_readonly %}
+    {% include "wcs/backoffice/includes/documentation-editor-link.html" %}
+  {% endif %}
+{% endblock %}
+
+{% block content %}
+  {{ block.super }}
+
+  {{ html_form.render|safe }}
+
+  {% if action.support_substitution_variables %}
+    {{ get_substitution_html_table|safe }}
+  {% endif %}
+
+{% endblock %}
+
+{% block sidebar-attrs %}
+  {% if not action.documentation %}style="display: none"{% endif %}
+{% endblock %}
+
+{% block sidebar-content %}
+  {% include "wcs/backoffice/includes/documentation.html" with element=action object=workflow %}
+{% endblock %}

wcs/templates/wcs/backoffice/workflow-global-action.html

@@ -11,12 +11,15 @@
     {% endif %}
   </ul>
   {% if not workflow.is_readonly %}
+    {% include "wcs/backoffice/includes/documentation-editor-link.html" %}
     <a rel="popup" href="options">{% trans "Options" %}</a>
     <a rel="popup" href="edit">{% trans "Change Name" %}</a>
   {% endif %}
 {% endblock %}
 
 {% block body %}
+  {% include "wcs/backoffice/includes/documentation.html" with element=action object=workflow %}
+
   <div class="bo-block">
     <h2>{% trans "Actions" %}</h2>
     {% if not action.items %}

wcs/templates/wcs/backoffice/workflow-inspect.html

@@ -5,6 +5,9 @@
 
 {% block content %}
 
+  {% if workflow.documentation %}
+    <div class="bo-block documentation documentation-inspect-top"><div class="ro-documentation">{{ workflow.documentation|safe }}</div></div>
+  {% endif %}
   <div class="pk-tabs inspect-tabs inspect-workflow-tabs">
     <div class="pk-tabs--tab-list" role="tablist">
       <button role="tab" aria-selected="true" aria-controls="inspect-statuses" id="tab-statuses" tabindex="0">{% trans "Statuses" %}</button>
@@ -43,6 +46,10 @@
               ><a href="{{ workflow.get_admin_url }}status/{{ status.id }}/" class="inspect-status--link">
                 <span class="inspect-status--colour" style="background-color: {{ status.colour|default:"white" }}"></span>
                 {{ status.name }}</a></h3>
+
+              {% if status.documentation %}
+                <div class="bo-block documentation"><div class="ro-documentation">{{ status.documentation|safe }}</div></div>
+              {% endif %}
               {% if status.is_endpoint %}
                 <p>
                   {% if status.forced_endpoint %}
@@ -55,6 +62,9 @@
               {% if status.backoffice_info_text %}<div>{{ status.backoffice_info_text|safe }}</div>{% endif %}
               {% for item in status.items %}
                 <h4><a href="{{ workflow.get_admin_url }}status/{{ status.id }}/items/{{ item.id }}/">{{ item.description }}</a></h4>
+                {% if item.documentation %}
+                  <div class="bo-block documentation"><div class="ro-documentation">{{ item.documentation|safe }}</div></div>
+                {% endif %}
                 {{ item.get_parameters_view|safe }}
               {% empty %}
                 <p>{% trans "No actions in this status." %}</p>
@@ -74,6 +84,9 @@
           {% for action in workflow.global_actions %}
             <div class="section global-action">
               <h3><a id="action-{{ action.id }}" href="{{ workflow.get_admin_url }}global-actions/{{ action.id }}/">{{ action.name }}</a></h3>
+              {% if action.documentation %}
+                <div class="bo-block documentation"><div class="ro-documentation">{{ action.documentation|safe }}</div></div>
+              {% endif %}
               <h4>{% trans "Triggers" %}</h4>
               <ul>
                 {% for trigger in action.triggers %}

wcs/templates/wcs/backoffice/workflow-status.html

@@ -11,6 +11,7 @@
     {% endif %}
   </ul>
   {% if not workflow.is_readonly %}
+    {% include "wcs/backoffice/includes/documentation-editor-link.html" %}
     <a href="options">{% trans "Options" %}</a>
     <a rel="popup" href="edit">{% trans "Change Name" %}</a>
   {% endif %}
@@ -19,6 +20,8 @@
 {% block content %}
   {{ block.super }}
 
+  {% include "wcs/backoffice/includes/documentation.html" with element=status object=workflow %}
+
   {% with visibility_mode=status.get_visibility_mode %}
     {% if visibility_mode != 'all' %}
       <div class="bo-block">

wcs/templates/wcs/backoffice/workflow-tests.html

@@ -29,12 +29,14 @@
               <a href="{{ action.id }}/" rel="popup" title="{% trans "Edit" %}">{% trans "Edit" %}</a>
             </span>
           {% endif %}
-          <span class="duplicate">
-            <a href="{{ action.id }}/duplicate" title="{% trans "Duplicate" %}">{% trans "Duplicate" %}</a>
-          </span>
-          <span class="remove">
-            <a href="{{ action.id }}/delete" rel="popup" title="{% trans "Delete" %}">{% trans "Delete" %}</a>
-          </span>
+          {% if not testdef.is_readonly %}
+            <span class="duplicate">
+              <a href="{{ action.id }}/duplicate" title="{% trans "Duplicate" %}">{% trans "Duplicate" %}</a>
+            </span>
+            <span class="remove">
+              <a href="{{ action.id }}/delete" rel="popup" title="{% trans "Delete" %}">{% trans "Delete" %}</a>
+            </span>
+          {% endif %}
         </p>
       </li>
     {% endfor %}

wcs/templates/wcs/backoffice/workflow.html

@@ -12,6 +12,7 @@
     {% endif %}
   </ul>
   {% if not workflow.is_readonly %}
+    {% include "wcs/backoffice/includes/documentation-editor-link.html" %}
     <a rel="popup" href="category">{% trans "change category" %}</a>
     <a rel="popup" href="edit">{% trans "change title" %}</a>
   {% endif %}
@@ -22,6 +23,8 @@
     {{ view.last_modification_block|safe }}
   </div>
 
+  {% include "wcs/backoffice/includes/documentation.html" with element=workflow object=workflow %}
+
   <div class="splitcontent-left">
     <div class="bo-block">
       <h3>{% trans "Possible Status" %}

wcs/templates/wcs/backoffice/wscall.html

@@ -5,14 +5,13 @@
     <h2>{% trans "Webservice Call" %} - {{ wscall.name }}</h2>
     {% if not wscall.is_readonly %}
       <span class="actions">
+        {% include "wcs/backoffice/includes/documentation-editor-link.html" %}
         <a href="edit">{% trans "Edit" %}</a>
       </span>
     {% endif %}
   </div>
 
-  {% if wscall.description %}
-    <div class="bo-block">{{ wscall.description }}</div>
-  {% endif %}
+  {% include "wcs/backoffice/includes/documentation.html" with element=wscall object=wscall %}
 
   <div class="bo-block">
     <h3>{% trans "Parameters" %}</h3>

wcs/testdef.py

@@ -33,8 +33,10 @@ from quixote import get_publisher, get_session_manager
 from urllib3 import HTTPResponse
 
 from wcs import sql
+from wcs.carddef import CardDef
 from wcs.compat import CompatHTTPRequest
 from wcs.fields import Field, PageField
+from wcs.formdef import FormDef
 from wcs.qommon.form import FileWithPreviewWidget, Form, get_selection_error_text
 from wcs.qommon.storage import Equal
 from wcs.qommon.template import TemplateError
@@ -76,7 +78,7 @@ class TestDefXmlProxy(XmlStorableObject):
             'boolean': 'bool',
             'jsonb': 'jsonb',
         }
-        excluded_fields = ['id', 'object_type', 'object_id']
+        excluded_fields = ['id']
         extra_fields = [
             ('_webservice_responses', 'webservice_responses'),
             ('workflow_tests', 'workflow_tests'),
@@ -89,26 +91,32 @@ class TestDefXmlProxy(XmlStorableObject):
         ] + extra_fields
 
     def export_jsonb_to_xml(self, element, attribute_name, **kwargs):
-        element.text = json.dumps(getattr(self, attribute_name))
+        element.text = json.dumps(getattr(self, attribute_name), indent=2)
 
     def import_jsonb_from_xml(self, element, **kwargs):
         return json.loads(element.text)
 
-    def export_workflow_tests_to_xml(self, element, attribute_name, **kwargs):
-        for subelement in self.workflow_tests.export_to_xml():
+    def export_workflow_tests_to_xml(self, element, attribute_name, include_id=False):
+        workflow_tests = self.workflow_tests.export_to_xml(include_id=include_id)
+        if include_id:
+            element.set('id', workflow_tests.get('id'))
+
+        for subelement in workflow_tests:
             element.append(subelement)
 
-    def import_workflow_tests_from_xml(self, element, **kwargs):
+    def import_workflow_tests_from_xml(self, element, include_id=False):
         from wcs.workflow_tests import WorkflowTests
 
-        return WorkflowTests.import_from_xml_tree(element)
+        return WorkflowTests.import_from_xml_tree(element, include_id=include_id)
 
-    def export_webservice_responses_to_xml(self, element, attribute_name, **kwargs):
+    def export_webservice_responses_to_xml(self, element, attribute_name, include_id=False):
         for response in self._webservice_responses:
-            element.append(response.export_to_xml())
+            element.append(response.export_to_xml(include_id=include_id))
 
-    def import_webservice_responses_from_xml(self, element, **kwargs):
-        return [WebserviceResponse.import_from_xml_tree(response) for response in element]
+    def import_webservice_responses_from_xml(self, element, include_id=False):
+        return [
+            WebserviceResponse.import_from_xml_tree(response, include_id=include_id) for response in element
+        ]
 
 
 class TestDef(sql.TestDef):
@@ -134,6 +142,11 @@ class TestDef(sql.TestDef):
         'tablerows',
         'ranked-items',
     )
+    backoffice_class = 'wcs.admin.tests.TestPage'
+
+    xml_root_node = TestDefXmlProxy.xml_root_node
+    get_table_name = TestDefXmlProxy.get_table_name
+    is_readonly = TestDefXmlProxy.is_readonly
 
     def __str__(self):
         return self.name
@@ -146,15 +159,18 @@ class TestDef(sql.TestDef):
             return self._workflow_tests
 
         workflow_tests_list = WorkflowTests.select([Equal('testdef_id', self.id)])
-        self._workflow_tests = workflow_tests_list[0] if workflow_tests_list else WorkflowTests()
-        self._workflow_tests.testdef = self
+        self.workflow_tests = workflow_tests_list[0] if workflow_tests_list else WorkflowTests()
         return self._workflow_tests
 
     @workflow_tests.setter
     def workflow_tests(self, value):
         self._workflow_tests = value
+        self._workflow_tests.testdef = self
 
     def get_webservice_responses(self):
+        if hasattr(self, '_webservice_responses'):
+            # this attribute is set by import/export, and should be used in snapshot context
+            return self._webservice_responses
         return WebserviceResponse.select([Equal('testdef_id', self.id)], order_by='name')
 
     def get_admin_url(self):
@@ -162,13 +178,27 @@ class TestDef(sql.TestDef):
         objects_dir = 'forms' if self.object_type == 'formdefs' else 'cards'
         return '%s/%s/%s/tests/%s/' % (base_url, objects_dir, self.object_id, self.id)
 
-    def store(self, *args, **kwargs):
-        super().store(*args, **kwargs)
+    def store(self, comment=None):
+        super().store()
 
         self.workflow_tests.testdef_id = self.id
-        self.workflow_tests.testdef = self
         self.workflow_tests.store()
 
+        if hasattr(self, '_webservice_responses'):
+            # first store after import, attach webservice responses and delete old ones on snapshot restore
+            response_ids = {x.id for x in self._webservice_responses}
+            for response in WebserviceResponse.select([Equal('testdef_id', self.id)]):
+                if response.id not in response_ids:
+                    response.remove_self()
+
+            for response in self._webservice_responses:
+                response.testdef_id = self.id
+                response.store()
+            del self._webservice_responses
+
+        if get_publisher().snapshot_class:
+            get_publisher().snapshot_class.snap(instance=self, comment=comment)
+
     @classmethod
     def remove_object(cls, id):
         super().remove_object(id)
@@ -538,11 +568,12 @@ class TestDef(sql.TestDef):
                     return widget
 
     def export_to_xml(self, include_id=False):
-        self._webservice_responses = self.get_webservice_responses()
-
-        testdef_xml = TestDefXmlProxy()
+        testdef_xml = TestDefXmlProxy(id=str(self.id))
         for field, dummy in TestDefXmlProxy.XML_NODES:  # pylint: disable=not-an-iterable
-            setattr(testdef_xml, field, getattr(self, field))
+            if field == '_webservice_responses':
+                testdef_xml._webservice_responses = self.get_webservice_responses()
+            else:
+                setattr(testdef_xml, field, getattr(self, field))
 
         return testdef_xml.export_to_xml(include_id=include_id)
 
@@ -555,20 +586,23 @@ class TestDef(sql.TestDef):
         return cls.import_from_xml_tree(tree, formdef, include_id=include_id)
 
     @classmethod
-    def import_from_xml_tree(cls, tree, formdef, include_id=False):
-        testdef = TestDef.create_from_formdata(formdef, formdef.data_class()())
-
+    def import_from_xml_tree(cls, tree, formdef=None, include_id=False, **kwargs):
         testdef_xml = TestDefXmlProxy.import_from_xml_tree(tree, include_id)
+
+        if not formdef:
+            klass = FormDef if testdef_xml.object_type == 'formdefs' else CardDef
+            formdef = klass.get(testdef_xml.object_id)
+
+        testdef = TestDef.create_from_formdata(formdef, formdef.data_class()())
+        testdef.id = int(testdef_xml.id) if testdef_xml.id else None
+
         for field, dummy in TestDefXmlProxy.XML_NODES:  # pylint: disable=not-an-iterable
+            if field in ('object_type', 'object_id'):
+                continue
+
             if hasattr(testdef_xml, field):
                 setattr(testdef, field, getattr(testdef_xml, field))
 
-        testdef.store()
-
-        for response in testdef._webservice_responses:
-            response.testdef_id = testdef.id
-            response.store()
-
         return testdef
 
 

wcs/users.py

@@ -22,7 +22,7 @@ import urllib.parse
 from quixote import get_publisher
 
 from wcs.api_utils import get_secret_and_orig, sign_url
-from wcs.sql_criterias import ArrayContains, Equal, Intersects, Not, Null, Or
+from wcs.sql_criterias import ArrayContains, Contains, Equal, Intersects, Not, Null, Or
 
 from .qommon import _, get_cfg
 from .qommon.misc import get_formatted_phone, http_post_request, simplify
@@ -307,7 +307,7 @@ class User(StorableObject):
 
         raise AttributeError()
 
-    def get_json_export_dict(self, full=False):
+    def get_json_export_dict(self, full=False, include_roles=False):
         data = {
             'id': self.id,
             'name': self.display_name,
@@ -316,12 +316,38 @@ class User(StorableObject):
             data['email'] = self.email
         if self.name_identifiers:
             data['NameID'] = self.name_identifiers
+        if self.test_uuid:
+            data['test_uuid'] = self.test_uuid
         if full:
             formdef = self.get_formdef()
             if formdef:
                 data.update({f.varname: self.form_data.get(f.id) for f in formdef.fields if f.varname})
+        if include_roles:
+            data['roles'] = [x.slug for x in get_publisher().role_class.select([Contains('id', self.roles)])]
         return data
 
+    @classmethod
+    def import_from_json(cls, data):
+        user = cls()
+        user.name = data['name']
+        user.email = data.get('email')
+        user.name_identifiers = data.get('NameID')
+        user.test_uuid = data['test_uuid']
+
+        formdef = user.get_formdef()
+        if formdef:
+            user.form_data = {}
+            for f in formdef.fields:
+                if f.varname and f.varname in data:
+                    user.form_data[f.id] = data[f.varname]
+
+        if data.get('roles'):
+            criterias = [Equal('slug', slug) for slug in data['roles']]
+            roles = get_publisher().role_class.select([Or(criterias)], order_by='name')
+            user.roles = [x.id for x in roles]
+
+        return user
+
     def set_deleted(self):
         self.deleted_timestamp = datetime.datetime.now()
         self.store()

wcs/views.py

@@ -90,6 +90,7 @@ def i18n_js(request):
         'close': _('Close'),
         'email_domain_suggest': _('Did you want to write'),
         'email_domain_fix': _('Apply fix'),
+        'preview_payload_structure': _('Preview payload structure'),
     }
     return HttpResponse(
         'WCS_I18N = %s;\n' % json.dumps(strings, cls=misc.JSONEncoder), content_type='application/javascript'

wcs/wf/dispatch.py

@@ -372,5 +372,8 @@ class DispatchWorkflowStatusItem(WorkflowStatusItem):
                     roles.remove(new_role_id)
             formdata.store()
 
+    def get_workflow_test_action(self, *args, **kwargs):
+        return self
+
 
 register_item_class(DispatchWorkflowStatusItem)

wcs/wf/form.py

@@ -118,6 +118,7 @@ class WorkflowFormFieldsFormDef(FormDef):
 class WorkflowFormFieldDefPage(FieldDefPage):
     section = 'workflows'
     blacklisted_attributes = ['display_locations', 'anonymise']
+    is_documentable = False
 
     def get_deletion_extra_warning(self):
         return None

wcs/wf/wscall.py

@@ -35,7 +35,13 @@ from wcs.workflows import (
     WorkflowStatusItem,
     register_item_class,
 )
-from wcs.wscalls import PayloadError, call_webservice, get_app_error_code, record_wscall_error
+from wcs.wscalls import (
+    PayloadError,
+    UnflattenKeysException,
+    call_webservice,
+    get_app_error_code,
+    record_wscall_error,
+)
 
 from ..qommon import _, force_str, pgettext
 from ..qommon.errors import ConnectionError
@@ -298,6 +304,15 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
                         ]
                     ),
                 },
+                hint=_(
+                    'The / in parameter name allows to generate complex objects. '
+                    'Thus a parameter named "element/child" containing "value" will generate the payload '
+                    '"element": {"child": "value"}. If the subkey, i.e. "child", is an integer it will '
+                    'become a list index and two elements "element/0", "element/1" (indexes should '
+                    ' start from zero) containing "value1" and "value2" will generate the payload '
+                    '"element": ["value1", "value2"]. It is possible to combine the two types, for '
+                    'example "element/0/key1" to generate a list of objects.'
+                ),
             )
 
         response_types = collections.OrderedDict([('json', _('JSON')), ('attachment', _('Attachment'))])
@@ -481,6 +496,17 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
                 exception=e,
             )
             return
+        except UnflattenKeysException as e:
+            get_publisher().record_error(
+                error_summary=e.get_summary(),
+                exception=e,
+                context='[WSCALL]',
+                formdata=formdata,
+                status_item=self,
+                notify=self.notify_on_errors,
+                record=self.record_on_errors,
+            )
+            return
 
         app_error_code = get_app_error_code(response, data, self.response_type)
         app_error_code_header = response.headers.get('x-error-code')

wcs/workflows.py

@@ -714,8 +714,9 @@ class WorkflowVariablesFieldsFormDef(FormDef):
         self.workflow = workflow
         if self.workflow.is_readonly():
             self.readonly = True
-        if workflow.variables_formdef and workflow.variables_formdef.fields:
-            self.fields = self.workflow.variables_formdef.fields
+        if workflow.variables_formdef:
+            self.documentation = workflow.variables_formdef.documentation
+            self.fields = self.workflow.variables_formdef.fields or []
         else:
             self.fields = []
 
@@ -768,8 +769,9 @@ class WorkflowBackofficeFieldsFormDef(FormDef):
     def __init__(self, workflow):
         self.id = None
         self.workflow = workflow
-        if workflow.backoffice_fields_formdef and workflow.backoffice_fields_formdef.fields:
-            self.fields = self.workflow.backoffice_fields_formdef.fields
+        if workflow.backoffice_fields_formdef:
+            self.documentation = workflow.backoffice_fields_formdef.documentation
+            self.fields = self.workflow.backoffice_fields_formdef.fields or []
         else:
             self.fields = []
 
@@ -804,6 +806,7 @@ class Workflow(StorableObject):
 
     name = None
     slug = None
+    documentation = None
     possible_status = None
     roles = None
     variables_formdef = None
@@ -1189,14 +1192,6 @@ class Workflow(StorableObject):
     def get_not_endpoint_status(self):
         return [x for x in self.possible_status if not x.is_endpoint()]
 
-    def has_options(self):
-        for status in self.possible_status:
-            for item in status.items:
-                for parameter in item.get_parameters():
-                    if not getattr(item, parameter):
-                        return True
-        return False
-
     def remove_self(self):
         for form in self.formdefs():
             form.workflow_id = None
@@ -1207,9 +1202,10 @@ class Workflow(StorableObject):
         root = ET.Element('workflow')
         if include_id and self.id and not str(self.id).startswith('_'):
             root.attrib['id'] = str(self.id)
-        ET.SubElement(root, 'name').text = self.name
-        if self.slug:
-            ET.SubElement(root, 'slug').text = self.slug
+        for attr in ('name', 'slug', 'documentation'):
+            value = getattr(self, attr, None)
+            if value:
+                ET.SubElement(root, attr).text = value
 
         WorkflowCategory.object_category_xml_export(self, root, include_id=include_id)
 
@@ -1238,6 +1234,8 @@ class Workflow(StorableObject):
             variables = ET.SubElement(root, 'variables')
             formdef = ET.SubElement(variables, 'formdef')
             ET.SubElement(formdef, 'name').text = '-'  # required by formdef xml import
+            if self.variables_formdef.documentation:
+                ET.SubElement(formdef, 'documentation').text = self.variables_formdef.documentation
             fields = ET.SubElement(formdef, 'fields')
             for field in self.variables_formdef.fields:
                 fields.append(field.export_to_xml(include_id=include_id))
@@ -1246,6 +1244,8 @@ class Workflow(StorableObject):
             variables = ET.SubElement(root, 'backoffice-fields')
             formdef = ET.SubElement(variables, 'formdef')
             ET.SubElement(formdef, 'name').text = '-'  # required by formdef xml import
+            if self.backoffice_fields_formdef.documentation:
+                ET.SubElement(formdef, 'documentation').text = self.backoffice_fields_formdef.documentation
             fields = ET.SubElement(formdef, 'fields')
             for field in self.backoffice_fields_formdef.fields:
                 fields.append(field.export_to_xml(include_id=include_id))
@@ -1297,8 +1297,9 @@ class Workflow(StorableObject):
 
         workflow.name = xml_node_text(tree.find('name'))
 
-        if tree.find('slug') is not None:
-            workflow.slug = xml_node_text(tree.find('slug'))
+        for attribute in ('slug', 'documentation'):
+            if tree.find(attribute) is not None:
+                setattr(workflow, attribute, xml_node_text(tree.find(attribute)))
 
         WorkflowCategory.object_category_xml_import(workflow, tree, include_id=include_id)
 
@@ -1365,6 +1366,7 @@ class Workflow(StorableObject):
                 raise WorkflowImportError(e.msg, details=e.details)
             else:
                 workflow.variables_formdef = WorkflowVariablesFieldsFormDef(workflow=workflow)
+                workflow.variables_formdef.documentation = imported_formdef.documentation
                 workflow.variables_formdef.fields = imported_formdef.fields
 
         variables = tree.find('backoffice-fields')
@@ -1381,6 +1383,7 @@ class Workflow(StorableObject):
                 raise WorkflowImportError(e.msg, details=e.details)
             else:
                 workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow=workflow)
+                workflow.backoffice_fields_formdef.documentation = imported_formdef.documentation
                 workflow.backoffice_fields_formdef.fields = imported_formdef.fields
 
         if unknown_referenced_objects_details:
@@ -1585,7 +1588,7 @@ class XmlSerialisable:
             node.attrib['type'] = self.key
         if include_id and getattr(self, 'id', None):
             node.attrib['id'] = self.id
-        for attribute in self.get_parameters():
+        for attribute in self.get_parameters() + ('documentation',):
             if getattr(self, '%s_export_to_xml' % attribute, None):
                 getattr(self, '%s_export_to_xml' % attribute)(node, include_id=include_id)
                 continue
@@ -1611,7 +1614,7 @@ class XmlSerialisable:
     def init_with_xml(self, elem, include_id=False, snapshot=False, check_datasources=True):
         if include_id and elem.attrib.get('id'):
             self.id = elem.attrib.get('id')
-        for attribute in self.get_parameters():
+        for attribute in self.get_parameters() + ('documentation',):
             el = elem.find(attribute)
             if getattr(self, '%s_init_with_xml' % attribute, None):
                 getattr(self, '%s_init_with_xml' % attribute)(el, include_id=include_id, snapshot=snapshot)
@@ -2513,6 +2516,7 @@ class WorkflowGlobalAction(SerieOfActionsMixin):
     name = None
     triggers = None
     backoffice_info_text = None
+    documentation = None
 
     def __init__(self, name=None):
         self.name = name
@@ -2575,11 +2579,11 @@ class WorkflowGlobalAction(SerieOfActionsMixin):
 
     def export_to_xml(self, include_id=False):
         status = ET.Element('action')
-        ET.SubElement(status, 'id').text = self.id
-        ET.SubElement(status, 'name').text = self.name
 
-        if self.backoffice_info_text:
-            ET.SubElement(status, 'backoffice_info_text').text = self.backoffice_info_text
+        for attr in ('id', 'name', 'backoffice_info_text', 'documentation'):
+            value = getattr(self, attr, None)
+            if value:
+                ET.SubElement(status, attr).text = str(value)
 
         items = ET.SubElement(status, 'items')
         for item in self.items:
@@ -2592,10 +2596,10 @@ class WorkflowGlobalAction(SerieOfActionsMixin):
         return status
 
     def init_with_xml(self, elem, include_id=False, snapshot=False):
-        self.id = xml_node_text(elem.find('id'))
-        self.name = xml_node_text(elem.find('name'))
-        if elem.find('backoffice_info_text') is not None:
-            self.backoffice_info_text = xml_node_text(elem.find('backoffice_info_text'))
+        for attr in ('id', 'name', 'backoffice_info_text', 'documentation'):
+            node = elem.find(attr)
+            if node is not None:
+                setattr(self, attr, xml_node_text(node))
 
         self.items = []
         for item in elem.find('items'):
@@ -2689,6 +2693,7 @@ class WorkflowStatus(SerieOfActionsMixin):
     forced_endpoint = False
     colour = '#FFFFFF'
     backoffice_info_text = None
+    documentation = None
     extra_css_class = ''
     loop_items_template = None
     after_loop_status = None
@@ -2948,18 +2953,24 @@ class WorkflowStatus(SerieOfActionsMixin):
 
     def export_to_xml(self, include_id=False):
         status = ET.Element('status')
-        ET.SubElement(status, 'id').text = str(self.id)
-        ET.SubElement(status, 'name').text = self.name
-        ET.SubElement(status, 'colour').text = self.colour
-        if self.extra_css_class:
-            ET.SubElement(status, 'extra_css_class').text = self.extra_css_class
+
+        for attr in (
+            'id',
+            'name',
+            'colour',
+            'extra_css_class',
+            'backoffice_info_text',
+            'loop_items_template',
+            'after_loop_status',
+            'documentation',
+        ):
+            value = getattr(self, attr, None)
+            if value:
+                ET.SubElement(status, attr).text = str(value)
 
         if self.forced_endpoint:
             ET.SubElement(status, 'forced_endpoint').text = 'true'
 
-        if self.backoffice_info_text:
-            ET.SubElement(status, 'backoffice_info_text').text = self.backoffice_info_text
-
         visibility_node = ET.SubElement(status, 'visibility')
         for role in self.visibility or []:
             ET.SubElement(visibility_node, 'role').text = str(role)
@@ -2968,28 +2979,25 @@ class WorkflowStatus(SerieOfActionsMixin):
         for item in self.items:
             items.append(item.export_to_xml(include_id=include_id))
 
-        if self.loop_items_template:
-            ET.SubElement(status, 'loop_items_template').text = self.loop_items_template
-        if self.after_loop_status:
-            ET.SubElement(status, 'after_loop_status').text = self.after_loop_status
-
         return status
 
     def init_with_xml(self, elem, include_id=False, snapshot=False, check_datasources=True):
-        self.id = xml_node_text(elem.find('id'))
-        self.name = xml_node_text(elem.find('name'))
-        if elem.find('colour') is not None:
-            self.colour = xml_node_text(elem.find('colour'))
-        if elem.find('extra_css_class') is not None:
-            self.extra_css_class = xml_node_text(elem.find('extra_css_class'))
+        for attr in (
+            'id',
+            'name',
+            'colour',
+            'extra_css_class',
+            'backoffice_info_text',
+            'loop_items_template',
+            'after_loop_status',
+            'documentation',
+        ):
+            node = elem.find(attr)
+            if node is not None:
+                setattr(self, attr, xml_node_text(node))
+
         if elem.find('forced_endpoint') is not None:
             self.forced_endpoint = elem.find('forced_endpoint').text == 'true'
-        if elem.find('backoffice_info_text') is not None:
-            self.backoffice_info_text = xml_node_text(elem.find('backoffice_info_text'))
-        if elem.find('loop_items_template') is not None:
-            self.loop_items_template = xml_node_text(elem.find('loop_items_template'))
-        if elem.find('after_loop_status') is not None:
-            self.after_loop_status = xml_node_text(elem.find('after_loop_status'))
 
         self.visibility = []
         for visibility_role in elem.findall('visibility/role'):
@@ -3047,6 +3055,7 @@ class WorkflowStatusItem(XmlSerialisable):
     category = None  # (key, label)
     id = None
     condition = None
+    documentation = None
 
     endpoint = True  # means it's not possible to interact, and/or cause a status change
     waitpoint = False  # means it's possible to wait (user interaction, or other event)

wcs/wscalls.py

@@ -17,6 +17,7 @@
 import collections
 import hashlib
 import json
+import re
 import urllib.parse
 import xml.etree.ElementTree as ET
 
@@ -52,6 +53,86 @@ class PayloadError(Exception):
     pass
 
 
+class UnflattenKeysException(Exception):
+    def get_summary(self):
+        return _('Webservice call failure because unable to unflatten payload keys (%s)') % self
+
+
+def unflatten_keys(d):
+    """Transform:
+
+       {"a/b/0/x": "1234"}
+
+    into:
+
+       {"a": {"b": [{"x": "1234"}]}}
+    """
+    if not isinstance(d, dict) or not d:  # unflattening an empty dict has no sense
+        return d
+
+    def split_key(key):
+        def map_key(x):
+            if misc.is_ascii_digit(x):
+                return int(x)
+            elif isinstance(x, str):
+                # allow / char escaping
+                return x.replace('//', '/')
+            return x
+
+        # split key by single / only
+        return [map_key(x) for x in re.split(r'(?<!/)/(?!/)', key)]
+
+    keys = [(split_key(key), key) for key in d]
+    try:
+        keys.sort()
+    except TypeError:
+        # sorting fail means that there is a mix between lists and dicts
+        raise UnflattenKeysException(_('there is a mix between lists and dicts'))
+
+    def set_path(path, orig_key, d, value, i=0):
+        assert path
+
+        key, tail = path[i], path[i + 1 :]
+
+        if not tail:  # end of path, set the value
+            if isinstance(key, int):
+                assert isinstance(d, list)
+                if len(d) != key:
+                    raise UnflattenKeysException(_('incomplete array before key "%s"') % orig_key)
+                d.append(value)
+            else:
+                assert isinstance(d, dict)
+                d[key] = value
+            return  # end of recursion
+
+        new = [] if isinstance(tail[0], int) else {}
+
+        if isinstance(key, int):
+            assert isinstance(d, list)
+            if len(d) < key:
+                raise UnflattenKeysException(
+                    _('incomplete array before %s in %s')
+                    % (('/'.join([str(x) for x in path[: i + 1]])), orig_key)
+                )
+            if len(d) == key:
+                d.append(new)
+            else:
+                new = d[key]
+        else:
+            new = d.setdefault(key, new)
+        set_path(path, orig_key, new, value, i + 1)
+
+    # Is the first level an array (ie key is like "0/param") or a dict (key is like "param/0") ?
+    if isinstance(keys[0][0][0], int):
+        new = []
+    else:
+        new = {}
+    for path, key in keys:
+        value = d[key]
+        set_path(path, key, new, value)
+    return new
+
+
 def get_app_error_code(response, data, response_type):
     app_error_code = 0
     app_error_code_header = response.headers.get('x-error-code')
@@ -167,6 +248,7 @@ def call_webservice(
                 else:
                     if payload[key]:
                         payload[key] = get_publisher().get_cached_complex_data(payload[key])
+        payload = unflatten_keys(payload)
 
     # if formdata has to be sent, it's the payload. If post_data exists,
     # it's added in formdata['extra']
@@ -250,7 +332,7 @@ class NamedWsCall(XmlStorableObject):
 
     name = None
     slug = None
-    description = None
+    documentation = None
     request = None
     notify_on_errors = False
     record_on_errors = False
@@ -261,7 +343,8 @@ class NamedWsCall(XmlStorableObject):
     XML_NODES = [
         ('name', 'str'),
         ('slug', 'str'),
-        ('description', 'str'),
+        ('description', 'str'),  # legacy
+        ('documentation', 'str'),
         ('request', 'request'),
         ('notify_on_errors', 'bool'),
         ('record_on_errors', 'bool'),
@@ -271,6 +354,16 @@ class NamedWsCall(XmlStorableObject):
         XmlStorableObject.__init__(self)
         self.name = name
 
+    def migrate(self):
+        changed = False
+        if getattr(self, 'description', None):  # 2024-04-07
+            self.documentation = getattr(self, 'description')
+            self.description = None
+            changed = True
+        if changed:
+            self.store(comment=_('Automatic update'), snapshot_store_user=False)
+        return changed
+
     def get_admin_url(self):
         base_url = get_publisher().get_backoffice_url()
         return '%s/settings/wscalls/%s/' % (base_url, self.slug)
@@ -331,7 +424,7 @@ class NamedWsCall(XmlStorableObject):
         request['post_formdata'] = bool(element.find('post_formdata') is not None)
         return request
 
-    def store(self, comment=None, application=None, *args, **kwargs):
+    def store(self, comment=None, snapshot_store_user=True, application=None, *args, **kwargs):
         assert not self.is_readonly()
         if self.slug is None:
             # set slug if it's not yet there
@@ -341,7 +434,9 @@ class NamedWsCall(XmlStorableObject):
         self.id = self.slug
         super().store(*args, **kwargs)
         if get_publisher().snapshot_class:
-            get_publisher().snapshot_class.snap(instance=self, comment=comment, application=application)
+            get_publisher().snapshot_class.snap(
+                instance=self, comment=comment, store_user=snapshot_store_user, application=application
+            )
 
     @classmethod
     def get_substitution_variables(cls):
@@ -353,13 +448,22 @@ class NamedWsCall(XmlStorableObject):
         if getattr(get_request(), 'disable_error_notifications', None) is True:
             notify_on_errors = False
             record_on_errors = False
-        data = call_webservice(
-            cache=True,
-            notify_on_errors=notify_on_errors,
-            record_on_errors=record_on_errors,
-            **(self.request or {}),
-        )[2]
-        return json.loads(force_str(data))
+        try:
+            data = call_webservice(
+                cache=True,
+                notify_on_errors=notify_on_errors,
+                record_on_errors=record_on_errors,
+                **(self.request or {}),
+            )[2]
+            return json.loads(force_str(data))
+        except UnflattenKeysException as e:
+            get_publisher().record_error(
+                error_summary=e.get_summary(),
+                exception=e,
+                context='[WSCALL]',
+                notify=notify_on_errors,
+                record=record_on_errors,
+            )
 
 
 class WsCallsSubstitutionProxy:
@@ -431,6 +535,15 @@ class WsCallRequestWidget(CompositeWidget):
                 'data-dynamic-display-child-of': method_widget.get_name(),
                 'data-dynamic-display-value': methods.get('POST'),
             },
+            hint=_(
+                'The / in parameter name allows to generate complex objects. '
+                'Thus a parameter named "element/child" containing "value" will generate the payload '
+                '"element": {"child": "value"}. If the subkey, i.e. "child", is an integer it will '
+                'become a list index and two elements "element/0", "element/1" (indexes should '
+                ' start from zero) containing "value1" and "value2" will generate the payload '
+                '"element": ["value1", "value2"]. It is possible to combine the two types, for '
+                'example "element/0/key1" to generate a list of objects.'
+            ),
         )
 
         self.add(