saml: always retry user creation when detecting duplicates on sso (#75777)

(only when it's done in the frontoffice, record as agent when in
backoffice)

tests/admin_pages/test_i18n.py

@@ -30,7 +30,7 @@ def pub():
     pub.cfg['language'] = {'language': 'en', 'multilinguism': True, 'languages': ['en', 'fr']}
     pub.write_cfg()
 
-    TranslatableMessage.do_table()
+    TranslatableMessage.do_table()  # update table with selected languages
 
     return pub
 
@@ -67,6 +67,9 @@ def test_i18n_page(pub):
     action2.triggers = []
     workflow.store()
 
+    workflow2 = Workflow(name='second workflow')
+    workflow2.add_status('Other Status')
+
     formdef = FormDef()
     formdef.name = 'test title'
     formdef.fields = [
@@ -127,11 +130,31 @@ def test_i18n_page(pub):
     # check table
     assert resp.pyquery('tr').length == TranslatableMessage.count()
 
-    # check form filter
+    # check filters
     assert resp.form['lang'].value == 'fr'
+    assert [x[2] for x in resp.form['formdef'].options] == [
+        'All forms and card models',
+        'test title',
+        'card test',
+    ]
+    resp.form['formdef'] = 'cards/1'
+    resp = resp.form.submit()
+    assert resp.pyquery('tr').length == 1
+    assert {x.text for x in resp.pyquery('tr td:first-child')} == {'card test'}
+
+    # check filtering on a formdef/carddef outputs related workflow strings
+    resp.form['formdef'] = 'forms/1'
+    resp = resp.form.submit()
+    assert resp.pyquery('tr').length == 12
+    assert 'test title' in {x.text for x in resp.pyquery('tr td:first-child')}
+    assert 'Global Manual' in {x.text for x in resp.pyquery('tr td:first-child')}
+    assert 'second workflow' not in {x.text for x in resp.pyquery('tr td:first-child')}
+
+    resp.form['formdef'] = ''
     resp.form['q'] = 'Email'
     resp = resp.form.submit()
     assert resp.pyquery('tr').length == 2  # (email subject, email body)
+    assert {x.text for x in resp.pyquery('tr td:first-child')} == {'Email body', 'Email Subject'}
 
     # translate a message
     resp = resp.click('edit', index=0)
@@ -297,7 +320,7 @@ def test_i18n_import(pub):
     )
     resp = resp.form.submit('submit').follow()
     resp = resp.click('Go to multilinguism')
-    assert resp.request.url == 'http://example.net/backoffice/i18n/?q=list&lang=fr'
+    assert resp.request.url == 'http://example.net/backoffice/i18n/?q=list&formdef=&lang=fr'
 
     # invalid file
     resp = app.get('/backoffice/i18n/')

tests/admin_pages/test_tests.py

@@ -710,7 +710,7 @@ def test_tests_manual_run(pub):
     assert 'Started by: Manual run.' in resp.text
     assert len(resp.pyquery('tr')) == 1
     assert 'Success!' in resp.text
-    assert 'Missing required fields: 0' in resp.text
+    assert 'Display details' not in resp.text
 
     resp = resp.click('First test')
     assert 'Edit data' in resp.text
@@ -722,7 +722,7 @@ def test_tests_manual_run(pub):
     assert len(resp.pyquery('span.test-failure')) == 0
 
     # add required field
-    formdef.fields.append(fields.StringField(id='2', label='String', varname='string'))
+    formdef.fields.append(fields.StringField(id='2', label='String field', varname='string'))
     formdef.store()
 
     resp = app.get('/backoffice/forms/1/tests/')  # run from test listing page
@@ -737,7 +737,9 @@ def test_tests_manual_run(pub):
     resp = resp.click('#%s' % result.id)
     assert 'Started by: Manual run.' in resp.text
     assert 'Success!' in resp.text
-    assert 'Missing required fields: 1' in resp.text
+
+    resp = resp.click('Display details')
+    assert 'String field' in resp.text
 
     # add validation to first field
     formdef.fields[0].validation = {'type': 'digits'}
@@ -767,6 +769,40 @@ def test_tests_manual_run(pub):
     app.get('/backoffice/forms/1/tests/results/42/', status=404)
 
 
+def test_tests_result_recorded_errors(pub):
+    create_superuser(pub)
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.fields = [
+        fields.ComputedField(
+            id='1',
+            label='Computed',
+            varname='computed',
+            value_template='{{ forms|objects:"test-title"|filter_by:"unknown"|filter_value:"xxx"|count }}',
+            freeze_on_initial_value=True,
+        ),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.store()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.name = 'First test'
+    testdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/1/tests/results/run').follow()
+    resp = resp.click('Display details')
+
+    assert 'Missing required fields' not in resp.text
+    assert 'Recorded errors:' in resp.text
+    assert escape('Invalid filter "unknown"') in resp.text
+
+
 def test_tests_run_order(pub):
     create_superuser(pub)
 

tests/admin_pages/test_workflow.py

@@ -74,6 +74,19 @@ def test_workflows_default(pub):
     assert 'Change Status Name' not in resp.text
     assert 'Delete' not in resp.text
 
+    # check status type icons/labels
+    resp = app.get('/backoffice/workflows/_default/')
+    assert [
+        (PyQuery(x).text(), re.search(r'status-type-[a-z]+', x.attrib['class']).group(0))
+        for x in resp.pyquery('#status-list li')
+    ] == [
+        ('Just Submitted (transition status)', 'status-type-transition'),
+        ('New (pause status)', 'status-type-waitpoint'),
+        ('Rejected (final status)', 'status-type-endpoint'),
+        ('Accepted (pause status)', 'status-type-waitpoint'),
+        ('Finished (final status)', 'status-type-endpoint'),
+    ]
+
 
 def test_workflows_new(pub):
     create_superuser(pub)
@@ -2353,6 +2366,29 @@ def test_workflows_backoffice_fields_backlinks_to_actions(pub):
     }
 
 
+def test_workflows_backoffice_fields_with_same_label(pub):
+    create_superuser(pub)
+
+    Workflow.wipe()
+    workflow = Workflow(name='foo')
+    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
+    workflow.backoffice_fields_formdef.fields = [
+        fields.StringField(id='bo1', varname='bo1', label='variable'),
+        fields.StringField(id='bo2', varname='bo2', label='variable'),
+    ]
+    status = workflow.add_status(name='baz')
+    action1 = status.add_action('set-backoffice-fields')
+    workflow.store()
+
+    app = login(get_app(pub))
+    resp = app.get(action1.get_admin_url())
+    assert resp.form['fields$element0$field_id'].options == [
+        ('', False, ''),
+        ('bo1', False, 'variable - Text (line) (bo1)'),
+        ('bo2', False, 'variable - Text (line) (bo2)'),
+    ]
+
+
 def test_workflows_fields_labels(pub):
     create_superuser(pub)
 
@@ -3141,11 +3177,62 @@ def test_workflows_create_formdata_deleted_field(pub):
     resp = app.get('/backoffice/workflows/%s/status/%s/items/_create_formdata/' % (wf.id, st2.id))
     assert resp.form['mappings$element1$field_id'].options == [
         ('', False, '---'),
-        ('1', False, 'string2'),
+        ('1', False, 'string2 - Text (line)'),
         ('0', True, '❗ string1 (deleted field)'),
     ]
 
 
+def test_workflows_create_formdata_fields_with_same_label(pub):
+    create_superuser(pub)
+
+    BlockDef.wipe()
+    block = BlockDef()
+    block.name = 'Test Block'
+    block.fields = [fields.StringField(id='123', required=True, label='Test')]
+    block.store()
+
+    FormDef.wipe()
+    target_formdef = FormDef()
+    target_formdef.name = 'target form'
+    target_formdef.enable_tracking_codes = True
+    target_formdef.fields = [
+        fields.StringField(id='0', label='string1', varname='foo'),
+        fields.StringField(id='1', label='string1', varname='bar'),
+        fields.BlockField(id='2', label='block1', varname='foo2', block_slug=block.slug),
+        fields.BlockField(id='3', label='block1', varname='bar2', block_slug=block.slug),
+        fields.BlockField(id='4', label='block2', varname='xxx', block_slug=block.slug),
+    ]
+    target_formdef.store()
+
+    Workflow.wipe()
+    wf = Workflow(name='create-formdata')
+
+    st2 = wf.add_status('Resubmit')
+
+    create_formdata = st2.add_action('create_formdata', id='_create_formdata')
+    create_formdata.formdef_slug = target_formdef.url_name
+    create_formdata.mappings = [
+        Mapping(field_id='0', expression='{{ "a" }}'),
+        Mapping(field_id='1', expression='{{ "b" }}'),
+    ]
+
+    wf.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/workflows/%s/status/%s/items/_create_formdata/' % (wf.id, st2.id))
+    assert resp.form['mappings$element1$field_id'].options == [
+        ('', False, '---'),
+        ('0', False, 'string1 - Text (line) (foo)'),
+        ('1', True, 'string1 - Text (line) (bar)'),
+        ('2', False, 'block1 - Field Block (Test Block) (foo2)'),
+        ('2$123', False, 'block1 (foo2) - Test - Text (line)'),
+        ('3', False, 'block1 - Field Block (Test Block) (bar2)'),
+        ('3$123', False, 'block1 (bar2) - Test - Text (line)'),
+        ('4', False, 'block2 - Field Block (Test Block)'),
+        ('4$123', False, 'block2 - Test - Text (line)'),
+    ]
+
+
 def test_workflows_create_carddata_action_config(pub):
     create_superuser(pub)
 
@@ -4212,3 +4299,32 @@ def test_workflows_edit_aggregationemail_action(pub):
     app = login(get_app(pub))
     resp = app.get(item.get_admin_url())
     assert '_submitter' not in [x[0] for x in resp.form['to$element0'].options]
+
+
+def test_workflows_function_and_role_with_same_name(pub):
+    create_superuser(pub)
+    pub.role_class.wipe()
+    role1 = pub.role_class(name='Foo')
+    role1.store()
+    role2 = pub.role_class(name='Foobar')
+    role2.store()
+    Workflow.wipe()
+
+    workflow = Workflow(name='foo')
+    workflow.roles = {'_receiver': 'Receiver', '_foobar': 'Foobar'}
+    st1 = workflow.add_status(name='baz')
+    commentable = st1.add_action('commentable', id='_commentable')
+    workflow.store()
+
+    app = login(get_app(pub))
+    resp = app.get(commentable.get_admin_url())
+    assert resp.form['by$element0'].options == [
+        ('None', True, '---'),
+        ('_submitter', False, 'User'),
+        ('_receiver', False, 'Receiver'),
+        ('_foobar', False, 'Foobar'),
+        ('logged-users', False, 'Logged Users'),
+        ('', False, '----'),
+        (str(role1.id), False, 'Foo'),
+        (str(role2.id), False, 'Foobar [role]'),  # same name as function -> role suffix
+    ]

tests/api/test_formdata.py

@@ -2697,7 +2697,7 @@ def test_api_access_restrict_to_anonymised_data(pub, local_user, auth):
                 sign_uri(url, user=local_user, orig=access.access_identifier, key=access.access_key), **kwargs
             )
 
-    resp = get_url('/api/forms/test/list?full=on')
+    resp = get_url('/api/forms/test/list?full=on&order_by=id')
     assert len(resp.json) == 10
     assert resp.json[0]['fields']['foobar'] == 'FOO BAR1'
     assert resp.json[0]['fields']['foobar2'] == 'FOO BAR 2'
@@ -2715,7 +2715,7 @@ def test_api_access_restrict_to_anonymised_data(pub, local_user, auth):
     access.restrict_to_anonymised_data = True
     access.store()
 
-    resp = get_url('/api/forms/test/list?full=on')
+    resp = get_url('/api/forms/test/list?full=on&order_by=id')
     assert len(resp.json) == 10
     assert 'foobar' not in resp.json[0]['fields']
     assert resp.json[0]['fields']['foobar2'] == 'FOO BAR 2'

tests/api/test_statistics.py

@@ -1584,6 +1584,67 @@ def test_statistics_multiple_forms_count(pub, formdef):
     assert resp.json['data']['series'][0]['data'] == []
 
 
+def test_statistics_multiple_forms_count_different_ids(pub):
+    data_source = {
+        'type': 'jsonvalue',
+        'value': json.dumps(
+            [{'id': 'foo', 'text': 'Foo'}, {'id': 'bar', 'text': 'Bar'}, {'id': 'baz', 'text': 'Baz'}]
+        ),
+    }
+
+    formdef1 = FormDef()
+    formdef1.name = 'xxx'
+    formdef1.fields = [
+        fields.ItemField(
+            id='1',
+            varname='test-item',
+            label='Test item',
+            data_source=data_source,
+            display_locations=['statistics'],
+        ),
+    ]
+    formdef1.store()
+
+    formdef2 = FormDef()
+    formdef2.name = 'yyy'
+    formdef2.fields = [
+        fields.ItemField(
+            id='2',
+            varname='test-item',
+            label='Test item',
+            data_source=data_source,
+            display_locations=['statistics'],
+        ),
+    ]
+    formdef2.store()
+
+    formdata = formdef1.data_class()()
+    formdata.data['1'] = 'foo'
+    formdata.data['1_display'] = 'Foo'
+    formdata.just_created()
+    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.store()
+
+    formdata = formdef2.data_class()()
+    formdata.data['2'] = 'baz'
+    formdata.data['2_display'] = 'Baz'
+    formdata.just_created()
+    formdata.receipt_time = datetime.datetime(2021, 3, 1, 2, 0).timetuple()
+    formdata.store()
+
+    url = '/api/statistics/forms/count/?form=%s&form=%s' % (formdef1.url_name, formdef2.url_name)
+    resp = get_app(pub).get(sign_uri(url))
+    assert resp.json['data']['series'] == [{'data': [1, 0, 1], 'label': 'Forms Count'}]
+    assert resp.json['data']['x_labels'] == ['2021-01', '2021-02', '2021-03']
+
+    # group by item field
+    resp = get_app(pub).get(sign_uri(url + '&group-by=test-item'))
+    assert resp.json['data']['x_labels'] == ['2021-01', '2021-02', '2021-03']
+    assert len(resp.json['data']['series']) == 2
+    assert {'data': [1, None, None], 'label': 'Foo'} in resp.json['data']['series']
+    assert {'data': [None, None, 1], 'label': 'Baz'} in resp.json['data']['series']
+
+
 def test_statistics_multiple_forms_count_subfilters(pub, formdef):
     category_a = Category(name='Category A')
     category_a.store()

tests/backoffice_pages/test_all.py

@@ -24,6 +24,7 @@ from wcs.qommon.ident.password_accounts import PasswordAccount
 from wcs.qommon.upload_storage import PicklableUpload
 from wcs.roles import logged_users_role
 from wcs.sql_criterias import Contains
+from wcs.wf.comment import WorkflowCommentPart
 from wcs.wf.create_formdata import Mapping
 from wcs.wf.form import WorkflowFormEvolutionPart, WorkflowFormFieldsFormDef
 from wcs.wf.register_comment import JournalEvolutionPart
@@ -537,7 +538,6 @@ def test_backoffice_listing_fts(pub):
     assert resp.text.count('data-link') == 17
     resp = app.get('/backoffice/management/form-title/')
     resp.forms['listing-settings']['filter'] = 'all'
-    resp.forms['listing-settings']['q'] = 'foo'
     resp.forms['listing-settings']['limit'] = '100'
     resp = resp.forms['listing-settings'].submit()
     assert resp.pyquery('tbody tr').length == 50
@@ -545,9 +545,29 @@ def test_backoffice_listing_fts(pub):
         '%s-%s' % (formdef.id, i) for i in range(50, 0, -1)
     ]
 
+    # search on text (foo is on all formdata so it gets the same set of results, but ordered differently)
+    resp.forms['listing-settings']['q'] = 'foo'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.pyquery('tbody tr').length == 50
+    assert {x.text for x in resp.pyquery('tbody tr .cell-id a')} == {
+        '%s-%s' % (formdef.id, i) for i in range(50, 0, -1)
+    }  # same set
+    assert [x.text for x in resp.pyquery('tbody tr .cell-id a')] != [
+        '%s-%s' % (formdef.id, i) for i in range(50, 0, -1)
+    ]  # but different order
+    # get first row, check it has b'foo' in its item field
+    formdata = formdef.data_class().get(resp.pyquery('tbody tr .cell-id a')[0].attrib['href'].strip('/'))
+    assert formdata.data[formdef.fields[1].id] == 'foo'
+
     resp.forms['listing-settings']['q'] = 'baz'
     resp = resp.forms['listing-settings'].submit()
     assert resp.pyquery('tbody tr').length == 24
+    results = [x.text for x in resp.pyquery('tbody tr .cell-id a')]
+    # force order, check it's same set but different order
+    resp.forms['listing-settings']['order_by'] = '-receipt_time'
+    resp = resp.forms['listing-settings'].submit()
+    assert {x.text for x in resp.pyquery('tbody tr .cell-id a')} == set(results)
+    assert [x.text for x in resp.pyquery('tbody tr .cell-id a')] != results
 
 
 def test_backoffice_legacy_urls(pub):
@@ -5422,6 +5442,7 @@ def test_backoffice_create_formdata_backoffice_submission(pub, create_formdata):
     assert target_formdata.submission_agent_id == str(create_formdata['admin'].id)
     assert target_formdata.user.id == user.id
     assert target_formdata.status == 'draft'
+    assert target_formdata.receipt_time
     assert resp.location == 'http://example.net/backoffice/management/target-form/%s/' % target_formdata.id
     resp = resp.follow()
     assert resp.location == 'http://example.net/backoffice/submission/target-form/%s/' % target_formdata.id
@@ -6284,3 +6305,33 @@ def test_status_visibility(pub):
     wf.store()
     resp = app.get(formdata.get_backoffice_url())
     assert resp.pyquery('.status').text() == 'st1 st2 st3'
+
+
+def test_backoffice_form_tracking_code_workflow_action(pub):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [fields.StringField(id='0', label='string')]
+    formdef.enable_tracking_codes = True
+    formdef.store()
+    formdef.data_class().wipe()
+
+    # as user
+    resp = get_app(pub).get('/test/')
+    resp.forms[0]['f0'] = 'foobar'
+    resp = resp.forms[0].submit('submit')  # -> validation
+    resp = resp.forms[0].submit('submit')  # -> done
+    formdata = formdef.data_class().select()[0]
+
+    user = create_user(pub)
+    resp = login(get_app(pub)).get('/backoffice/management/').follow()
+    resp.forms[0]['query'] = formdata.tracking_code
+    resp = resp.forms[0].submit()
+    resp = resp.follow()
+    resp.forms['wf-actions']['comment'] = 'Test comment'
+    resp = resp.forms['wf-actions'].submit('button_commentable')
+
+    # check action has been recorded as agent
+    formdata.refresh_from_storage()
+    assert isinstance(formdata.evolution[-1].parts[0], WorkflowCommentPart)
+    assert formdata.evolution[-1].who == str(user.id)

tests/backoffice_pages/test_carddata.py

@@ -1186,6 +1186,118 @@ def test_backoffice_cards_update_data_from_json(pub):
     assert [x for x in card.iter_evolution_parts(ContentSnapshotPart)][-1].user_id == user.id
 
 
+def test_backoffice_cards_update_data_from_json_custom_id(pub):
+    user = create_user(pub)
+
+    workflow = CardDef.get_default_workflow()
+    workflow.id = None
+    workflow.add_status('status2', 'st2')
+    workflow.store()
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'test'
+    carddef.fields = [
+        fields.StringField(id='1', label='Test', varname='string'),
+        fields.StringField(id='2', label='Custom id', varname='custom_id'),
+    ]
+    carddef.id_template = '{{form_var_custom_id}}'
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.workflow_id = workflow.id
+    carddef.backoffice_submission_roles = user.roles
+    carddef.store()
+    carddef.data_class().wipe()
+
+    card = carddef.data_class()()
+    card.data = {'1': 'plop', '2': 'test'}
+    card.just_created()
+    card.store()
+
+    app = login(get_app(pub))
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Export Data')
+    resp.form['format'] = 'json'
+    resp = resp.form.submit('submit')
+    job_id = urllib.parse.parse_qs(urllib.parse.urlparse(resp.location).query)['job'][0]
+    job = AfterJob.get(job_id)
+    json_export = json.loads(job.file_content)
+    assert len(json_export['data']) == 1
+
+    # update
+    del json_export['data'][0]['uuid']  # ignore uuid
+    json_export['data'][0]['fields']['string'] = 'plop 2'
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Import data from a file')
+    resp.forms[0]['file'] = Upload('test.json', json.dumps(json_export).encode(), 'application/json')
+    resp.form['update_existing_cards'].checked = True
+    resp = resp.forms[0].submit()
+    assert '/backoffice/processing?job=' in resp.location
+    resp = resp.follow()
+
+    assert carddef.data_class().count() == 1
+    card.refresh_from_storage()
+    assert card.data == {'1': 'plop 2', '2': 'test'}
+
+    # no id and no uuid, but an existing id will be computed, -> update
+    json_export['data'][0]['id'] = None
+    json_export['data'][0]['uuid'] = None
+    json_export['data'][0]['fields']['string'] = 'plop 3'
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Import data from a file')
+    resp.forms[0]['file'] = Upload('test.json', json.dumps(json_export).encode(), 'application/json')
+    resp.form['update_existing_cards'].checked = True
+    resp = resp.forms[0].submit()
+    assert '/backoffice/processing?job=' in resp.location
+    resp = resp.follow()
+    assert carddef.data_class().count() == 1
+    card.refresh_from_storage()
+    assert card.data == {'1': 'plop 3', '2': 'test'}
+
+    # different id -> create new one, and id will then be updated according to template
+    json_export['data'][0]['id'] = 'hello'
+    json_export['data'][0]['fields']['custom_id'] = 'he'  # doesn't match
+    json_export['data'][0]['fields']['string'] = 'plop 4'
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Import data from a file')
+    resp.forms[0]['file'] = Upload('test.json', json.dumps(json_export).encode(), 'application/json')
+    resp.form['update_existing_cards'].checked = True
+    resp = resp.forms[0].submit()
+    assert '/backoffice/processing?job=' in resp.location
+    resp = resp.follow()
+    assert carddef.data_class().count() == 2
+    assert carddef.data_class().get_by_id('he').data == {'1': 'plop 4', '2': 'he'}
+
+    # asked not to update, but same id, it should be skipped
+    json_export['data'][0]['id'] = str(card.id)
+    json_export['data'][0]['fields']['string'] = 'plop 6'
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Import data from a file')
+    resp.forms[0]['file'] = Upload('test.json', json.dumps(json_export).encode(), 'application/json')
+    resp.form['update_existing_cards'].checked = False
+    resp = resp.forms[0].submit()
+    assert '/backoffice/processing?job=' in resp.location
+    resp = resp.follow()
+    assert carddef.data_class().count() == 2
+    card.refresh_from_storage()
+    assert card.data == {'1': 'plop 3', '2': 'test'}
+
+    # asked not to update, but same id would be computed, it should be skipped
+    json_export['data'][0]['id'] = None
+    json_export['data'][0]['uuid'] = None
+    json_export['data'][0]['fields']['string'] = 'plop 7'
+    json_export['data'][0]['fields']['custom_id'] = 'test'
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Import data from a file')
+    resp.forms[0]['file'] = Upload('test.json', json.dumps(json_export).encode(), 'application/json')
+    resp.form['update_existing_cards'].checked = False
+    resp = resp.forms[0].submit()
+    assert '/backoffice/processing?job=' in resp.location
+    resp = resp.follow()
+    assert carddef.data_class().count() == 2
+    card.refresh_from_storage()
+    assert card.data == {'1': 'plop 3', '2': 'test'}
+
+
 def test_backoffice_cards_wscall_failure_display(http_requests, pub):
     user = create_user(pub)
 

tests/backoffice_pages/test_custom_view.py

@@ -1173,21 +1173,25 @@ def test_backoffice_custom_view_sort_field(pub):
             items=['foo', 'bar', 'baz'],
             display_locations=['validation', 'summary', 'listings'],
         ),
+        fields.StringField(
+            id='2',
+            label='field 2',
+        ),
     ]
     formdef.workflow_roles = {'_receiver': 1}
     formdef.store()
 
     formdef.data_class().wipe()
     formdata = formdef.data_class()()
-    formdata.data = {'1': 'foo', '1_display': 'foo'}
+    formdata.data = {'1': 'foo', '1_display': 'foo', '2': 'foo foo'}
     formdata.jump_status('new')
     formdata.store()
     formdata = formdef.data_class()()
-    formdata.data = {'1': 'bar', '1_display': 'bar'}
+    formdata.data = {'1': 'bar', '1_display': 'bar', '2': 'foo foo'}
     formdata.jump_status('new')
     formdata.store()
     formdata = formdef.data_class()()
-    formdata.data = {'1': 'baz', '1_display': 'baz'}
+    formdata.data = {'1': 'baz', '1_display': 'baz', '2': 'foo'}
     formdata.jump_status('new')
     formdata.store()
 
@@ -1220,6 +1224,19 @@ def test_backoffice_custom_view_sort_field(pub):
     resp = app.get('/backoffice/management/form-title/shared-custom-test-view/')
     assert resp.text.count('<tr') == 4
 
+    # check rank takes over when searching on text
+    custom_view.order_by = '-f1'
+    custom_view.store()
+    resp = app.get('/backoffice/management/form-title/shared-custom-test-view/')
+    resp.forms['listing-settings']['q'] = 'foo'
+    resp = resp.forms['listing-settings'].submit()
+    assert re.findall(r'<a href="(\d)/">1-(\d)</a>', resp.text) == [('1', '1'), ('2', '2'), ('3', '3')]
+
+    # but can still be overridden by query string
+    resp.forms['listing-settings']['order_by'] = '-f1'
+    resp = resp.forms['listing-settings'].submit()
+    assert re.findall(r'<a href="(\d)/">1-(\d)</a>', resp.text) == [('1', '1'), ('3', '3'), ('2', '2')]
+
 
 def test_carddata_custom_view(pub):
     user = create_user(pub)

tests/backoffice_pages/test_export.py

@@ -216,8 +216,8 @@ def test_backoffice_csv(pub):
     assert resp_csv.text.splitlines() == [
         '"3rd field (identifier)","3rd field"',
         '"foo",""',
-        '"A","aa"',
         '"C","cc"',
+        '"A","aa"',
     ]
 
 
@@ -403,6 +403,12 @@ def test_backoffice_csv_export_fields(pub):
     formdata.jump_status('new')
     formdata.store()
 
+    # add an extra empty formdata
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.jump_status('new')
+    formdata.store()
+
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/')
     resp_csv = resp.click('Export a Spreadsheet')
@@ -413,6 +419,7 @@ def test_backoffice_csv_export_fields(pub):
     resp.forms['listing-settings']['567'].checked = True
     resp.forms['listing-settings']['678'].checked = True
     resp.forms['listing-settings']['890'].checked = True
+    resp.forms['listing-settings']['order_by'] = 'id'
     resp = resp.forms['listing-settings'].submit()
     resp_csv = resp.click('Export a Spreadsheet')
     resp_csv.form['format'] = 'csv'
@@ -426,8 +433,7 @@ def test_backoffice_csv_export_fields(pub):
     ]
     line1 = resp_csv.text.splitlines()[1].split(',')[-5:]
     line2 = resp_csv.text.splitlines()[2].split(',')[-5:]
-    if line1[0] == '"blah2@example.invalid"':
-        line1, line2 = line2, line1
+    line3 = resp_csv.text.splitlines()[3].split(',')[-5:]
     assert line1 == [
         '"blah@example.invalid"',
         '"2020-04-24"',
@@ -442,6 +448,7 @@ def test_backoffice_csv_export_fields(pub):
         '"No"',
         '"2.5"',
     ]
+    assert line3 == ['""', '""', '""', '""', '""']
 
     # export as ods
     resp_csv = resp.click('Export a Spreadsheet')

tests/backoffice_pages/test_i18n.py

@@ -27,7 +27,7 @@ def pub():
         'default_site_language': 'http',
     }
     pub.write_cfg()
-    TranslatableMessage.do_table()
+    TranslatableMessage.do_table()  # update table with selected languages
     return pub
 
 

tests/backoffice_pages/test_submission.py

@@ -869,6 +869,14 @@ def test_backoffice_submission_drafts_order(pub):
         f'form-title/{x}/' for x in reversed(formdata_ids)
     ]
 
+    formdata.receipt_time = None  # check a missing receipt_time is ok
+    formdata.store()
+    resp = app.get('/backoffice/submission/')
+    assert [x.attrib['href'] for x in resp.pyquery('.biglist.empty a:not(.fake)')] == [
+        f'form-title/{x}/' for x in reversed(formdata_ids)
+    ]
+    assert 'unknown date' in resp.pyquery('li.smallitem:first').text()
+
 
 def test_backoffice_submission_prefill_user(pub):
     user = create_user(pub)

tests/form_pages/test_all.py

@@ -4936,7 +4936,10 @@ def test_create_formdata_locked_prefill_parent(create_formdata):
 
 
 def test_js_libraries(pub):
-    create_formdef()
+    formdef = create_formdef()
+    formdef.enable_tracking_codes = True  # will force gadjo.js -> jquery-ui
+    formdef.store()
+
     resp = get_app(pub).get('/test/', status=200)
     assert 'jquery.js' not in resp.text
     assert 'jquery.min.js' in resp.text
@@ -4947,6 +4950,8 @@ def test_js_libraries(pub):
     resp = get_app(pub).get('/test/', status=200)
     assert 'jquery.js' in resp.text
     assert 'jquery.min.js' not in resp.text
+    assert 'jquery-ui.js' in resp.text
+    assert 'jquery-ui.min.js' not in resp.text
     assert 'qommon.forms.js' in resp.text
 
     pub.cfg['branding'] = {'included_js_libraries': ['jquery.js']}
@@ -4956,6 +4961,44 @@ def test_js_libraries(pub):
     assert 'jquery.min.js' not in resp.text
     assert 'qommon.forms.js' in resp.text
 
+    pub.cfg['branding'] = {'included_js_libraries': ['jquery.js', 'jquery-ui.js']}
+    pub.write_cfg()
+    resp = get_app(pub).get('/test/', status=200)
+    assert 'jquery.js' not in resp.text
+    assert 'jquery.min.js' not in resp.text
+    assert 'qommon.forms.js' in resp.text
+
+    pub.cfg['branding'] = {'included_js_libraries': ['jquery.js']}
+    pub.write_cfg()
+    formdef.enable_tracking_codes = False  # no popup, no jquery-ui (and no i18n.js)
+    formdef.store()
+    resp = get_app(pub).get('/test/', status=200)
+    assert 'jquery-ui.js' not in resp.text
+    assert 'jquery-ui.min.js' not in resp.text
+    assert 'select2.js' not in resp.text
+    assert 'i18n.js' not in resp.text
+
+    # add autocomplete field
+    formdef.fields = [
+        fields.ItemField(
+            id='1',
+            label='string',
+            data_source={'type': 'jsonp', 'value': 'http://remote.example.net/jsonp'},
+        ),
+    ]
+    formdef.store()
+    resp = get_app(pub).get('/test/', status=200)
+    assert 'select2.js' in resp.text
+    assert 'select2.css' in resp.text
+    assert 'i18n.js' in resp.text
+
+    pub.cfg['branding'] = {'included_js_libraries': ['jquery.js', 'select2.js']}
+    pub.write_cfg()
+    resp = get_app(pub).get('/test/', status=200)
+    assert 'select2.js' not in resp.text
+    assert 'select2.css' not in resp.text
+    assert 'i18n.js' in resp.text
+
 
 def test_after_submit_location(pub):
     create_user(pub)

tests/form_pages/test_block.py

@@ -2598,3 +2598,57 @@ def test_block_prefill_full_block_email(pub):
         },
         '1_display': 'foo@example.net',
     }
+
+
+def test_block_titles_and_empty_block_on_summary_page(pub, emails):
+    FormDef.wipe()
+    BlockDef.wipe()
+    create_user(pub)
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [
+        fields.StringField(id='123', label='Test', required=False, varname='foo'),
+    ]
+    block.digest_template = '{{block_var_foo}}'
+    block.store()
+
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [
+        fields.PageField(id='1', label='Form Page'),
+        fields.PageField(id='3', label='Hidden Page', condition={'type': 'django', 'value': 'False'}),
+        fields.TitleField(id='4', label='Second Form Title'),
+        fields.BlockField(id='5', label='Second Block Test', required=False, block_slug='foobar'),
+        fields.PageField(id='6', label='Form Page'),
+        fields.TitleField(id='7', label='Form Title'),
+        fields.BlockField(id='8', label='Block Test', required=False, block_slug='foobar'),
+    ]
+    formdef.store()
+
+    # filled
+    app = login(get_app(pub), username='foo', password='foo')
+    resp = app.get(formdef.get_url())
+    resp = resp.form.submit('submit')  # -> second page
+    resp.form['f8$element0$f123'] = 'Blah Field'
+    resp = resp.form.submit('submit')  # -> validation page
+    resp = resp.form.submit('submit').follow()  # -> submitted
+    assert 'Form Page' in resp.text
+    assert 'Form Title' in resp.text
+    assert 'Blah Field' in resp.text
+    assert 'Form Page' in emails.get('New form (form title)')['msg'].get_payload()[0].get_payload()
+    assert 'Form Title' in emails.get('New form (form title)')['msg'].get_payload()[0].get_payload()
+    assert 'Blah Field' in emails.get('New form (form title)')['msg'].get_payload()[0].get_payload()
+
+    # empty
+    emails.empty()
+    app = login(get_app(pub), username='foo', password='foo')
+    resp = app.get(formdef.get_url())
+    resp = resp.form.submit('submit')  # -> second page
+    resp.form['f8$element0$f123'] = ''  # left empty
+    resp = resp.form.submit('submit')  # -> validation page
+    resp = resp.form.submit('submit').follow()  # -> submitted
+    assert 'Form Page' not in resp.text
+    assert 'Form Title' not in resp.text
+    assert 'Form Page' not in emails.get('New form (form title)')['msg'].get_payload()[0].get_payload()
+    assert 'Form Title' not in emails.get('New form (form title)')['msg'].get_payload()[0].get_payload()

tests/form_pages/test_i18n.py

@@ -37,7 +37,7 @@ def pub():
     with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
         pub.site_options.write(fd)
 
-    TranslatableMessage.do_table()
+    TranslatableMessage.do_table()  # update table with selected languages
 
     return pub
 

tests/form_pages/test_prefill.py

@@ -10,6 +10,7 @@ from webtest import Upload
 from wcs import fields
 from wcs.admin.settings import UserFieldsFormDef
 from wcs.blocks import BlockDef
+from wcs.carddef import CardDef
 from wcs.categories import Category
 from wcs.data_sources import NamedDataSource
 from wcs.wf.create_formdata import Mapping
@@ -401,6 +402,62 @@ def test_form_page_template_list_prefill(pub):
     assert 'invalid value selected' not in resp.text
 
 
+def test_form_page_template_list_prefill_by_text(pub):
+    NamedDataSource.wipe()
+    data_source = NamedDataSource(name='foobar')
+    data_source.data_source = {
+        'type': 'jsonvalue',
+        'value': '[{"id": 1, "text": "foo"}, {"id": 2, "text": "bar"}]',
+    }
+    data_source.store()
+
+    formdef = create_formdef()
+    formdef.data_class().wipe()
+    formdef.fields = [
+        fields.ItemField(
+            id='1',
+            label='item',
+            varname='item',
+            required=True,
+            data_source={'type': data_source.slug},
+            prefill={'type': 'string', 'value': 'bar'},
+        )
+    ]
+    formdef.store()
+
+    resp = get_app(pub).get('/test/')
+    assert resp.form['f1'].value == '2'
+    assert 'invalid value selected' not in resp.text
+
+    # check with card data source
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'Test'
+    carddef.fields = [
+        fields.StringField(id='0', label='blah', varname='blah'),
+    ]
+    carddef.digest_templates = {'default': '{{ form_var_blah }}'}
+    carddef.store()
+    carddef.data_class().wipe()
+
+    carddata1 = carddef.data_class()()
+    carddata1.data = {'0': 'foo'}
+    carddata1.just_created()
+    carddata1.store()
+
+    carddata2 = carddef.data_class()()
+    carddata2.data = {'0': 'bar'}
+    carddata2.just_created()
+    carddata2.store()
+
+    formdef.data_source = {'type': 'carddef:test'}
+    formdef.store()
+
+    resp = get_app(pub).get('/test/')
+    assert resp.form['f1'].value == str(carddata2.id)
+    assert 'invalid value selected' not in resp.text
+
+
 def test_form_page_query_string_list_prefill(pub):
     create_user(pub)
     formdef = create_formdef()

tests/form_pages/test_tracking_code.py

@@ -11,6 +11,7 @@ from wcs.categories import Category
 from wcs.formdef import FormDef
 from wcs.qommon.template import Template
 from wcs.tracking_code import TrackingCode
+from wcs.wf.comment import WorkflowCommentPart
 
 from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
 from .test_all import assert_current_page, create_formdef, create_user, get_displayed_tracking_code
@@ -190,6 +191,10 @@ def test_form_tracking_code(pub, nocache):
     # check using /code/load?code= is not allowed
     resp = app.get('/code/load?code=ABC', status=405)
 
+    # check posting to /code/load with an empty code gives a proper error
+    resp = app.post('/code/load', status=400)
+    assert 'missing parameter' in resp.text
+
 
 def test_form_tracking_code_js_order(pub, nocache):
     formdef = create_formdef()
@@ -891,3 +896,30 @@ def test_temporary_access_url(pub, freezer):
     context = pub.substitutions.get_context_variables()
     tmpl = Template('{% temporary_access_url %}')
     assert tmpl.render(context) == ''
+
+
+def test_form_tracking_code_workflow_action(pub):
+    formdef = create_formdef()
+    formdef.fields = [fields.StringField(id='0', label='string')]
+    formdef.enable_tracking_codes = True
+    formdef.store()
+    formdef.data_class().wipe()
+
+    resp = get_app(pub).get('/test/')
+    resp.forms[0]['f0'] = 'foobar'
+    resp = resp.forms[0].submit('submit')  # -> validation
+    resp = resp.forms[0].submit('submit')  # -> done
+    formdata = formdef.data_class().select()[0]
+
+    resp = get_app(pub).get(formdata.get_url(), status=302)  # redirection to login
+
+    resp = get_app(pub).get('/')
+    resp.forms[0]['code'] = formdata.tracking_code
+    resp = resp.forms[0].submit().follow().follow()
+    resp.forms['wf-actions']['comment'] = 'Test comment'
+    resp = resp.forms['wf-actions'].submit('button_commentable')
+
+    # check action has been recorded as submitter
+    formdata.refresh_from_storage()
+    assert isinstance(formdata.evolution[-1].parts[0], WorkflowCommentPart)
+    assert formdata.evolution[-1].who == '_submitter'

tests/test_ctl.py

@@ -415,10 +415,22 @@ open(os.path.join(get_publisher().app_dir, 'runscript.test'), 'w').close()
         )
     call_command('runscript', '--domain=example.net', os.path.join(pub.app_dir, 'test2.py'))
     assert os.path.exists(os.path.join(pub.app_dir, 'runscript.test'))
+
     os.unlink(os.path.join(pub.app_dir, 'runscript.test'))
     call_command('runscript', '--all-tenants', os.path.join(pub.app_dir, 'test2.py'))
     assert os.path.exists(os.path.join(pub.app_dir, 'runscript.test'))
 
+    os.unlink(os.path.join(pub.app_dir, 'runscript.test'))
+    call_command(
+        'runscript', '--all-tenants', '--exclude-tenants=example.net', os.path.join(pub.app_dir, 'test2.py')
+    )
+    assert not os.path.exists(os.path.join(pub.app_dir, 'runscript.test'))
+
+    call_command(
+        'runscript', '--all-tenants', '--exclude-tenants=example2.net', os.path.join(pub.app_dir, 'test2.py')
+    )
+    assert os.path.exists(os.path.join(pub.app_dir, 'runscript.test'))
+
 
 def test_import_site():
     with pytest.raises(CommandError):

tests/test_formdef_import.py

@@ -1,3 +1,4 @@
+import decimal
 import io
 import time
 import xml.etree.ElementTree as ET
@@ -290,6 +291,22 @@ def test_workflow_options_with_int(pub):
     assert formdef.workflow_options['foo'] == fd2.workflow_options['foo']
 
 
+def test_workflow_options_with_float(pub):
+    formdef = FormDef()
+    formdef.name = 'foo'
+    formdef.workflow_options = {'foo': 123.2}
+    fd2 = assert_xml_import_export_works(formdef)
+    assert formdef.workflow_options['foo'] == fd2.workflow_options['foo']
+
+
+def test_workflow_options_with_decimal(pub):
+    formdef = FormDef()
+    formdef.name = 'foo'
+    formdef.workflow_options = {'foo': decimal.Decimal('123.2')}
+    fd2 = assert_xml_import_export_works(formdef)
+    assert formdef.workflow_options['foo'] == fd2.workflow_options['foo']
+
+
 def test_workflow_options_with_list(pub):
     formdef = FormDef()
     formdef.name = 'foo'

tests/test_i18n.py

@@ -39,7 +39,7 @@ def test_translation_columns(pub):
     assert not column_exists_in_table(cur, 'translatable_messages', 'string_fr')
     pub.cfg['language'] = {'language': 'en', 'multilinguism': True, 'languages': ['fr', 'de']}
     pub.write_cfg()
-    TranslatableMessage.do_table()
+    TranslatableMessage.do_table()  # update table with selected languages
     assert column_exists_in_table(cur, 'translatable_messages', 'string_de')
     assert column_exists_in_table(cur, 'translatable_messages', 'string_fr')
     # check it's not removed

tests/test_snapshots.py

@@ -826,6 +826,11 @@ def test_workflow_snapshot_browse(pub):
     workflow.store()
     assert pub.snapshot_class.count() == 1
 
+    # create a new snapshot
+    workflow.name = 'new name'
+    workflow.store()
+    assert pub.snapshot_class.count() == 2
+
     app = login(get_app(pub))
 
     resp = app.get('/backoffice/workflows/%s/history/' % workflow.id)
@@ -835,6 +840,18 @@ def test_workflow_snapshot_browse(pub):
     assert '<p>%s</p>' % localstrftime(snapshot.timestamp) in resp.text
 
     # check restore/export links of sidebar
+    # latest version has its restore link disabled
+    assert [(x.text, 'disabled' in x.attrib['class']) for x in resp.pyquery('#sidebar [role="button"]')] == [
+        ('Restore version', True),
+        ('Export version', False),
+        ('Inspect version', False),
+    ]
+    resp = resp.click('&gt;')
+    assert [(x.text, 'disabled' in x.attrib['class']) for x in resp.pyquery('#sidebar [role="button"]')] == [
+        ('Restore version', False),
+        ('Export version', False),
+        ('Inspect version', False),
+    ]
     resp.click('Restore version')
     resp_export = resp.click('Export version')
     assert 'snapshot-workflow' in resp_export.headers['Content-Disposition']
@@ -933,6 +950,10 @@ def test_workflow_snapshot_restore(pub):
     workflow = Workflow(name='test')
     workflow.store()
 
+    # create a new snapshot
+    workflow.name = 'new name'
+    workflow.store()
+
     app = login(get_app(pub))
 
     resp = app.get('/backoffice/workflows/%s/history/' % workflow.id)
@@ -1069,6 +1090,10 @@ def test_workflow_with_model_snapshot_browse(pub):
             == 1 + i
         )
 
+    # create a new snapshot
+    workflow.name = 'new name'
+    workflow.store()
+
     resp = app.get('/backoffice/workflows/%s/history/' % workflow.id)
     resp = resp.click(href='%s/restore' % snapshot.id)
     assert resp.form['action'].value == 'as-new'

tests/test_testdef.py

@@ -2,11 +2,9 @@ import datetime
 import decimal
 import json
 import time
-from unittest import mock
 
 import pytest
 import responses
-from quixote import get_publisher
 
 from wcs import fields
 from wcs.blocks import BlockDef
@@ -1000,6 +998,50 @@ def test_computed_field_value_too_long(pub):
     testdef.run(formdef)
 
 
+def test_computed_field_forms_template_access(pub):
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.fields = [
+        fields.PageField(
+            id='0',
+            label='1st page',
+            post_conditions=[
+                {
+                    'condition': {'type': 'django', 'value': 'form_var_computed == 1'},
+                    'error_message': 'Not enough chars.',
+                }
+            ],
+        ),
+        fields.ComputedField(
+            id='1',
+            label='Computed',
+            varname='computed',
+            value_template='{{ forms|objects:"test-title"|count }}',
+            freeze_on_initial_value=True,
+        ),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+    formdata.store()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.run(formdef)
+
+    formdef.fields[
+        1
+    ].value_template = '{{ forms|objects:"test-title"|filter_by:"unknown"|filter_value:"xxx"|count }}'
+    formdef.store()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    with pytest.raises(TestError) as excinfo:
+        testdef.run(formdef)
+    assert str(excinfo.value) == 'Page 1 post condition was not met (form_var_computed == 1).'
+    assert testdef.recorded_errors == ['Invalid filter "unknown"']
+
+
 def test_expected_error(pub):
     formdef = FormDef()
     formdef.name = 'test title'
@@ -1092,34 +1134,6 @@ def test_expected_error_conditional_field(pub):
     )
 
 
-def test_record_error_raises_exception(pub, monkeypatch):
-    formdef = FormDef()
-    formdef.name = 'test title'
-    formdef.fields = [
-        fields.StringField(id='1', label='String field digits'),
-    ]
-    formdef.store()
-
-    formdata = formdef.data_class()()
-    formdata.just_created()
-    formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
-    formdata.data['1'] = '1'
-
-    testdef = TestDef.create_from_formdata(formdef, formdata)
-    testdef.run(formdef)
-
-    def record_error():
-        try:
-            1 / 0
-        except ZeroDivisionError as e:
-            get_publisher().record_error('Error', formdef=formdef, exception=e)
-
-    with pytest.raises(ZeroDivisionError) as excinfo:
-        with mock.patch('wcs.qommon.form.StringWidget.parse', side_effect=record_error):
-            testdef.run(formdef)
-    assert str(excinfo.value) == 'division by zero'
-
-
 def test_is_in_backoffice(pub):
     formdef = FormDef()
     formdef.name = 'test title'

tests/test_widgets.py

@@ -1633,6 +1633,11 @@ def test_numeric_widget():
     assert widget.has_error()
     assert widget.get_error() == 'You should enter a number, for example: 123.'
 
+    widget = NumericWidget('test', min_value=decimal.Decimal('1E+1'))
+    mock_form_submission(req, widget, {'test': '0'})
+    assert widget.has_error()
+    assert widget.get_error() == 'You should enter a number greater than or equal to 10.'
+
     widget = NumericWidget('test', min_value=decimal.Decimal('1E+1'))
     mock_form_submission(req, widget, {'test': '5'})
     assert widget.has_error()

tests/utilities.py

@@ -189,6 +189,7 @@ def create_temporary_pub(pickle_mode=False, lazy_mode=False):
         sql.Application.do_table()
         sql.ApplicationElement.do_table()
         sql.SearchableFormDef.do_table()
+        sql.TranslatableMessage.do_table()
         sql.init_global_table()
 
         conn.close()

tests/workflow/test_all.py

@@ -1736,6 +1736,7 @@ def test_global_timeouts_latest_arrival(pub):
     formdata1.jump_status('new')
     # enter in status 8 days ago
     formdata1.evolution[-1].time = time.localtime(time.time() - 8 * 86400)
+    formdata1.store()
     # but get a new comment 1 day ago
     formdata1.evolution.append(Evolution(formdata1))
     formdata1.evolution[-1].time = time.localtime(time.time() - 1 * 86400)

tests/workflow/test_carddata.py

@@ -1262,10 +1262,10 @@ def test_edit_carddata_partial_block_field(pub, admin_user):
     resp = login(get_app(pub), username='admin', password='admin').get(edit.get_admin_url())
     assert resp.form['mappings$element1$field_id'].options == [
         ('', False, '---'),
-        ('0', False, 'foo'),
-        ('1', False, 'block field'),
-        ('1$123', True, 'block field - Test'),
-        ('1$234', False, 'block field - Test2'),
+        ('0', False, 'foo - Text (line)'),
+        ('1', False, 'block field - Field Block (foobar)'),
+        ('1$123', True, 'block field - Test - Text (line)'),
+        ('1$234', False, 'block field - Test2 - Text (line)'),
     ]
     resp = resp.form.submit('submit')
 

tests/workflow/test_display_message.py

@@ -3,7 +3,8 @@ from quixote import cleanup
 
 from wcs.workflows import Workflow
 
-from ..utilities import clean_temporary_pub, create_temporary_pub
+from ..admin_pages.test_all import create_superuser
+from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
 
 
 def setup_module(module):
@@ -42,3 +43,22 @@ def test_display_message_migrate(pub):
     workflow.migrate()
     assert not workflow.possible_status[0].items[0].level
     assert workflow.possible_status[0].items[0].message == '<div class="errornotice blah">message</div>'
+
+
+def test_display_message_rich_text(pub):
+    create_superuser(pub)
+
+    workflow = Workflow(name='display message to')
+    st1 = workflow.add_status('Status1', 'st1')
+    display_message = st1.add_action('displaymsg')
+    display_message.message = '<p>hello world</p>'
+    workflow.store()
+
+    app = login(get_app(pub))
+    resp = app.get(display_message.get_admin_url())
+    assert resp.pyquery('textarea[data-godo-schema]')  # godo
+
+    display_message.message = '<table><tr><td>hello world</td></tr></table>'
+    workflow.store()
+    resp = app.get(display_message.get_admin_url())
+    assert resp.pyquery('textarea[data-config]')  # ckeditor

tests/workflow/test_jump.py

@@ -3,6 +3,7 @@ import time
 from unittest import mock
 
 import pytest
+from pyquery import PyQuery
 from quixote import cleanup
 
 from wcs.fields import DateField, StringField
@@ -11,7 +12,8 @@ from wcs.qommon.http_request import HTTPRequest
 from wcs.wf.jump import JumpWorkflowStatusItem, _apply_timeouts
 from wcs.workflows import Workflow, perform_items
 
-from ..utilities import clean_temporary_pub, create_temporary_pub
+from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
+from .test_all import admin_user  # noqa pylint: disable=unused-import
 
 
 def setup_module(module):
@@ -26,6 +28,7 @@ def teardown_module(module):
 def pub(request):
     pub = create_temporary_pub()
     pub.cfg['language'] = {'language': 'en'}
+    pub.cfg['identification'] = {'methods': ['password']}
     pub.write_cfg()
     req = HTTPRequest(None, {'SERVER_NAME': 'example.net', 'SCRIPT_NAME': ''})
     req.response.filter = {}
@@ -540,3 +543,42 @@ def test_conditional_jump_vs_tracing(pub):
         ('register-comment', str(comment.id)),
         ('jump', str(jump2.id)),
     ]
+
+
+def test_timeout_tracing(pub, admin_user):
+    workflow = Workflow(name='timeout')
+    st1 = workflow.add_status('Status1', 'st1')
+    st2 = workflow.add_status('Status2', 'st2')
+
+    jump = st1.add_action('timeout', id='_jump')
+    jump.timeout = 30 * 60  # 30 minutes
+    jump.status = 'st2'
+
+    add_message = st2.add_action('register-comment')
+    add_message.comment = 'hello'
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.workflow_id = workflow.id
+    assert formdef.get_workflow().id == workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    rewind(formdata, seconds=40 * 60)
+    formdata.store()
+    formdata.record_workflow_event('backoffice-created')
+    _apply_timeouts(pub)
+
+    resp = login(get_app(pub), username='admin', password='admin').get(
+        formdata.get_backoffice_url() + 'inspect'
+    )
+    assert [PyQuery(x).text() for x in resp.pyquery('#inspect-timeline li > *:nth-child(2)')] == [
+        'Created (backoffice submission)',
+        'Status1',
+        'Timeout jump',
+        'Status2',
+        'History Message',
+    ]

wcs/admin/fields.py

@@ -105,6 +105,12 @@ class FieldDefPage(Directory):
                 )
             else:
                 r += htmltext('<h3 class="field-edit--subtitle">%s</h3>') % self.field.description
+            existing_varnames = {
+                x.varname for x in self.objectdef.fields if x.varname if x.id != self.field.id
+            }
+            r += htmltext(
+                '<script id="other-fields-varnames">%s</script>' % json.dumps(list(existing_varnames))
+            )
             for widget in form.widgets:
                 if hasattr(widget, 'get_widget'):
                     add_element_widget = widget.get_widget('add_element')

wcs/admin/tests.py

@@ -405,6 +405,27 @@ class TestsDirectory(Directory):
         return redirect('.')
 
 
+class TestResultDetailPage(Directory):
+    _q_exports = ['']
+
+    def __init__(self, component, test_result):
+        self.result_index = component
+
+        try:
+            self.result = test_result.results[int(component)]
+        except (KeyError, ValueError):
+            raise TraversalError()
+
+    def _q_traverse(self, path):
+        get_response().breadcrumb.append(
+            (str(self.result_index) + '/', _('Details of %(test_name)s') % {'test_name': self.result['name']})
+        )
+        return super()._q_traverse(path)
+
+    def _q_index(self):
+        return render_to_string('wcs/backoffice/test-result-detail.html', context={'result': self.result})
+
+
 class TestResultPage(Directory):
     _q_exports = ['']
 
@@ -422,7 +443,12 @@ class TestResultPage(Directory):
         )
         return super()._q_traverse(path)
 
+    def _q_lookup(self, component):
+        return TestResultDetailPage(component, self.test_result)
+
     def _q_index(self):
+        get_response().add_javascript(['popup.js'])
+
         testdefs = TestDef.select_for_objectdef(self.objectdef)
         testdefs_by_id = {x.id: x for x in testdefs}
         for test in self.test_result.results:
@@ -522,6 +548,7 @@ class TestsAfterJob(AfterJob):
                 'id': test.id,
                 'name': str(test),
                 'error': getattr(test, 'error', None),
+                'recorded_errors': test.recorded_errors,
                 'missing_required_fields': test.missing_required_fields,
             }
             for test in testdefs

wcs/admin/utils.py

@@ -89,9 +89,18 @@ def snapshot_info_block(snapshot, url_name='view/', url_prefix='../../', url_suf
         r += htmltext('</p>')
 
     r += htmltext('<div>')
-    r += htmltext(
-        '<a class="button button-paragraph" href="%s%s/restore" role="button" rel="popup">%s</a>'
-    ) % (url_prefix, snapshot.id, _('Restore version'))
+    if snapshot.id == snapshot.first:
+        r += htmltext(
+            '<a class="button button-paragraph disabled" href="#" role="button" rel="popup">%s</a>'
+        ) % (_('Restore version'),)
+    else:
+        r += htmltext(
+            '<a class="button button-paragraph" href="%s%s/restore" role="button" rel="popup">%s</a>'
+        ) % (
+            url_prefix,
+            snapshot.id,
+            _('Restore version'),
+        )
     r += htmltext('<a class="button button-paragraph" href="%s%s/export" role="button">%s</a>') % (
         url_prefix,
         snapshot.id,

wcs/backoffice/data_management.py

@@ -229,6 +229,9 @@ class CardPage(FormPage):
             CheckboxWidget,
             'update_existing_cards',
             title=_('Update existing cards (only for JSON imports)'),
+            hint=_('Cards will be matched using their unique identifier ("uuid" property).')
+            if not self.formdef.id_template
+            else _('Cards will be matched using their custom identifier ("id" property).'),
             value=False,
         )
         form.add_submit('submit', _('Submit'))
@@ -533,7 +536,14 @@ class ImportFromJsonAfterJob(AfterJob):
             carddata = self.carddef.data_class()()
             carddata.data = {}
             if update_existing_cards:
-                if json_data.get('uuid'):
+                if self.carddef.id_template and json_data.get('id'):
+                    try:
+                        carddata = self.carddef.data_class().get_by_id(json_data.get('id'))
+                    except KeyError:
+                        pass  # new card will get this id when computing id_template
+                    else:
+                        orig_data = copy.copy(carddata.data)
+                elif json_data.get('uuid'):
                     try:
                         normalized_uuid = str(uuid.UUID(json_data.get('uuid')))
                     except ValueError:
@@ -568,6 +578,25 @@ class ImportFromJsonAfterJob(AfterJob):
                 card_status = None
                 card_status_id = None
 
+            if self.carddef.id_template:
+                # check id is unique
+                carddata.set_auto_fields()
+                try:
+                    carddata_with_same_id = self.carddef.data_class().get_by_id(carddata.id_display)
+                except KeyError:
+                    pass  # unique id, fine
+                else:
+                    if update_existing_cards and carddata.id == carddata_with_same_id.id:  # fine
+                        orig_data = copy.copy(carddata.data)
+                    elif update_existing_cards:  # overwrite
+                        orig_data = copy.copy(carddata_with_same_id.data)
+                        carddata_with_same_id.data = carddata.data
+                        carddata = carddata_with_same_id
+                    else:
+                        # not asked to update, and we do not want to create a duplicate, so skip silently
+                        self.increment_count()
+                        continue
+
             if carddata.id is None:
                 # no id, this is a new card
                 carddata.submission_context = {

wcs/backoffice/i18n.py

@@ -32,7 +32,7 @@ from wcs.mail_templates import MailTemplate
 from wcs.qommon import _, errors, get_cfg, misc, ods, template
 from wcs.qommon.afterjobs import AfterJob
 from wcs.qommon.form import CheckboxWidget, FileWidget, Form, RadiobuttonsWidget, TextWidget
-from wcs.sql_criterias import Equal, FtsMatch, ILike, Or
+from wcs.sql_criterias import ArrayPrefixMatch, Equal, FtsMatch, ILike, Or
 from wcs.workflows import Workflow
 
 
@@ -82,6 +82,18 @@ class I18nDirectory(Directory):
         if get_request().form.get('q'):
             search_term = get_request().form.get('q')
             criterias.append(Or([ILike('string', search_term), FtsMatch(search_term, extra_normalize=False)]))
+        if get_request().form.get('formdef'):
+            kind, kind_id = get_request().form.get('formdef').split('/')
+            formdef_class = FormDef if kind == 'forms' else CardDef
+            formdef = formdef_class.get(kind_id, lightweight=True)
+            criterias.append(
+                Or(
+                    [
+                        ArrayPrefixMatch('locations', f'{kind}/{kind_id}/'),
+                        ArrayPrefixMatch('locations', f'workflows/{formdef.workflow_id}/'),
+                    ]
+                )
+            )
 
         offset = misc.get_int_or_400(get_request().form.get('offset', 0))
         limit = misc.get_int_or_400(get_request().form.get('limit', 20))
@@ -95,9 +107,11 @@ class I18nDirectory(Directory):
             'pagination_links': pagination_links(offset, limit, total_count, load_js=False),
             'messages': TranslatableMessage.select(criterias, offset=offset, limit=limit, order_by='string'),
             'query': get_request().get_query(),
+            'selected_formdef': get_request().form.get('formdef'),
             'non_translatable': get_request().form.get('non-translatable'),
+            'formdefs': FormDef.select(lightweight=True, order_by='name'),
+            'carddefs': CardDef.select(lightweight=True, order_by='name'),
         }
-
         get_response().add_javascript(['popup.js'])
         return template.QommonTemplateResponse(
             templates=['wcs/backoffice/i18n.html'], context=context, is_django_native=True

wcs/backoffice/management.py

@@ -1485,6 +1485,23 @@ class FormPage(Directory, TempfileDirectoryMixin):
 
         return r.getvalue()
 
+    def get_default_order_by(self, system_default_order_by='-receipt_time'):
+        default_order_by = get_publisher().get_site_option('default-sort-order') or system_default_order_by
+        if self.view:
+            default_order_by = self.view.order_by or default_order_by
+        return default_order_by
+
+    def get_order_by_from_query(self, system_default_order_by='-receipt_time'):
+        order_by = misc.get_order_by_or_400(get_request().form.get('order_by'))
+        default_order_by = self.get_default_order_by(system_default_order_by=system_default_order_by)
+
+        if get_request().form.get('q') and not order_by:
+            order_by = 'rank'
+        if not order_by:
+            order_by = default_order_by
+
+        return order_by
+
     def get_fields_sidebar(
         self,
         selected_filter,
@@ -1508,8 +1525,8 @@ class FormPage(Directory, TempfileDirectoryMixin):
         if limit:
             r += htmltext('<input type="hidden" name="limit" value="%s"/>') % limit
 
-        if order_by is None:
-            order_by = get_publisher().get_site_option('default-sort-order') or '-receipt_time'
+        if not order_by or order_by == self.get_default_order_by():
+            order_by = ''
         r += htmltext('<input type="hidden" name="order_by" value="%s"/>') % order_by
 
         r += htmltext('<h3>%s</h3>') % self.search_label
@@ -2351,13 +2368,9 @@ class FormPage(Directory, TempfileDirectoryMixin):
             get_request().form.get('limit', get_publisher().get_site_option('default-page-size') or 20)
         )
         offset = misc.get_int_or_400(get_request().form.get('offset', 0))
-        order_by = misc.get_order_by_or_400(get_request().form.get('order_by'))
-        if self.view and not order_by:
-            order_by = self.view.order_by
-        if not order_by:
-            order_by = get_publisher().get_site_option('default-sort-order') or '-receipt_time'
-        query = get_request().form.get('q')
+        order_by = self.get_order_by_from_query()
 
+        query = get_request().form.get('q')
         qs = ''
         if get_request().get_query():
             qs = '?' + get_request().get_query()
@@ -2558,7 +2571,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
         user = get_request().user
         query = get_request().form.get('q')
         criterias = self.get_criterias_from_query()
-        order_by = misc.get_order_by_or_400(get_request().form.get('order_by', None))
+        order_by = self.get_order_by_from_query(system_default_order_by='-id')
         skip_header_line = bool(get_request().form.get('skip_header_line'))
 
         count = self.formdef.data_class().count()
@@ -2616,7 +2629,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
         user = get_user_from_api_query_string() or get_request().user
         query = get_request().form.get('q')
         criterias = self.get_criterias_from_query()
-        order_by = misc.get_order_by_or_400(get_request().form.get('order_by', None))
+        order_by = self.get_order_by_from_query(system_default_order_by='-id')
         skip_header_line = bool(get_request().form.get('skip_header_line'))
 
         count = self.formdef.data_class().count()
@@ -2650,7 +2663,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
         user = get_request().user
         query = get_request().form.get('q')
         criterias = self.get_criterias_from_query()
-        order_by = misc.get_order_by_or_400(get_request().form.get('order_by', None))
+        order_by = self.get_order_by_from_query()
 
         job = JsonFileExportAfterJob(
             self.formdef,
@@ -2675,9 +2688,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
         selected_filter = self.get_filter_from_query(default='all')
         selected_filter_operator = self.get_filter_operator_from_query()
         criterias = self.get_criterias_from_query()
-        order_by = misc.get_order_by_or_400(get_request().form.get('order_by', None))
-        if self.view and not order_by:
-            order_by = self.view.order_by
+        order_by = self.get_order_by_from_query()
         query = get_request().form.get('q') if not anonymise else None
         offset = None
         if 'offset' in get_request().form:
@@ -4078,6 +4089,10 @@ class FormBackOfficeStatusPage(FormStatusPage):
                 global_event = trace if trace.is_global_event() else None
                 r += trace.print_event(formdata=self.filled, global_event=global_event)
 
+            if trace.status_id != last_status_id:
+                last_status_id = trace.status_id
+                r += htmltext(trace.print_status(filled=self.filled))
+
             if trace.action_item_key:
                 r += htmltext(
                     trace.print_action(
@@ -4085,10 +4100,6 @@ class FormBackOfficeStatusPage(FormStatusPage):
                     )
                 )
 
-            if trace.status_id != last_status_id:
-                last_status_id = trace.status_id
-                r += htmltext(trace.print_status(filled=self.filled))
-
         return r.getvalue()
 
     def inspect_markers_stack(self):

wcs/backoffice/submission.py

@@ -487,7 +487,7 @@ class SubmissionDirectory(Directory):
                     formdef._formdatas = [
                         x for x in data_class.get_ids(formdata_ids) if x.backoffice_submission is True
                     ]
-                    formdef._formdatas.sort(key=lambda x: x.receipt_time)
+                    formdef._formdatas.sort(key=lambda x: x.receipt_time or time.gmtime(0))
                 skip &= not (bool(formdef._formdatas))
             if skip:
                 return
@@ -526,7 +526,9 @@ class SubmissionDirectory(Directory):
                     label = '%s ' % formdata.get_submission_channel_label()
                 label += _('#%(id)s, %(time)s') % {
                     'id': formdata.id,
-                    'time': misc.localstrftime(formdata.receipt_time),
+                    'time': misc.localstrftime(formdata.receipt_time)
+                    if formdata.receipt_time
+                    else _('unknown date'),
                 }
                 if formdata.submission_agent_id:
                     agent_user = get_publisher().user_class.get(

wcs/ctl/management/commands/__init__.py

@@ -32,10 +32,12 @@ class TenantCommand(BaseCommand):
         )
         if self.support_all_tenants:
             parser.add_argument('--all-tenants', action='store_true')
+            parser.add_argument('--exclude-tenants', metavar='TENANTS')
 
     def get_domains(self, **options):
         domain = options.get('domain')
         all_tenants = options.get('all_tenants')
+        exclude_tenants = options.get('exclude_tenants')
         if domain and all_tenants:
             raise CommandError('--domain and --all-tenants are exclusive')
         if not (domain or all_tenants):
@@ -44,6 +46,7 @@ class TenantCommand(BaseCommand):
             domains = [domain]
         else:
             domains = [x.hostname for x in get_publisher_class().get_tenants()]
+        domains = [x for x in domains if x not in (exclude_tenants or '').split(',')]
         return domains
 
     def execute(self, *args, **kwargs):

wcs/fields/block.py

@@ -260,7 +260,7 @@ class BlockField(WidgetField):
             # skip if there are no values
             return (None, {})
         value_info, value_details = super().get_value_info(data)
-        if value_info is None and value_details:
+        if value_info is None and value_details not in (None, {}, {'value_id': None}):
             # buggy digest template created an empty value, switch it to an empty string
             # so it's not considered empty in summary page.
             value_info = ''

wcs/fields/item.py

@@ -319,6 +319,15 @@ class ItemField(WidgetField, MapOptionsMixin, ItemFieldMixin, ItemWithImageField
             return data_sources.get_id_by_option_text(self.data_source, text_value)
         return text_value
 
+    def get_prefill_value(self, user=None, force_string=True):
+        value, explicit_lock = super().get_prefill_value(user=user, force_string=False)
+        if value and self.data_source:
+            data_source = data_sources.get_object(self.data_source)
+            struct_value = data_source.get_structured_value(value)
+            if struct_value:
+                value = str(struct_value.get('id'))
+        return (value, explicit_lock)
+
     def get_display_mode(self, data_source=None):
         if not data_source:
             data_source = data_sources.get_object(self.data_source)

wcs/fields/numeric.py

@@ -75,6 +75,8 @@ class NumericField(WidgetField):
         return misc.parse_decimal(value)
 
     def convert_value_to_str(self, value):
+        if value == '':
+            return value
         return django_number_format(value, use_l10n=True)
 
     def from_json_value(self, value):

wcs/formdata.py

@@ -247,11 +247,11 @@ class Evolution:
     def datetime(self):
         return datetime.datetime(*self.time[:6])
 
-    def set_user(self, formdata, user):
-        if user is None:
-            self.who = None
-        elif formdata.is_submitter(user):
+    def set_user(self, formdata, user, check_submitter=True):
+        if formdata.is_submitter(user) and check_submitter:
             self.who = '_submitter'
+        elif user is None:
+            self.who = None
         else:
             self.who = user.id
 
@@ -1531,9 +1531,10 @@ class FormData(StorableObject):
     ):
         # noqa pylint: disable=too-many-arguments
         data = {}
-        data['id'] = str(self.id)
         if hasattr(self, 'uuid'):
             data['uuid'] = self.uuid
+        data['id'] = self.identifier
+        data['internal-id'] = str(self.id)
         data['display_id'] = self.get_display_id()
         data['display_name'] = self.get_display_name()
         data['digests'] = self.digests

wcs/formdef.py

@@ -19,6 +19,7 @@ import collections
 import contextlib
 import copy
 import datetime
+import decimal
 import glob
 import itertools
 import json
@@ -1342,6 +1343,12 @@ class FormDef(StorableObject):
             elif isinstance(value, int):
                 element.attrib['type'] = 'int'
                 element.text = str(value)
+            elif isinstance(value, float):
+                element.attrib['type'] = 'float'
+                element.text = str(value)
+            elif isinstance(value, decimal.Decimal):
+                element.attrib['type'] = 'decimal'
+                element.text = str(value)
             elif isinstance(value, (set, tuple, list)):
                 element.attrib['type'] = 'list'
                 for child_value in value:
@@ -1497,6 +1504,10 @@ class FormDef(StorableObject):
         def get_value_from_xml(element):
             if element.attrib.get('type') == 'int':
                 return int(xml_node_text(element))
+            elif element.attrib.get('type') == 'float':
+                return float(xml_node_text(element))
+            elif element.attrib.get('type') == 'decimal':
+                return decimal.Decimal(xml_node_text(element))
             if element.attrib.get('type') == 'date':
                 return time.strptime(element.text, '%Y-%m-%d')
             elif element.attrib.get('type') == 'bool':

wcs/forms/root.py

@@ -282,6 +282,8 @@ class TrackingCodesDirectory(Directory):
         if get_request().get_method() != 'POST':
             raise MethodNotAllowedError(allowed_methods=['POST', 'PUT'])
         code = get_request().form.get('code')
+        if not code:
+            raise RequestError('missing parameter')
         formdata = TrackingCodeDirectory.get_formdata_from_code(code)
         return redirect(formdata.get_temporary_access_url(duration=300))
 
@@ -754,11 +756,6 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
                 css_class = 'cancel form-discard'
             form.add_submit('cancel', cancel_label, css_class=css_class, attrs={'aria-label': aria_label})
 
-        if self.has_draft_support():
-            form.add_submit(
-                'savedraft', _('Save Draft'), css_class='save-draft', attrs={'style': 'display: none'}
-            )
-
         # add fake field as honey pot
         honeypot = form.add(
             StringWidget, 'f00', value='', title=_('leave this field blank to prove your humanity'), size=25
@@ -1205,7 +1202,6 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
 
         if self.has_draft_support():
             form.add_submit('removedraft')
-            form.add_submit('savedraft')
 
         if not form.is_submitted():
             if 'mt' in get_request().form:
@@ -1300,7 +1296,6 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
             form.add_submit('previous')
             if self.has_draft_support():
                 form.add_submit('removedraft')
-                form.add_submit('savedraft')
             form.add_submit('submit')
             if page_no > 0 and form.get_submit() == 'previous':
                 return self.previous_page(page_no, magictoken)
@@ -1328,11 +1323,6 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
 
             form_data.update(data)
 
-            if self.has_draft_support() and form.get_submit() == 'savedraft':
-                form_data.update(computed_data)
-                filled = self.save_draft(form_data, page_no)
-                return redirect(filled.get_url())
-
             for field in submitted_fields:
                 if not field.is_visible(form_data, self.formdef) and 'f%s' % field.id in form._names:
                     del form._names['f%s' % field.id]
@@ -1443,7 +1433,7 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
                     return template.error_page(self.already_submitted_message)
             elif self.has_draft_support():
                 # if there's no draft yet and drafts are supported, create one
-                filled = self.save_draft(form_data, page_no)
+                self.save_draft(form_data, page_no)
 
             # the page has been successfully submitted, maybe new pages
             # should be revealed.
@@ -1486,7 +1476,6 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
                     form.add_submit('cancel')
                     if self.has_draft_support():
                         form.add_submit('removedraft')
-                        form.add_submit('savedraft')
 
             else:
                 return self.page(self.pages[page_no])
@@ -1511,10 +1500,6 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
             if self.has_draft_support() and form.get_submit() == 'removedraft':
                 return self.removedraft()
 
-            if self.has_draft_support() and form.get_submit() == 'savedraft':
-                filled = self.save_draft(form_data, page_no=-1)
-                return redirect(filled.get_url())
-
             # so it gets FakeFileWidget in preview mode
             form = self.create_view_form(form_data, use_tokens=self.has_confirmation_page())
             if self.formdef.has_captcha_enabled() and not (
@@ -2002,10 +1987,6 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
             cancel_label = _('Discard')
             css_class = 'cancel form-discard'
         form.add_submit('cancel', cancel_label, css_class=css_class)
-        if self.has_draft_support():
-            form.add_submit(
-                'savedraft', _('Save Draft'), css_class='save-draft', attrs={'style': 'display: none'}
-            )
         form.add_hidden('step', '2')
         magictoken = get_request().form['magictoken']
         form.add_hidden('magictoken', magictoken)

wcs/locale/fr/LC_MESSAGES/django.po

@@ -4,8 +4,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: wcs 0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-01-16 16:33+0100\n"
-"PO-Revision-Date: 2024-01-16 16:33+0100\n"
+"POT-Creation-Date: 2024-01-26 15:05+0100\n"
+"PO-Revision-Date: 2024-01-26 15:05+0100\n"
 "Last-Translator: Thomas Noël <tnoel@entrouvert.com>\n"
 "Language-Team: french\n"
 "Language: fr\n"
@@ -1015,7 +1015,7 @@ msgstr "Type"
 #: admin/fields.py admin/forms.py admin/settings.py admin/workflows.py
 #: api_export_import.py backoffice/journal.py backoffice/management.py
 #: formdef.py forms/root.py root.py templates/wcs/backoffice/carddef.html
-#: templates/wcs/backoffice/forms.html
+#: templates/wcs/backoffice/forms.html templates/wcs/backoffice/i18n.html
 msgid "Forms"
 msgstr "Formulaires"
 
@@ -2536,6 +2536,11 @@ msgstr ""
 msgid "Test \"%s\" has been successfully imported."
 msgstr "Le test « %s » a été importé correctement."
 
+#: admin/tests.py templates/wcs/backoffice/test-result-detail.html
+#, python-format
+msgid "Details of %(test_name)s"
+msgstr "Détails de %(test_name)s"
+
 #: admin/tests.py
 #, python-format
 msgid "Result #%s"
@@ -2651,7 +2656,7 @@ msgstr "Rechercher"
 msgid "Filter on Roles"
 msgstr "Filtrer par rôle"
 
-#: admin/users.py
+#: admin/users.py templates/wcs/backoffice/i18n.html
 msgid "Filter"
 msgstr "Filtrer"
 
@@ -3280,7 +3285,7 @@ msgstr "Non-catégorisés"
 msgid "Forms and card models"
 msgstr "Formulaires et modèles de fiche"
 
-#: admin/workflows.py carddef.py
+#: admin/workflows.py carddef.py templates/wcs/backoffice/i18n.html
 msgid "Card models"
 msgstr "Modèles de fiche"
 
@@ -3706,6 +3711,19 @@ msgid "Update existing cards (only for JSON imports)"
 msgstr ""
 "Mettre à jour les fiches existantes (uniquement pour les fichiers JSON)"
 
+#: backoffice/data_management.py
+msgid ""
+"Cards will be matched using their unique identifier (\"uuid\" property)."
+msgstr ""
+"La correspondance avec les fiches existantes se fera sur base de leur "
+"identifiant unique (propriété « uuid »)."
+
+#: backoffice/data_management.py
+msgid "Cards will be matched using their custom identifier (\"id\" property)."
+msgstr ""
+"La correspondance avec les fiches existantes se fera sur base de leur "
+"identifiant personnalisé (propriété « id »)."
+
 #: backoffice/data_management.py backoffice/i18n.py
 #: templates/wcs/backoffice/card-data-import-form.html
 msgid "Import File"
@@ -4857,6 +4875,10 @@ msgstr "Prédemande"
 msgid "#%(id)s, %(time)s"
 msgstr "n°%(id)s, %(time)s"
 
+#: backoffice/submission.py
+msgid "unknown date"
+msgstr "date inconnue"
+
 #: blocks.py
 msgid "Field block"
 msgstr "Bloc de champs"
@@ -6319,10 +6341,6 @@ msgstr "Annuler la saisie"
 msgid "Discard form"
 msgstr "Abandonner la saisie"
 
-#: forms/root.py
-msgid "Save Draft"
-msgstr "Enregistrer en tant que brouillon"
-
 #: forms/root.py
 msgid "leave this field blank to prove your humanity"
 msgstr "Laissez ce champ vide pour prouver votre humanité"
@@ -6990,6 +7008,10 @@ msgstr "format invalide"
 msgid "must start with http:// or https:// and have a domain name"
 msgstr "doit commencer par http:// ou https:// et avoir un nom de domaine"
 
+#: qommon/form.py
+msgid "This identifier is also used by another field."
+msgstr "Cet identifiant est également utilisé par un autre champ."
+
 #: qommon/form.py
 msgid "must only consist of letters, numbers, or underscore"
 msgstr "uniquement des lettres, des chiffres et le tiret bas (_)"
@@ -9404,8 +9426,12 @@ msgid "Search:"
 msgstr "Chercher :"
 
 #: templates/wcs/backoffice/i18n.html
-msgid "Search non-translatable strings"
-msgstr "Chercher dans les textes marqués à ne pas traduire"
+msgid "All forms and card models"
+msgstr "Tous les formulaires et modèles de fiche"
+
+#: templates/wcs/backoffice/i18n.html
+msgid "Include non-translatable strings"
+msgstr "Inclure les textes marqués à ne pas traduire"
 
 #: templates/wcs/backoffice/i18n.html
 msgid "Language:"
@@ -9740,6 +9766,14 @@ msgstr "Voir toutes les erreurs"
 msgid "No errors, congratulations!"
 msgstr "Aucune erreur, bravo !"
 
+#: templates/wcs/backoffice/test-result-detail.html
+msgid "Recorded errors:"
+msgstr "Erreurs enregistrées :"
+
+#: templates/wcs/backoffice/test-result-detail.html
+msgid "Missing required fields:"
+msgstr "Champs obligatoires manquants :"
+
 #: templates/wcs/backoffice/test-result.html
 msgid "Result"
 msgstr "Résultat"
@@ -9756,14 +9790,14 @@ msgstr "Démarré par :"
 msgid "Date:"
 msgstr "Date :"
 
-#: templates/wcs/backoffice/test-result.html
-msgid "Missing required fields:"
-msgstr "Champs obligatoires manquants :"
-
 #: templates/wcs/backoffice/test-result.html
 msgid "Success!"
 msgstr "Succès !"
 
+#: templates/wcs/backoffice/test-result.html
+msgid "Display details"
+msgstr "Afficher les détails"
+
 #: templates/wcs/backoffice/test-results.html
 #: templates/wcs/backoffice/tests.html
 msgid "Run tests"
@@ -9982,6 +10016,18 @@ msgstr "Il n’y a pas encore de statuts définis dans ce workflow."
 msgid "Use drag and drop with the handles to reorder status."
 msgstr "Vous pouvez utiliser les poignées ⣿ pour ordonner les statuts."
 
+#: templates/wcs/backoffice/workflow.html
+msgid "final status"
+msgstr "statut final"
+
+#: templates/wcs/backoffice/workflow.html
+msgid "pause status"
+msgstr "statut de pause"
+
+#: templates/wcs/backoffice/workflow.html
+msgid "transition status"
+msgstr "statut de transition"
+
 #: templates/wcs/backoffice/workflow.html
 msgid "Workflow Functions"
 msgstr "Fonctions dans ce workflow"
@@ -11819,6 +11865,10 @@ msgstr "Usager"
 msgid "Signed API calls"
 msgstr "Appels signés aux API"
 
+#: workflows.py
+msgid "role"
+msgstr "rôle"
+
 #: workflows.py
 msgid "Add Function"
 msgstr "Ajouter une fonction"

wcs/publisher.py

@@ -439,6 +439,7 @@ class WcsPublisher(QommonPublisher):
             sql.Application.do_table()
             sql.ApplicationElement.do_table()
             sql.SearchableFormDef.do_table()
+            sql.TranslatableMessage.do_table()
             sql.do_meta_table()
             from .carddef import CardDef
             from .formdef import FormDef

wcs/qommon/form.py

@@ -1573,7 +1573,7 @@ class NumericWidget(WcsExtraStringWidget):
             self.value = parse_decimal(value, do_raise=True, keep_none=True)
         except (ArithmeticError, TypeError, ValueError):
             self.set_error_code('bad_input')
-        if self.value:
+        if self.value is not None:
             if self.min_value is not None and self.value < self.min_value:
                 self.set_error_code('range_underflow')
             elif self.max_value is not None and self.value > self.max_value:
@@ -1934,6 +1934,14 @@ class VarnameWidget(ValidatedStringWidget):
 
     regex = r'^[a-zA-Z][a-zA-Z0-9_]*'
 
+    def render_content(self):
+        r = TemplateIO(html=True)
+        r += super().render_content()  # <input>
+        r += htmltext('<span style="display: none" class="inline-hint-message">%s</span>') % _(
+            'This identifier is also used by another field.'
+        )
+        return r.getvalue()
+
     def _parse(self, request):
         ValidatedStringWidget._parse(self, request)
         if self.error:
@@ -3962,3 +3970,11 @@ class DjangoConditionWidget(StringWidget):
                 Condition({'type': 'django', 'value': self.value}).validate()
             except ValidationError as e:
                 self.set_error(str(e))
+
+
+def get_rich_text_widget_class(content):
+    # use godo.js if all tags in existing content are supported
+    tags = set(re.findall(r'<([a-z]+)[\s>]', content or ''))
+    if tags.issubset(set(RichTextWidget.ALL_TAGS)):
+        return RichTextWidget
+    return WysiwygTextWidget

wcs/qommon/http_response.py

@@ -56,14 +56,13 @@ class HTTPResponse(quixote.http_response.HTTPResponse):
             mappings['jquery.js'] = '../xstatic/jquery.js'
             mappings['jquery-ui.js'] = '../xstatic/jquery-ui.js'
             mappings['select2.js'] = '../xstatic/select2.js'
-        if not get_request().is_in_backoffice():
+        if get_request().is_in_backoffice():
+            included_js_libraries = []
+        else:
             branding_cfg = get_publisher().cfg.get('branding') or {}
-            for included_js_library in branding_cfg.get('included_js_libraries') or []:
-                mappings[included_js_library] = ''
+            included_js_libraries = branding_cfg.get('included_js_libraries') or []
         for script_name in script_names:
             mapped_script_name = mappings.get(script_name, script_name)
-            if not mapped_script_name:
-                continue
             if mapped_script_name not in self.javascript_scripts:
                 if script_name == 'qommon.map.js':
                     self.add_javascript(['jquery.js'])
@@ -92,7 +91,8 @@ class HTTPResponse(quixote.http_response.HTTPResponse):
                             'jquery.fileupload.js',
                         ]
                     )
-                self.javascript_scripts.append(str(mapped_script_name))
+                if script_name not in included_js_libraries:
+                    self.javascript_scripts.append(str(mapped_script_name))
                 if script_name == 'afterjob.js':
                     self.add_javascript_code(
                         'var QOMMON_ROOT_URL = "%s";\n'
@@ -117,8 +117,10 @@ class HTTPResponse(quixote.http_response.HTTPResponse):
                 if script_name == 'qommon.admin.js':
                     self.add_javascript(['../../i18n.js', 'jquery.js', 'qommon.slugify.js'])
                 if script_name == 'select2.js':
-                    self.add_javascript(['jquery.js', '../../i18n.js', 'qommon.forms.js', 'select2.js'])
-                    self.add_css_include('select2.css')
+                    self.add_javascript(['jquery.js', '../../i18n.js', 'qommon.forms.js'])
+                    if script_name not in included_js_libraries:
+                        # assume a theme embedding select2.js will also include the css parts
+                        self.add_css_include('select2.css')
 
     def add_javascript_code(self, code):
         if not self.javascript_code_parts:

wcs/qommon/misc.py

@@ -1133,7 +1133,7 @@ def get_int_or_400(value):
 
 
 def get_order_by_or_400(value):
-    if value is None:
+    if value in (None, ''):
         return None
     if not re.match(r'-?[a-z0-9_-]+$', value):
         raise RequestError()

wcs/qommon/static/css/dc2/admin.scss

@@ -546,6 +546,32 @@ ul#status-list li a {
 	border-right: 5px solid transparent;
 }
 
+ul#status-list a.biglistitem--content {
+	display: flex;
+	justify-content: space-between;
+}
+
+.status-type {
+	&::before {
+		color: #505050;
+		font-size: 70%;
+		font-family: FontAwesome;
+		width: 2em;
+		display: block;
+		text-align: center;
+		cursor: help;
+	}
+	&--endpoint::before {
+		content: "\f04d";  /* stop */
+	}
+	&--waitpoint::before {
+		content: "\f04c";  /* pause */
+	}
+	&--transition::before {
+		content: "\f04e";  /* forward */
+	}
+}
+
 ul#evolutions span.time:before {
 	font-family: FontAwesome;
 	content: "\f017  "; /* clock-o */
@@ -2670,6 +2696,9 @@ div#main-content > h3.field-edit--subtitle {
 	display: flex;
 	align-items: baseline;
 	justify-content: space-between;
+	select {
+		max-width: 40ex;
+	}
 	fieldset {
 		border: none;
 		padding: 0;
@@ -3075,3 +3104,15 @@ ul.objects-list.single-links li.list-item-no-usage p {
 	padding: 0 0.5ex 0 2ex;
 	margin: 0;
 }
+
+.inline-hint-message {
+	&::before {
+		font-family: FontAwesome;
+		content: "\f24a";  /* sticky note */
+		margin-right: 0.3em;
+	}
+	margin-left: 0.7em;
+	background: #ffc;
+	border-radius: 0.3em;
+	padding: 0.5em;
+}

wcs/qommon/static/js/qommon.admin.js

@@ -463,4 +463,21 @@ $(function() {
             showRelatedObjectPopup(this);
         }
     });
+
+    const other_field_varnames_element = document.getElementById('other-fields-varnames')
+    const varname_field_widget = document.getElementById('form_varname')
+    if (other_field_varnames_element && varname_field_widget) {
+      const other_field_varnames = JSON.parse(other_field_varnames_element.textContent)
+      const message_span = document.querySelector('#form_varname + .inline-hint-message');
+      ['keyup', 'change'].forEach(event_type =>
+        varname_field_widget.addEventListener(event_type, function(event) {
+          if (other_field_varnames.indexOf(this.value) != -1) {
+            message_span.style.display = 'inline-block'
+          } else {
+            message_span.style.display = 'none'
+          }
+        })
+      )
+      varname_field_widget.dispatchEvent(new Event('keyup'))
+    }
 });

wcs/qommon/static/js/qommon.js

@@ -86,9 +86,9 @@ $(function() {
       if ($widgets.length > 1) {
         var values = $widgets.find('select').map((idx, elt) => {return $(elt).val()}).toArray().slice(1)
         if (values.every(v => (v === ""))) { // all empty
-          $widgets.find('select').first().find('option[value=""]').text($(this).attr('data-first-element-empty-label'));
+          $widgets.find('select').first().find('option[value=""]').first().text($(this).attr('data-first-element-empty-label'));
         } else {
-          $widgets.find('select').first().find('option[value=""]').text('---');
+          $widgets.find('select').first().find('option[value=""]').first().text('---');
         }
       }
     }).trigger('change');

wcs/qommon/static/js/wcs.listing.js

@@ -106,20 +106,17 @@ function prepare_row_links() {
 }
 
 function prepare_column_headers() {
-  if ($('input[name="order_by"]').length == 0) {
-    /* if we don't have an order_by field, that means we do not support server
-     * side sorting, so we abort here */
-    return;
-  }
   var current_key = $('input[name="order_by"]').val();
   var sort_key = null;
-  var reversed = true;
-  if (current_key[0] === '-') {
-    sort_key = current_key.substring(1);
-    reversed = true;
-  } else {
-    sort_key = current_key;
-    reversed = false;
+  var reversed = false;
+  if (current_key) {
+    if (current_key[0] === '-') {
+      sort_key = current_key.substring(1);
+      reversed = true;
+    } else {
+      sort_key = current_key;
+      reversed = false;
+    }
   }
   if (reversed) {
     $('#listing thead th[data-field-sort-key="' + sort_key + '"]').addClass('headerSortUp');
@@ -127,10 +124,12 @@ function prepare_column_headers() {
     $('#listing thead th[data-field-sort-key="' + sort_key + '"]').addClass('headerSortDown');
   }
   $('#listing thead th[data-field-sort-key]').addClass('header').click(function() {
-    new_key = $(this).data('field-sort-key');
-    if (current_key === sort_key) {
-      if (reversed === false) {
+    var new_key = $(this).data('field-sort-key');
+    if (sort_key === new_key) { // same column, reverse on second click, reset on third click
+      if (! reversed) {
         new_key = '-' + new_key
+      } else {
+        new_key = ''
       }
     }
     $('input[name="order_by"]').val(new_key);

wcs/sql.py

@@ -473,7 +473,6 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
 
     with atomic():
         table_name = get_formdef_table_name(formdef)
-        new_table = False
 
         cur.execute(
             '''SELECT COUNT(*) FROM information_schema.tables
@@ -505,7 +504,6 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
                                         formdata_id integer REFERENCES %s (id) ON DELETE CASCADE)'''
                 % (table_name, table_name)
             )
-            new_table = True
 
         # make sure the table will not be changed while we work on it
         cur.execute('LOCK TABLE %s;' % table_name)
@@ -525,7 +523,6 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
         if 'fts' not in existing_fields:
             # full text search, column and index
             cur.execute('''ALTER TABLE %s ADD COLUMN fts tsvector''' % table_name)
-            cur.execute('''CREATE INDEX %s_fts ON %s USING gin(fts)''' % (table_name, table_name))
 
         if 'criticality_level' not in existing_fields:
             # criticality leve, with default value
@@ -595,8 +592,7 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
             # them even if not asked to.
             redo_views(conn, cur, formdef, rebuild_global_views=rebuild_global_views)
 
-        if new_table:
-            do_formdef_indexes(formdef, created=True, conn=conn, cur=cur)
+        do_formdef_indexes(formdef, cur=cur)
 
         if own_conn:
             cur.close()
@@ -749,30 +745,26 @@ AS $${code}$$;
         )
 
 
-def do_formdef_indexes(formdef, created, conn, cur):
+def do_formdef_indexes(formdef, cur, concurrently=False):
     table_name = get_formdef_table_name(formdef)
     evolutions_table_name = table_name + '_evolutions'
 
-    create_index = 'CREATE INDEX IF NOT EXISTS'
+    if concurrently:
+        create_index = 'CREATE INDEX CONCURRENTLY IF NOT EXISTS'
+    else:
+        create_index = 'CREATE INDEX IF NOT EXISTS'
 
-    cur.execute(
-        '''%s %s_fid ON %s (formdata_id, id)''' % (create_index, evolutions_table_name, evolutions_table_name)
-    )
+    cur.execute(f'{create_index} {evolutions_table_name}_fid ON {evolutions_table_name} (formdata_id, id)')
+    cur.execute(f'{create_index} {table_name}_fts ON {table_name} USING gin(fts)')
 
     attrs = ['receipt_time', 'anonymised', 'user_id', 'status']
     if isinstance(formdef, CardDef):
         attrs.append('id_display')
     for attr in attrs:
-        cur.execute(
-            '%(create_index)s %(table_name)s_%(attr)s_idx ON %(table_name)s (%(attr)s)'
-            % {'create_index': create_index, 'table_name': table_name, 'attr': attr}
-        )
-    for attr in ('concerned_roles_array', 'actions_roles_array'):
+        cur.execute(f'{create_index} {table_name}_{attr}_idx ON {table_name} ({attr})')
+    for attr in ('concerned_roles_array', 'actions_roles_array', 'workflow_roles_array'):
         idx_name = 'idx_' + attr + '_' + table_name
-        cur.execute(
-            '%(create_index)s %(idx_name)s ON %(table_name)s USING gin (%(attr)s)'
-            % {'create_index': create_index, 'table_name': table_name, 'idx_name': idx_name, 'attr': attr}
-        )
+        cur.execute(f'{create_index} {idx_name} ON {table_name} USING gin ({attr})')
 
 
 def do_user_table():
@@ -858,7 +850,6 @@ def do_user_table():
     if 'fts' not in existing_fields:
         # full text search
         cur.execute('''ALTER TABLE %s ADD COLUMN fts tsvector''' % table_name)
-        cur.execute('''CREATE INDEX %s_fts ON %s USING gin(fts)''' % (table_name, table_name))
 
     if 'verified_fields' not in existing_fields:
         cur.execute('ALTER TABLE %s ADD COLUMN verified_fields text[]' % table_name)
@@ -880,10 +871,7 @@ def do_user_table():
     for field in existing_fields - needed_fields:
         cur.execute('''ALTER TABLE %s DROP COLUMN %s''' % (table_name, field))
 
-    # create indexes
-    cur.execute('CREATE INDEX IF NOT EXISTS users_name_idx ON users (name)')
-    cur.execute('CREATE INDEX IF NOT EXISTS users_name_identifiers_idx ON users USING gin(name_identifiers)')
-
+    SqlUser.do_indexes(cur)
     cur.close()
 
 
@@ -1002,12 +990,12 @@ def do_session_table():
     # migrations
     if 'last_update_time' not in existing_fields:
         cur.execute('''ALTER TABLE %s ADD COLUMN last_update_time timestamp DEFAULT NOW()''' % table_name)
-        cur.execute('''CREATE INDEX %s_ts ON %s (last_update_time)''' % (table_name, table_name))
 
     # delete obsolete fields
     for field in existing_fields - needed_fields:
         cur.execute('''ALTER TABLE %s DROP COLUMN %s''' % (table_name, field))
 
+    Session.do_indexes(cur)
     cur.close()
 
 
@@ -1095,12 +1083,7 @@ def do_custom_views_table():
     for field in existing_fields - needed_fields:
         cur.execute('''ALTER TABLE %s DROP COLUMN %s''' % (table_name, field))
 
-    # add indexes
-    cur.execute(
-        '''CREATE INDEX IF NOT EXISTS %s_formdef_type_id ON %s(formdef_type, formdef_id)'''
-        % (table_name, table_name)
-    )
-
+    CustomView.do_indexes(cur)
     cur.close()
 
 
@@ -1150,11 +1133,7 @@ def do_snapshots_table():
     for field in existing_fields - needed_fields:
         cur.execute('''ALTER TABLE %s DROP COLUMN %s''' % (table_name, field))
 
-    # add index
-    cur.execute(
-        '''CREATE INDEX IF NOT EXISTS %s_object_by_date ON %s(object_type, object_id, timestamp DESC)'''
-        % (table_name, table_name)
-    )
+    Snapshot.do_indexes(cur)
     cur.close()
 
 
@@ -1209,15 +1188,7 @@ def do_loggederrors_table():
     for field in existing_fields - needed_fields:
         cur.execute('''ALTER TABLE %s DROP COLUMN %s''' % (table_name, field))
 
-    create_index = 'CREATE INDEX IF NOT EXISTS'
-
-    # build indexes
-    for attr in ('formdef_id', 'workflow_id'):
-        cur.execute(
-            '%(create_index)s %(table_name)s_%(attr)s_idx ON %(table_name)s (%(attr)s)'
-            % {'create_index': create_index, 'table_name': table_name, 'attr': attr}
-        )
-
+    LoggedError.do_indexes(cur)
     cur.close()
 
 
@@ -1538,19 +1509,15 @@ def do_global_views(conn, cur):
         , PRIMARY KEY(formdef_id, id)
     )"""
     )
-    cur.execute(
-        '''CREATE INDEX IF NOT EXISTS %s_fts ON %s USING gin(fts)''' % ('wcs_all_forms', 'wcs_all_forms')
-    )
-
+    create_index = 'CREATE INDEX IF NOT EXISTS'
     for attr in ('receipt_time', 'anonymised', 'user_id', 'status'):
-        cur.execute(
-            '''CREATE INDEX IF NOT EXISTS %s_%s ON %s (%s)''' % ('wcs_all_forms', attr, 'wcs_all_forms', attr)
-        )
-    for attr in ('concerned_roles_array', 'actions_roles_array'):
-        cur.execute(
-            '''CREATE INDEX IF NOT EXISTS %s_%s ON %s USING gin (%s)'''
-            % ('wcs_all_forms', attr, 'wcs_all_forms', attr)
-        )
+        cur.execute(f'{create_index} wcs_all_forms_{attr} ON wcs_all_forms ({attr})')
+    for attr in ('fts', 'concerned_roles_array', 'actions_roles_array'):
+        cur.execute(f'{create_index} wcs_all_forms_{attr} ON wcs_all_forms USING gin({attr})')
+    cur.execute(
+        f'''{create_index} wcs_all_forms_actions_roles_live ON wcs_all_forms
+            USING gin(actions_roles_array) WHERE (anonymised IS NULL AND is_at_endpoint = false)'''
+    )
 
     # make sure the table will not be changed while we work on it
     with atomic():
@@ -1680,6 +1647,20 @@ class SqlMixin:
     _numerical_id = True
     _table_select_skipped_fields = []
     _has_id = True
+    _sql_indexes = None
+
+    @classmethod
+    def do_indexes(cls, cur, concurrently=False):
+        if concurrently:
+            create_index = 'CREATE INDEX CONCURRENTLY IF NOT EXISTS'
+        else:
+            create_index = 'CREATE INDEX IF NOT EXISTS'
+        for index in cls.get_sql_indexes():
+            cur.execute(f'{create_index} {index}')
+
+    @classmethod
+    def get_sql_indexes(cls):
+        return cls._sql_indexes or []
 
     @classmethod
     def keys(cls, clause=None):
@@ -2365,7 +2346,8 @@ class SqlDataMixin(SqlMixin):
             cur.execute(sql_statement, params)
             results = cur.fetchall()
 
-        return [res_time for row in results if (res_time := row[1].total_seconds()) > 0]
+        # row[1] will have the resolution time as computed by postgresql
+        return [row[1].total_seconds() for row in results if row[1].total_seconds() >= 0]
 
     def _set_auto_fields(self, cur):
         if self.set_auto_fields():
@@ -2840,6 +2822,12 @@ class SqlUser(SqlMixin, wcs.users.User):
         ('is_active', 'bool'),
         ('preferences', 'jsonb'),
     ]
+    _sql_indexes = [
+        'users_name_idx ON users (name)',
+        'users_name_identifiers_idx ON users USING gin(name_identifiers)',
+        'users_fts ON users USING gin(fts)',
+        'users_roles_idx ON users USING gin(roles)',
+    ]
 
     id = None
 
@@ -3217,6 +3205,9 @@ class Session(SqlMixin, wcs.sessions.BasicSession):
         ('session_data', 'bytea'),
     ]
     _numerical_id = False
+    _sql_indexes = [
+        'sessions_ts ON sessions (last_update_time)',
+    ]
 
     @classmethod
     def select_recent_with_visits(cls, seconds=30 * 60, **kwargs):
@@ -3468,6 +3459,9 @@ class CustomView(SqlMixin, wcs.custom_views.CustomView):
         ('columns', 'jsonb'),
         ('filters', 'jsonb'),
     ]
+    _sql_indexes = [
+        'custom_views_formdef_type_id ON custom_views (formdef_type, formdef_id)',
+    ]
 
     @invalidate_substitution_cache
     def store(self):
@@ -3553,6 +3547,9 @@ class Snapshot(SqlMixin, wcs.snapshots.Snapshot):
         ('application_version', 'varchar'),
     ]
     _table_select_skipped_fields = ['serialization', 'patch']
+    _sql_indexes = [
+        'snapshots_object_by_date ON snapshots (object_type, object_id, timestamp DESC)',
+    ]
 
     @invalidate_substitution_cache
     def store(self):
@@ -3691,6 +3688,10 @@ class LoggedError(SqlMixin, wcs.logged_errors.LoggedError):
         ('first_occurence_timestamp', 'timestamptz'),
         ('latest_occurence_timestamp', 'timestamptz'),
     ]
+    _sql_indexes = [
+        'loggederrors_formdef_id_idx ON loggederrors (formdef_id)',
+        'loggederrors_workflow_id_idx ON loggederrors (workflow_id)',
+    ]
 
     @invalidate_substitution_cache
     def store(self):
@@ -3824,6 +3825,9 @@ class TranslatableMessage(SqlMixin):
         ('last_update_time', 'timestamptz'),
         ('translatable', 'boolean'),
     ]
+    _sql_indexes = [
+        'translatable_messages_fts ON translatable_messages USING gin(fts)',
+    ]
 
     id = None
 
@@ -3866,8 +3870,7 @@ class TranslatableMessage(SqlMixin):
             if field not in existing_fields:
                 cur.execute('ALTER TABLE %s ADD COLUMN %s VARCHAR' % (table_name, field))
 
-        cur.execute('''CREATE INDEX IF NOT EXISTS %s_fts ON %s USING gin(fts)''' % (table_name, table_name))
-
+        cls.do_indexes(cur)
         cur.close()
 
     @classmethod
@@ -4125,6 +4128,9 @@ class WorkflowTrace(SqlMixin):
         ('action_item_key', 'varchar'),
         ('action_item_id', 'varchar'),
     ]
+    _sql_indexes = [
+        'workflow_traces_idx ON workflow_traces (formdef_type, formdef_id, formdata_id)',
+    ]
 
     id = None
     formdef_type = None
@@ -4176,11 +4182,7 @@ class WorkflowTrace(SqlMixin):
                 % table_name
             )
 
-        cur.execute(
-            '''CREATE INDEX IF NOT EXISTS workflow_traces_idx
-               ON workflow_traces (formdef_type, formdef_id, formdata_id)'''
-        )
-
+        cls.do_indexes(cur)
         cur.close()
 
     def store(self):
@@ -4298,6 +4300,10 @@ class Audit(SqlMixin):
         ('extra_data', 'jsonb'),
         ('frozen', 'jsonb'),  # plain copy of user email, object name and slug
     ]
+    _sql_indexes = [
+        'audit_id_idx ON audit USING btree (id)',
+    ]
+
     id = None
 
     @classmethod
@@ -4341,8 +4347,7 @@ class Audit(SqlMixin):
         for field in existing_fields - needed_fields:
             cur.execute('''ALTER TABLE %s DROP COLUMN %s''' % (table_name, field))
 
-        cur.execute('CREATE INDEX IF NOT EXISTS audit_id_idx ON audit USING btree (id)')
-
+        cls.do_indexes(cur)
         cur.close()
 
     def store(self):
@@ -4410,6 +4415,9 @@ class Application(SqlMixin):
         ('created_at', 'timestamptz'),
         ('updated_at', 'timestamptz'),
     ]
+    _sql_indexes = [
+        'applications_slug ON applications (slug)',
+    ]
 
     id = None
 
@@ -4441,8 +4449,7 @@ class Application(SqlMixin):
                                    )'''
                 % table_name
             )
-        cur.execute('CREATE UNIQUE INDEX IF NOT EXISTS %s_slug ON %s (slug)' % (table_name, table_name))
-
+        cls.do_indexes(cur)
         cur.close()
 
     def store(self):
@@ -4498,6 +4505,9 @@ class ApplicationElement(SqlMixin):
         ('created_at', 'timestamptz'),
         ('updated_at', 'timestamptz'),
     ]
+    _sql_indexes = [
+        'application_elements_object_idx ON application_elements (object_type, object_id)',
+    ]
 
     id = None
 
@@ -4524,10 +4534,7 @@ class ApplicationElement(SqlMixin):
                 % table_name
             )
 
-        cur.execute(
-            'CREATE INDEX IF NOT EXISTS %s_object_idx ON %s (object_type, object_id)'
-            % (table_name, table_name)
-        )
+        cls.do_indexes(cur)
         cur.execute(
             '''SELECT COUNT(*) FROM information_schema.constraint_column_usage
                         WHERE table_name = %s
@@ -4684,6 +4691,9 @@ def get_period_query(
 
 class SearchableFormDef(SqlMixin):
     _table_name = 'searchable_formdefs'
+    _sql_indexes = [
+        'searchable_formdefs_fts ON searchable_formdefs USING gin(fts)',
+    ]
 
     @classmethod
     @atomic
@@ -4710,9 +4720,7 @@ class SearchableFormDef(SqlMixin):
                 'ALTER TABLE %s ADD CONSTRAINT %s_unique UNIQUE (object_type, object_id)'
                 % (cls._table_name, cls._table_name)
             )
-        cur.execute(
-            '''CREATE INDEX IF NOT EXISTS %s_fts ON %s USING gin(fts)''' % (cls._table_name, cls._table_name)
-        )
+        cls.do_indexes(cur)
         cur.close()
 
         from wcs.carddef import CardDef
@@ -5024,7 +5032,7 @@ def get_period_total(
 # latest migration, number + description (description is not used
 # programmaticaly but will make sure git conflicts if two migrations are
 # separately added with the same number)
-SQL_LEVEL = (98, 'add index on carddata/id_display')
+SQL_LEVEL = (100, 'always create translation messages table')
 
 
 def migrate_global_views(conn, cur):
@@ -5188,9 +5196,10 @@ def migrate():
         # 67: re-migrate legacy tokens
         do_tokens_table()
         migrate_legacy_tokens()
-    if sql_level < 79:
+    if sql_level < 100:
         # 68: multilinguism
         # 79: add translatable column to TranslatableMessage table
+        # 100: always create translation messages table
         TranslatableMessage.do_table()
     if sql_level < 87:
         # 72: add testdef table
@@ -5268,7 +5277,7 @@ def migrate():
         # 62: use setweight on formdata & user indexation (reapply)
         # 96: change to fts normalization
         set_reindex('formdata', 'needed', conn=conn, cur=cur)
-    if sql_level < 98:
+    if sql_level < 99:
         # 24: add index on evolution(formdata_id)
         # 35: add indexes on formdata(receipt_time) and formdata(anonymised)
         # 36: add index on formdata(user_id)
@@ -5276,8 +5285,8 @@ def migrate():
         # 56: add GIN indexes to concerned_roles_array and actions_roles_array
         # 74: (late migration) change evolution index to be on (fomdata_id, id)
         # 97&98: add index on carddata/id_display
-        for formdef in FormDef.select() + CardDef.select():
-            do_formdef_indexes(formdef, created=False, conn=conn, cur=cur)
+        # 99: add more indexes
+        set_reindex('sqlindexes', 'needed', conn=conn, cur=cur)
     if sql_level < 30:
         # 30: actually remove evo.who on anonymised formdatas
         for formdef in FormDef.select():
@@ -5303,9 +5312,10 @@ def migrate():
             do_formdef_tables(formdef, rebuild_views=False, rebuild_global_views=False)
         migrate_views(conn, cur)
         set_reindex('formdata', 'needed', conn=conn, cur=cur)
-    if sql_level < 82:
+    if sql_level < 99:
         # 81: add statistics data column to wcs_all_forms
         # 82: add statistics data column to wcs_all_forms, for real
+        # 99: add more indexes
         migrate_global_views(conn, cur)
     if sql_level < 60:
         # 59: switch wcs_all_forms to a trigger-maintained table
@@ -5353,6 +5363,23 @@ def migrate():
 def reindex():
     conn, cur = get_connection_and_cursor()
 
+    if is_reindex_needed('sqlindexes', conn=conn, cur=cur):
+        for klass in (
+            SqlUser,
+            Session,
+            CustomView,
+            Snapshot,
+            LoggedError,
+            TranslatableMessage,
+            WorkflowTrace,
+            Audit,
+            Application,
+            ApplicationElement,
+        ):
+            klass.do_indexes(cur, concurrently=True)
+        for formdef in FormDef.select() + CardDef.select():
+            do_formdef_indexes(formdef, cur=cur, concurrently=True)
+
     if is_reindex_needed('user', conn=conn, cur=cur):
         for user in SqlUser.select(iterator=True):
             user.store()

wcs/sql_criterias.py

@@ -456,3 +456,15 @@ class StatusReachedTimeoutCriteria(Criteria):
             WHERE {formdef_evolution_table}.formdata_id = {formdef_table}.id 
                 AND {formdef_evolution_table}.status = ANY(%({statuses})s)
                 AND {formdef_evolution_table}.time <= NOW() - {duration} * interval '1 day')'''
+
+
+class ArrayPrefixMatch(Criteria):
+    def __init__(self, attribute, value, **kwargs):
+        value = value + '%'
+        super().__init__(attribute, value, **kwargs)
+
+    def as_sql(self):
+        return '''exists (select 1 from unnest(%s) v where v LIKE %%(c%s)s)''' % (
+            self.attribute,
+            id(self.value),
+        )

wcs/statistics/views.py

@@ -510,7 +510,7 @@ class FormsCountView(RestrictedView):
         if fields:
             return fields[0]
 
-    def get_group_labels(self, group_by_field, formdef, group_by):
+    def get_group_labels(self, formdef, group_by):
         group_labels = {}
         if group_by == 'status':
             group_labels = {'wf-%s' % status.id: status.name for status in formdef.workflow.possible_status}
@@ -521,11 +521,11 @@ class FormsCountView(RestrictedView):
                     group_labels['wf-%s' % status.id] = _('Done')
                 else:
                     group_labels['wf-%s' % status.id] = _('In progress')
-        elif group_by_field.key == 'bool':
+        elif formdef.group_by_field.key == 'bool':
             group_labels = {True: _('Yes'), False: _('No')}
-        elif group_by_field.key in ('item', 'items'):
+        elif formdef.group_by_field.key in ('item', 'items'):
             options = formdef.form_page.get_item_filter_options(
-                group_by_field, selected_filter='all', anonymised=True
+                formdef.group_by_field, selected_filter='all', anonymised=True
             )
             group_labels = {option[0]: option[1] for option in options}
 
@@ -551,23 +551,25 @@ class FormsCountView(RestrictedView):
             group_labels['backoffice'] = _('Backoffice')
             return
         elif group_by == 'simple-status':
-            group_by_field = self.get_group_by_field(formdefs[0].form_page, 'status')
+            group_by_key = 'status'
         else:
-            group_by_field = self.get_group_by_field(formdefs[0].form_page, group_by)
+            group_by_key = group_by
 
-        if not group_by_field:
-            return
+        for formdef in formdefs:
+            formdef.group_by_field = self.get_group_by_field(formdef.form_page, group_by_key)
+            if not formdef.group_by_field:
+                return
 
-        if group_by_field.key == 'status':
+        if group_by_key == 'status':
             totals_kwargs['group_by'] = 'status'
         else:
-            totals_kwargs['group_by'] = "statistics_data->'%s'" % group_by_field.varname
+            totals_kwargs['group_by'] = "statistics_data->'%s'" % formdefs[0].group_by_field.varname
 
         if self.request.GET.get('hide_none_label') == 'true':
             totals_kwargs['criterias'].append(StrictNotEqual(totals_kwargs['group_by'], '[]'))
 
         for formdef in formdefs:
-            group_labels.update(self.get_group_labels(group_by_field, formdef, group_by))
+            group_labels.update(self.get_group_labels(formdef, group_by))
 
     def get_grouped_time_data(self, totals, group_labels):
         totals_by_time = collections.OrderedDict(

wcs/templates/wcs/backoffice/i18n.html

@@ -16,9 +16,27 @@
   <form action="." class="i18n-filter-form">
     <span>
       <label>{% trans "Search:" %} <input type="search" name="q" value="{{q|default:""}}"></label>
+      <select name="formdef">
+        <option value="">{% trans "All forms and card models" %}</option>
+        <optgroup label="{% trans "Forms" %}">
+          {% for formdef in formdefs %}
+            {% with option="forms/"|add:formdef.id %}
+              <option value="{{ option }}" {% if selected_formdef == option %}selected{% endif %}>{{ formdef.name|truncatechars:60 }}</option>
+            {% endwith %}
+          {% endfor %}
+        </optgroup>
+        <optgroup label="{% trans "Card models" %}">
+          {% for carddef in carddefs %}
+            {% with option="cards/"|add:carddef.id %}
+              <option value="{{ option }}" {% if selected_formdef == option %}selected{% endif %}>{{ carddef.name|truncatechars:60 }}</option>
+            {% endwith %}
+          {% endfor %}
+        </optgroup>
+      </select>
       <label><input name="non_translatable" type="checkbox"
                     {% if non_translatable %}checked{% endif %}
-        >{% trans "Search non-translatable strings" %}</label>
+        >{% trans "Include non-translatable strings" %}</label>
+      <button type="submit">{% trans "Filter" %}</button>
     </span>
     <fieldset class="radiogroup"><legend>{% trans "Language:" %}</legend>
       {% for language in supported_languages %}

wcs/templates/wcs/backoffice/snapshots.html

@@ -56,7 +56,7 @@
                   <td colspan="2">
                     <a href="{{snapshot.id}}/view/">{% trans "View" %}</a>
                     —
-                    <a data-popup href="{{snapshot.id}}/restore">{% trans "Restore" %}</a>
+                    <a data-popup {% if forloop.first %}class="disabled" href="#"{% else %}href="{{snapshot.id}}/restore"{% endif %}>{% trans "Restore" %}</a>
                     —
                     <a href="{{snapshot.id}}/export">{% trans "Export" %}</a>
                   </td>

wcs/templates/wcs/backoffice/test-result-detail.html

@@ -0,0 +1,26 @@
+{% extends "wcs/backoffice.html" %}
+{% load i18n %}
+
+{% block appbar-title %}
+  {% blocktrans with test_name=result.name %}Details of {{ test_name }}{% endblocktrans %}
+{% endblock %}
+
+{% block body %}
+  <div class="section">
+    <ul>
+      {% if result.recorded_errors %}
+        <li>{% trans "Recorded errors:" %}</li>
+        <ul>
+          {% for error in result.recorded_errors %}
+            <li>{{ error }}</li>
+          {% endfor %}
+        </ul>
+      {% endif %}
+      {% if result.missing_required_fields %}
+        <li>
+          {% trans "Missing required fields:" %} {{ result.missing_required_fields|join:"," }}
+        </li>
+      {% endif %}
+    </ul>
+  </div>
+{% endblock %}

wcs/templates/wcs/backoffice/test-result.html

@@ -20,13 +20,17 @@
         <thead>
           <th>{% trans "Name" %}</th>
           <th>{% trans "Result" %}</th>
+          <th>{% trans "Details" %}</th>
         </thead>
         <tbody>
           {% for test in test_result.results %}
             <tr>
               <td><a {% if test.url %}href="{{ test.url }}"{% else %}disabled{% endif %}>{{ test.name }}</a></td>
-              <td title="{% trans "Missing required fields:" %} {{ test.missing_required_fields|length }}">
-                {% firstof test.error _("Success!") %}
+              <td>{% firstof test.error _("Success!") %}</td>
+              <td>
+                {% if test.missing_required_fields or test.recorded_errors %}
+                  <a rel="popup" data-selector="div.section" href="{{ forloop.counter0 }}/">{% trans "Display details" %}</a>
+                {% endif %}
               </td>
             </tr>
           {% endfor %}

wcs/templates/wcs/backoffice/workflow.html

@@ -40,11 +40,19 @@
         {% endif %}
         {% spaceless %}
           {% for status in workflow.possible_status %}
-            <li class="biglistitem {{ status.get_visibility_mode }}-status"
+            <li class="biglistitem {{ status.get_visibility_mode }}-status
+                       status-type-{% if status.is_endpoint %}endpoint{% elif status.is_waitpoint %}waitpoint{% else %}transition{% endif %}"
                 data-id="{{ status.id }}">
               <a class="biglistitem--content" href="status/{{ status.id }}/"
                  {% if status.colour %}style="border-color: {{status.colour}}"{% endif %}
-              >{{ status.name }}</a></li>
+              >{{ status.name }}
+                {% if status.is_endpoint %}<span class="status-type status-type--endpoint" title="{% trans "final status" %}"
+                  ><span class="sr-only">({% trans "final status" %})</span></span>
+                {% elif status.is_waitpoint %}<span class="status-type status-type--waitpoint" title="{% trans "pause status" %}"
+                  ><span class="sr-only">({% trans "pause status" %})</span></span>
+                {% else %}<span class="status-type status-type--transition" title="{% trans "transition status" %}"
+                  ><span class="sr-only">({% trans "transition status" %})</span></span>{% endif %}
+              </a></li>
           {% endfor %}
         {% endspaceless %}
         </ul>

wcs/testdef.py

@@ -144,8 +144,8 @@ class TestDef(sql.TestDef):
 
     @contextmanager
     def fake_request(self):
-        def record_error(self, error_summary=None, context=None, exception=None, *args, **kwargs):
-            raise exception
+        def record_error(error_summary=None, exception=None, *args, **kwargs):
+            self.recorded_errors.append(error_summary or str(exception))
 
         real_record_error = get_publisher().record_error
 
@@ -163,6 +163,7 @@ class TestDef(sql.TestDef):
             get_publisher().record_error = real_record_error
 
     def run(self, objectdef):
+        self.recorded_errors = []
         self.missing_required_fields = []
         with self.fake_request():
             try:

wcs/wf/backoffice_fields.py

@@ -40,11 +40,24 @@ class SetBackofficeFieldRowWidget(CompositeWidget):
         if not value:
             value = {}
 
+        label_counters = {}
+        for field in workflow.get_backoffice_fields():
+            if not issubclass(field.__class__, WidgetField):
+                continue
+            label = f'{field.label} - {field.get_type_label()}'
+            label_counters.setdefault(label, 0)
+            label_counters[label] += 1
+        repeated_labels = {x for x, y in label_counters.items() if y > 1}
+
         fields = [('', '', '')]
         for field in workflow.get_backoffice_fields():
             if not issubclass(field.__class__, WidgetField):
                 continue
-            fields.append((field.id, f'{field.label} - {field.get_type_label()}', field.id))
+            label = f'{field.label} - {field.get_type_label()}'
+            if label in repeated_labels and field.varname:
+                label = f'{field.label} - {field.get_type_label()} ({field.varname})'
+            fields.append((field.id, label, field.id))
+
         self.add(
             SingleSelectWidget,
             name='field_id',

wcs/wf/create_formdata.py

@@ -74,15 +74,27 @@ class MappingWidget(CompositeWidget):
         )
 
     def _fields_to_options(self, formdef):
+        label_counters = {}
+        for field in formdef.get_widget_fields():
+            label = f'{field.label} - {field.get_type_label()}'
+            label_counters.setdefault(label, 0)
+            label_counters[label] += 1
+        repeated_labels = {x for x, y in label_counters.items() if y > 1}
+
         options = [(None, '---', '')]
         for field in formdef.get_widget_fields():
-            options.append((field.id, field.label, str(field.id)))
+            label = f'{field.label} - {field.get_type_label()}'
+            block_label = field.label
+            if label in repeated_labels and field.varname:
+                label = f'{field.label} - {field.get_type_label()} ({field.varname})'
+                block_label = f'{field.label} ({field.varname})'  # do not repeat block type
+            options.append((field.id, label, str(field.id)))
             if field.key == 'block':
                 for subfield in field.block.get_widget_fields():
                     options.append(
                         (
                             f'{field.id}${subfield.id}',
-                            f'{field.label} - {subfield.label}',
+                            f'{block_label} - {subfield.label} - {subfield.get_type_label()}',
                             f'{field.id}${subfield.id}',
                         )
                     )
@@ -724,6 +736,7 @@ class CreateFormdataWorkflowStatusItem(WorkflowStatusItem):
 
         if self.draft:
             new_formdata.status = 'draft'
+            new_formdata.receipt_time = time.localtime()
             new_formdata.store()
             if formdef.enable_tracking_codes:
                 code.formdata = new_formdata  # this will .store() the code

wcs/wf/display_message.py

@@ -21,7 +21,12 @@ from quixote import get_publisher
 from quixote.html import htmltext
 
 from wcs.qommon import _, ezt, misc
-from wcs.qommon.form import ComputedExpressionWidget, SingleSelectWidget, TextWidget, WidgetListOfRoles
+from wcs.qommon.form import (
+    ComputedExpressionWidget,
+    SingleSelectWidget,
+    WidgetListOfRoles,
+    get_rich_text_widget_class,
+)
 from wcs.qommon.template import Template
 from wcs.workflows import WorkflowGlobalAction, WorkflowStatusItem, register_item_class
 
@@ -107,7 +112,7 @@ class DisplayMessageWorkflowStatusItem(WorkflowStatusItem):
         in_global_action = isinstance(self.parent, WorkflowGlobalAction)
         if 'message' in parameters:
             form.add(
-                TextWidget,
+                get_rich_text_widget_class(self.message),
                 '%smessage' % prefix,
                 title=_('Message'),
                 value=self.message,

wcs/wf/resubmit.py

@@ -14,6 +14,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+import time
+
 from quixote import get_publisher, get_request, get_session
 
 from wcs.formdef import FormDef
@@ -79,6 +81,7 @@ class ResubmitWorkflowStatusItem(WorkflowStatusItem):
             formdef = FormDef.get_by_urlname(self.formdef_slug)
         new_formdata = formdef.data_class()()
         new_formdata.status = 'draft'
+        new_formdata.receipt_time = time.localtime()
         new_formdata.user_id = formdata.user_id
         new_formdata.submission_context = (formdata.submission_context or {}).copy()
         new_formdata.submission_channel = formdata.submission_channel

wcs/workflows.py

@@ -1303,7 +1303,13 @@ class Workflow(StorableObject):
             # use empty string instead of None so it's not automatically
             # picked as default value by the browser
             t.append(('', '----', ''))
-            t.extend(get_user_roles())
+            existing_labels = {str(x[1]) for x in t}
+            t.extend(
+                [
+                    (x[0], x[1] if x[1] not in existing_labels else '%s [%s]' % (x[1], _('role')), x[2])
+                    for x in get_user_roles()
+                ]
+            )
         return t
 
     def get_add_role_label(self):
@@ -2070,16 +2076,21 @@ class WorkflowGlobalActionTimeoutTrigger(WorkflowGlobalActionTrigger):
         endpoint_status = workflow.get_endpoint_status()
         endpoint_status_ids = ['wf-%s' % x.id for x in endpoint_status]
 
-        # check "finalized" trigger with dedicated SQL criteria
-        finalized_triggers = [
+        # check "optimized" trigger with dedicated SQL criteria
+        optimized_triggers = [
             (action, trigger)
             for action, trigger in triggers
-            if trigger.anchor == 'finalized' and 'form_var' not in str(trigger.timeout)
+            if 'form_var' not in str(trigger.timeout)
+            and (
+                (trigger.anchor == 'finalized')
+                or (trigger.anchor in '1st-arrival' and trigger.anchor_status_first)
+                or (trigger.anchor in 'latest-arrival' and trigger.anchor_status_latest)
+            )
         ]
         triggers = [
-            (action, trigger) for action, trigger in triggers if (action, trigger) not in finalized_triggers
+            (action, trigger) for action, trigger in triggers if (action, trigger) not in optimized_triggers
         ]
-        for action, trigger in finalized_triggers:
+        for action, trigger in optimized_triggers:
             formdef_timeouts = {}
             for formdef in itertools.chain(workflow.formdefs(), workflow.carddefs()):
                 get_publisher().reset_formdata_state()
@@ -2099,9 +2110,14 @@ class WorkflowGlobalActionTimeoutTrigger(WorkflowGlobalActionTrigger):
                 criterias = [StrictNotEqual('status', 'draft'), Null('anonymised')]
                 # now we need the criteria for our timeout
                 trigger_timeout = formdef_timeouts[f'{formdef.xml_root_node}/{formdef.id}']
-                criterias.append(
-                    StatusReachedTimeoutCriteria(data_class, endpoint_status_ids, trigger_timeout)
-                )
+                # limit to forms/cards with appropriate status in their history
+                if trigger.anchor == 'finalized':
+                    status_ids = endpoint_status_ids
+                elif trigger.anchor == '1st-arrival':
+                    status_ids = [trigger.anchor_status_first]
+                elif trigger.anchor == 'latest-arrival':
+                    status_ids = [trigger.anchor_status_latest]
+                criterias.append(StatusReachedTimeoutCriteria(data_class, status_ids, trigger_timeout))
                 cls.run_trigger_check(
                     formdef,
                     triggers=[(action, trigger)],
@@ -2335,7 +2351,7 @@ class SerieOfActionsMixin:
 
     def handle_form(self, form, filled, user, evo):
         evo.time = time.localtime()
-        evo.set_user(formdata=filled, user=user)
+        evo.set_user(formdata=filled, user=user, check_submitter=get_request().is_in_frontoffice())
         if not filled.evolution:
             filled.evolution = []